#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/TEST_SCRIPTS/CMODE/NADQ02_FC_Disk_FW_Download_Background.thpl#1 $
#
## @summary Mode
##  Disk Firmware Download in background/foreground mode
#
##
## @description
## Verify that the drive can download disk firmware in foreground and background mode properly
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground
##
## @steps
## The test will execute steps mentioned below:
##
##
## @status  Automated
########################################
### Library functions
##########################################
use strict;
use Storage::Tahiti_Common_Lib;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
  $SATA_SETUP
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$SATA_SETUP     = $params->get( 'SATA_SETUP',     default => 'n' );
################
# Testcase name
################
my $TC_name;
$TC_name = "408_NADQ02_NVMe_Th_Ping_Pong";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home = $ENV{HOME};

my $test_wait_time = 12 * 3600;
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( Disk_FW_Ping_Pong => "Disk_FW_Ping_Pong." );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

    version_test( node_present => \@Nodes, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode,
        Tahiti          => "yes"
    );

    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Host        = host($FILER_C);
        logcomment("disable autogiveback ");
        my $prompts_answers = [ 'Do you want to disable auto-giveback?' => 'y', 'Do you want to continue' => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback false", 'connectrec-match_table' => $prompts_answers );
        logcomment("Disable background firmwar update option");
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable off" );
        logcomment("Stop firmware update command");
        try {
            my $prompts = [ '.*Are you sure you want to.*' => 'y', '.*stop firmware update qualification.*' => 'y' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk_fw_update -S\"", 'connectrec-match_table' => $prompts);
            logcomment("Firmware Qualification download stopped ");
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            logcomment("FW Download stop test Failed.");
        };
        logcomment("$FILER_C : Turn off the raid options");
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.enable off" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.spares.enable off" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.scrub.enable off" );
    }

    #Checking for firmware files; if not exist, then making the test NORUN - BURT1042571
    my $disk_dir      = "$Home/NDATE/FIRMWARE/DISKFW/HIGH";
    my $firmware_list = tahiti_map_fw_dsk_rev_model(
        Node             => $Nodes[0],
        disk_fw_location => $disk_dir,
    );
    my %firmware_hash = %$firmware_list;
    my %hash;
    my ( $key, $value );
    while ( ( $key, $value ) = each %firmware_hash ) {
        push( @{ $hash{$value} }, $key );
    }
    my @drive_list = grep { (/\S/) } keys(%firmware_hash);
    logcomment("The list of drives on which firmware will download : @drive_list ");
    my @firmware_list = unique( values(%firmware_hash) );
    logcomment("The list of firmware files present : @firmware_list ");
    if ( !@firmware_list ) {
        logcomment("**NORUN**: NO FIRMWARE FILES ARE AVAILABALE FOR FILER");
        return $TCD::FAIL;
    }

    #Checking for NT version disks on Filer , then making the test NORUN - BURT1134959
    logcomment("Checking for NT revision on disks on filer , making no run if found");
    my ( $status, $firmware_hash ) = check_disk_firmware_revision_number(
        Node             => $Nodes[0],
        disk_fw_location => $disk_dir,
        timeout          => 1200,
    );
    if ( $status == 0 ) {
        return $TCD::FAIL;
    } elsif ( $status == 1 ) {
        return $TCD::FAIL;
    } elsif ( $status == 2 ) {
        return $TCD::FAIL;
    } elsif ( $status == 3 ) {
        return $TCD::FAIL;
    } else {
        logcomment(" All Firmwares are having valid revision numbers ");
    }

    return $TCD::PASS;
}

sub Disk_FW_Ping_Pong {
    my @subtests;
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&Disk_FW_Ping_Pong_sub, -runid, "Disk_FW_Ping_Pong_sub_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $status = status_return( subtest_ref => [@subtests] );
    logcomment("Total test status is : $status");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub Disk_FW_Ping_Pong_sub {
    my $test_status = 0;
    $Test->description("Disk_FW_Download Ping Pong");
    my ( @Nodes, %filer_nvol );
    my $Home = $ENV{HOME};
    my $total_drv_count_s2;

    my %scsi_result_s3  = {};
    my %scsi_result_s11 = {};

    my @valid_drvs;
    my @serial_nums;
    my $drv_fw_mod_lst = {};
    my $shelf;

    push( @Nodes, shift(@_) );
    logcomment("Total Nodes  - @Nodes");
    logcomment( "Filer passed to the subtest :: " . $Nodes[0]->name() );

    #Checking for non root volumes
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $vol = nvme_check_non_root_aggr( Node => $Node );
        logcomment("Non -root volumes are ");
    }

    #STEP 0 - Copy low and Hig firlware files to filer

    #STEP 1
    tahiti_eye_catcher( Test => $Test, string => "STEP 1 of 12: List all drives " );
    my $disk_list = tahiti_list_only_tahiti_drv( node_present => [ $Nodes[0] ] );
    my @dsk_list = keys %$disk_list;
    $total_drv_count_s2 = scalar(@dsk_list);

    #STEP 2
    tahiti_eye_catcher( Test => $Test, string => "STEP 2 of 12:  Check to see if there is any unusual increase in the drive error logs" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        $scsi_result_s3{$FILER_C} = scsi_disk_errors( Node => $Node );
        logcomment("Record the log sense data from each drive completed : $FILER_C");
    }

    #STEP 3
    tahiti_eye_catcher( Test => $Test, string => "STEP 3 of 12:  Record device information before testing. All device in the system" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Get Drive owned by $FILER_C");
        my @drv_own = tahiti_get_disk_owned( Node => $Node );
        logcomment("Drives @drv_own belongs to  $FILER_C");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );

        my @sel_drv;

        foreach my $disk_t (@drv_own) {
            if ( $disk_t =~ /P/ ) {
                ( $disk_t = $disk_t ) =~ s/P\d//g;
                push( @sel_drv, $disk_t );
            }
            ( my $mydisk = $disk_t ) =~ s/$FILER_C\://g;
            push( @sel_drv, $disk_t );
        }

        @sel_drv = unique(@sel_drv);

        foreach my $drv (@sel_drv) {
            logcomment("Excute scsi inquiry on $drv");
            my $output = $Api_Set_Obj->execute_command( 'command' => "priv set -q test;scsi inquiry $drv" );
        }

    }

    #STEP 4
    tahiti_eye_catcher( Test => $Test, string => "STEP 4 of 12: Record HDD FRU information for all HDD FRU in the system" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $stsb_status;
        if ( $SATA_SETUP eq 'Y' ) {
            logcomment("Checking for stsb/acorn command is available ");
            $stsb_status = get_filer_check( Node => $Node );
            chomp($stsb_status);
            logcomment("DEBUG :$stsb_status");
            my @drv_own = tahiti_get_disk_owned( Node => $Node );
            logcomment("Drives @drv_own belongs to  $FILER_C");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            foreach my $disk_t (@drv_own) {

                if ( $disk_t =~ /P/ ) {
                    ( $disk_t = $disk_t ) =~ s/P\d//g;
                }
                ( $disk_t = $disk_t ) =~ s/$FILER_C\://g;
                my $cmd = "$stsb_status get_status $disk_t";
                my $output = $Api_Set_Obj->execute_command( 'command' => $cmd );
            }
        } else {
            logcomment("Not a SATA testbed. Does not record device information after firmware download");
        }
    }

    my ( $first_time, $end_time );
    my $time_now = time;
    my $time_end = $time_now + $test_wait_time;
    my ( $sec, $min, $hrs ) = gmtime($test_wait_time);
    my $loop = 1;
    my $firm_delay_time;
    if ( $total_drv_count_s2 <= 48 ) {
        $firm_delay_time = 60;
    } else {
        $firm_delay_time = 30;
    }
    logcomment("DEBUG : Firmware delay time / sleep time in between ping pong step : $firm_delay_time ");

    while ( $time_now < $time_end ) {

        #STEP 5
        tahiti_eye_catcher( Test => $Test, string => "STEP 5 of 12: Download older (LOW) verion FW to target drives : $loop loop" );
        my @drv_dwlded;
        my $flag = 0;
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            logcomment("Downgrade firmware for filer :$FILER_C using firmware files from DISKFW/LOW directory");
            my $low_dir = "$Home/NDATE/FIRMWARE/DISKFW/LOW";
            my $drv_dwlded = tahiti_disk_fw_update_low_or_high( Node => $Node, FW_DIR_LOC => $low_dir, FORCE_UPDTE => "yes" );

            if ( ref $drv_dwlded eq 'ARRAY' ) {
                logcomment("Filer $FILER_C firmware downgrade compelted on @$drv_dwlded drives , Will be rechecked with psmadmin output");
                logcomment("**DEBUG** : Drive Download Out : @$drv_dwlded");
            } elsif ($drv_dwlded) {
                return logresult( "INFO", msg => 1 );
            }

            logcomment("Check drive firmware ");
            tahiti_get_all_disk( node_present => [$Node] );
        }

        # STEP 6
        tahiti_eye_catcher( Test => $Test, string => "STEP 6 of 12: Record all device information after firmware download" );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            logcomment("Get Drive owned by $FILER_C");
            my @drv_own = tahiti_get_disk_owned( Node => $Node );
            logcomment("Drives @drv_own belongs to  $FILER_C");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );

            my @sel_drv;

            foreach my $disk_t (@drv_own) {
                if ( $disk_t =~ /P/ ) {
                    ( $disk_t = $disk_t ) =~ s/P\d//g;
                    push( @sel_drv, $disk_t );
                }
                ( my $mydisk = $disk_t ) =~ s/$FILER_C\://g;
                push( @sel_drv, $disk_t );
            }

            @sel_drv = unique(@sel_drv);

            foreach my $drv (@sel_drv) {
                logcomment("Excute scsi inquiry on $drv");
                my $output = $Api_Set_Obj->execute_command( 'command' => "priv set -q test;scsi inquiry $drv" );
            }

            my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
            $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0" );
            $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
            $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0" );
            $Api_Set_Obj->execute_raw_command( 'command' => 'system timeout modify 0' );
            $Api_Set_Obj->execute_raw_command( 'command' => 'run local dbg to off' );
            $Api_Set_Obj->execute_raw_command( 'command' => 'event config modify -suppression on -console on -console-log-level DEBUG' );
            sleep 5;
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );

            #Execute disklist and get all required details in hash based on serial number.
            my $dr_out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk_list" );
            foreach my $line ( split( /\n/, $dr_out ) ) {
                if ( $line =~ /(\S+\.(\S+)\.\S+\.\S+)/ ) {
                    $shelf = $2;
                }
            }
            logcomment("Tahiti shelf ID : $shelf");

            foreach my $line ( split( /\n/, $dr_out ) ) {
                if ( $line =~ /(\S+\.(\S+)\.\S+\.(\S+))\s+\S+\s+(\S+)\s+(\S+)\s+(\S+)/ ) {
                    my $ser = $6;
                    $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'}      = $1;
                    $drv_fw_mod_lst->{$ser}->{'SHELF'}           = $2;
                    $drv_fw_mod_lst->{$ser}->{'DSLOT'}           = $3;
                    $drv_fw_mod_lst->{$ser}->{'MODEL'}           = $4;
                    $drv_fw_mod_lst->{$ser}->{'FW_REV_DSK_LIST'} = $5;
                }
            }

            #Get all firmware files from low directory
            my $fw_dir  = "$Home/NDATE/FIRMWARE/DISKFW/LOW";
            my @disk_fw = `cd $fw_dir;ls *.LOD`;
            logcomment("disk_fw : @disk_fw");

            # Get drives where firmware files are valid - match with fw files in dir and push to valid array.
            @serial_nums = keys %$drv_fw_mod_lst;
            foreach my $mods (@disk_fw) {
                my ($modfw) = $mods =~ /(\S+)\.(\S+)\.\S+/;
                foreach my $ser (@serial_nums) {
                    my $mode_dlist = $drv_fw_mod_lst->{$ser}->{'MODEL'};
                    if ( $modfw eq $mode_dlist ) {
                        push( @valid_drvs, $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'} );
                        $drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'} = $2;
                    }
                }
            }

            # Match slot with slots captures in hash and firmware from psm output corresponding to drive slota
            sleep 2;
            my $Api_Set_ObjCL = $Node->get_7m_or_nodescope_apiset();
            $Api_Set_ObjCL->execute_raw_command( 'command' => "set -rows 0" );
            $Api_Set_ObjCL->execute_raw_command( 'command' => " " );
            $Api_Set_ObjCL->execute_raw_command( 'command' => "\013" );
            my $psmadmin_out = $Api_Set_ObjCL->execute_raw_command( 'command' => "run local psmadmin inband_cli 0x.$shelf 'bridge nvme show controller'" );
            if ( $psmadmin_out =~ /auto\.session\.change|\:debug|kern\.uptime|\:notice|\:info/ ) {
                logcomment("Output of psmadmin has some debug messages, re-try command again");
                $Api_Set_ObjCL->execute_raw_command( 'command' => " " );
                $Api_Set_ObjCL->execute_raw_command( 'command' => "\013" );
                $psmadmin_out = $Api_Set_ObjCL->execute_raw_command( 'command' => "run local psmadmin inband_cli 0x.$shelf 'bridge nvme show controller'" );
            }
            foreach my $line ( split( /\n/, $psmadmin_out ) ) {
                next if ( $line =~ /^\-|Node|NAME/ );
                if ( $line =~ /(\S+)\s+(\S+)\s+\S+\s+(\S+)\s+\S+/ ) {
                    my $psmslot = $2;
                    my $psmfw   = $3;
                    foreach my $ser (@serial_nums) {
                        my $drv_slot = $drv_fw_mod_lst->{$ser}->{'DSLOT'};
                        if ( $psmslot == $drv_slot ) {
                            logcomment("Drive : $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'} ==> FW in DRIECTORY : $drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'} ==> PSM FW : $psmfw");
                            if ( $psmfw =~ /$drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'}/ ) {
                                logcomment("Drive : $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'} :Firmware revision on psm slot : $psmslot  is $psmfw and FIRMWARE used to update - $drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'} updated");
                                last;
                            } else {
                                my $drv = $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'};
                                my $ans = grep /$drv/, @valid_drvs;
                                logcomment("**WARNING** : $FILER_C : Drive : $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'} : Firmware revision for psm slot : $psmslot is $psmfw and FIRMWARE used to update - $drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'}") if $ans;

                                #logresult( 'FATAL', "$FILER_C – Mismatch in firmware version." ) if $ans;
                                #return logresult( "INFO", msg => 1 );
                            }
                        }
                    }
                }
            }

            #STEP 7
            tahiti_eye_catcher( Test => $Test, string => "STEP 7 of 12: Download newer (HIGH) FW to target drives and look for any download failures" );
            my $high_dir = "$Home/NDATE/FIRMWARE/DISKFW/HIGH";
            my @drv_dwlded;
            logcomment("Upgrade firmware for filer :$FILER_C using firmware files from DISKFW/HIGHdirectory");
            my $drv_dwlded = tahiti_disk_fw_update_low_or_high( Node => $Node, FW_DIR_LOC => $high_dir, FORCE_UPDTE => "yes" );

            if ( ref $drv_dwlded eq 'ARRAY' ) {
                logcomment("Filer $FILER_C firmware Upgrade compelted on @$drv_dwlded drives , Will be rechecked with psmadmin output");
                logcomment("**DEBUG** : Drive Download Out : @$drv_dwlded");
            } elsif ($drv_dwlded) {
                return logresult( "INFO", msg => 1 );
            }

            logcomment("Check drive firmware ");
            tahiti_get_all_disk( node_present => [$Node] );

            my @disk_fw = `cd $high_dir;ls *.LOD`;
            foreach my $mods (@disk_fw) {
                my ($modfw) = $mods =~ /(\S+)\.(\S+)\.\S+/;
                foreach my $ser (@serial_nums) {
                    my $mode_dlist = $drv_fw_mod_lst->{$ser}->{'MODEL'};
                    if ( $modfw eq $mode_dlist ) {
                        push( @valid_drvs, $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'} );
                        $drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'} = $2;
                    }
                }
            }

            # Match slot with slots captures in hash and firmware from psm output corresponding to drive slota
            my $Api_Set_ObjCL = $Node->get_7m_or_nodescope_apiset();
            $Api_Set_ObjCL->execute_raw_command( 'command' => "set -rows 0" );
            $Api_Set_ObjCL->execute_raw_command( 'command' => " " );
            $Api_Set_ObjCL->execute_raw_command( 'command' => "\013" );
            my $psmadmin_out = $Api_Set_ObjCL->execute_raw_command( 'command' => "run local psmadmin inband_cli 0x.$shelf 'bridge nvme show controller'" );
            if ( $psmadmin_out =~ /auto\.session\.change|\:debug|kern\.uptime|\:notice|\:info/ ) {
                logcomment("Output of psmadmin has some debug messages, re-try command again");
                $Api_Set_ObjCL->execute_raw_command( 'command' => " " );
                $Api_Set_ObjCL->execute_raw_command( 'command' => "\013" );
                $psmadmin_out = $Api_Set_ObjCL->execute_raw_command( 'command' => "run local psmadmin inband_cli 0x.$shelf 'bridge nvme show controller'" );
            }
            foreach my $line ( split( /\n/, $psmadmin_out ) ) {
                next if ( $line =~ /^\-|Node|NAME/ );
                if ( $line =~ /(\S+)\s+(\S+)\s+\S+\s+(\S+)\s+\S+/ ) {
                    my $psmslot = $2;
                    my $psmfw   = $3;
                    foreach my $ser (@serial_nums) {
                        my $drv_slot = $drv_fw_mod_lst->{$ser}->{'DSLOT'};
                        if ( $psmslot == $drv_slot ) {
                            logcomment("Drive : $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'} ==> FW in DRIECTORY : $drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'} ==> PSM FW : $psmfw");
                            if ( $psmfw =~ /$drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'}/ ) {
                                logcomment("Drive : $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'} :Firmware revision on psm slot : $psmslot is $psmfw and FIRMWARE used to update - $drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'} updated");
                                last;
                            } else {
                                my $drv = $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'};
                                my $ans = grep /$drv/, @valid_drvs;
                                logcomment("**WARNING** : $FILER_C : Drive : $drv_fw_mod_lst->{$ser}->{'DRIVE_NAME'} :Firmware revision of psm slot : $psmslot is $psmfw and FIRMWARE used to update - $drv_fw_mod_lst->{$ser}->{'FW_IN_DIR'} ") if $ans;

                                #logresult( 'FATAL', "$FILER_C – Mismatch in firmware version." ) if $ans;
                                #return logresult( "INFO", msg => 1 );
                            }
                        }
                    }
                }
            }
        }

        #STEP 8
        tahiti_eye_catcher( Test => $Test, string => "STEP 8 of 12: Record all device information after firmware download" );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            logcomment("Get Drive owned by $FILER_C");
            my @drv_own = tahiti_get_disk_owned( Node => $Node );
            logcomment("Drives @drv_own belongs to  $FILER_C");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );

            my @sel_drv;

            foreach my $disk_t (@drv_own) {
                if ( $disk_t =~ /P/ ) {
                    ( $disk_t = $disk_t ) =~ s/P\d//g;
                    push( @sel_drv, $disk_t );
                }
                ( my $mydisk = $disk_t ) =~ s/$FILER_C\://g;
                push( @sel_drv, $disk_t );
            }

            @sel_drv = unique(@sel_drv);

            foreach my $drv (@sel_drv) {
                logcomment("Excute scsi inquiry on $drv");
                my $output = $Api_Set_Obj->execute_command( 'command' => "priv set -q test;scsi inquiry $drv" );
            }
        }

        #STEP 9
        tahiti_eye_catcher( Test => $Test, string => "STEP 9 of 12:  Repeat step 5 to 8 for 12 hours" );
        logcomment("User selected -w=$TEST_WAIT_TIME so the looping time will be $hrs HOURS $min MINUTES and $sec SECONDS");
        logcomment( "Start time for looping is " . scalar( localtime() ) );
        $time_now = time;
        my $time_left = $time_end - $time_now;
        my ( $lsec, $lmin, $lhrs ) = gmtime($time_left);
        if ( $time_left > 0 ) {
            logcomment("Remaining time for looping is $lhrs HOURS $lmin MINUTES and $lsec SECONDS");
        }
        $loop++;
        sleep($firm_delay_time);
        logcomment("Waiting time is $firm_delay_time seconds per firmware download");

    }    #End of While

    #STEP 10
    tahiti_eye_catcher( Test => $Test, string => "STEP 10 of 12:Record HDD FRU information for all HDD FRU in the system" );
    foreach my $Node (@Nodes) {
        my $stsb_status;
        my $FILER_C = $Node->name();
        if ( $SATA_SETUP eq 'Y' ) {
            logcomment("Checking for stsb/acorn command is available ");
            $stsb_status = get_filer_check( Node => $Node );
            chomp($stsb_status);
            logcomment("DEBUG :$stsb_status");
            my @drv_own = tahiti_get_disk_owned( Node => $Node );
            logcomment("Drives @drv_own belongs to  $FILER_C");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            foreach my $disk_t (@drv_own) {

                if ( $disk_t =~ /P/ ) {
                    ( $disk_t = $disk_t ) =~ s/P\d//g;
                }
                ( $disk_t = $disk_t ) =~ s/$FILER_C\://g;
                my $cmd = "$stsb_status get_status $disk_t";
                my $output = $Api_Set_Obj->execute_command( 'command' => $cmd );
            }
        } else {
            logcomment("Not a SATA testbed. Does not record device information after firmware download");
        }
    }

    #STEP 11
    tahiti_eye_catcher( Test => $Test, string => "STEP 11 of 12: Check to see if there is any unusual increase in the drive error logs" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        $scsi_result_s11{$FILER_C} = scsi_disk_errors( Node => $Node );
        logcomment("Record the log sense data from each drive completed : $FILER_C");
    }
    logcomment("Compare log sense data");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $before  = $scsi_result_s3{$FILER_C};
        my $after   = $scsi_result_s11{$FILER_C};
        my $value   = compare_scsi_disk(
            scsi_disk_error_before => $before,
            scsi_disk_error_after  => $after
        );
        if ( $value == 0 ) {
            logcomment("The output of log sense data is identical to step 3");
        } else {
            $test_status = 1;
            logcomment("**FAIL**: THE OUTPUT OF LOG SENSE DATA IS DIFFERENT FROM STEP 3 - STEP : 11");
            return logresult( "INFO", msg => 1 );
        }
    }

    #STEP 12 :
    tahiti_eye_catcher( Test => $Test, string => "STEP 12 of 12: Disable disk fw debug level" );
    logcomment("Reset back disk fw debug level ");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=0" );
    }
    logresult( "INFO", msg => $test_status );
}

#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode,
        Tahiti          => "yes"
    );

    ###########################################################################################
    ## Delete/Reset test specific options and values
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Host        = host($FILER_C);
        logcomment("disable autogiveback ");
        my $prompts_answers = [ 'Do you want to disable auto-giveback?' => 'y', 'Do you want to continue' => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => " " );
        $Api_Set_Obj->execute_raw_command( command => "\013" );
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback false", 'connectrec-match_table' => $prompts_answers );
        logcomment("Disable background firmwar update option");
        sleep 2;
        $Api_Set_Obj->execute_raw_command( command => "\013" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable off" );
        logcomment("Stop firmware update command");
        try {
            my $prompts = [ '.*Are you sure you want to.*' => 'y', '.*stop firmware update qualification.*' => 'y' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk_fw_update -S\"", 'connectrec-match_table' => $prompts);
            logcomment("Firmware Qualification download stopped ");
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            logcomment("FW Download stop test Failed.");
        };
        logcomment("$FILER_C : Turn off the raid options");
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.enable off" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.spares.enable off" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.scrub.enable off" );
    }

    return $TCD::PASS;
}