#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/BR/NADQ02_NVMe_PwrCyc_FW_DL.thpl#20 $
## @summary Mode
##  Verify that if a drive is power cycled while under heavy IO
##
##
## @description
## Verify that if a drive is power cycled while under heavy IO it can come back online in a
## reasonable time and tracks that it has gone through a power loss,
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  SATA :Dual Path
##  SAS  :Dual Path
##
## @usage
## The test can be run independently or with other tests as part of STEST.
## This test will be conducted on 6-48 drives(half loaded to 2 full shelves).
##
## @dependencies
##
## @steps
## The test will execute steps mentioned below:
## 1.   List all drives
##    1a.  Update to lower disk firmware
##    1b.  Check background firmware update option
##    1c.  Copy high firmware files and statrte disk_fw_update -Q
##    1d.  Start background firmware download in qualification mode
## 2.   Get nvme drives
## 3.   Verify IO instances
## 4.   Power cycle each drive in succession in the filesystem that IO is running on
## 5.   Dwell time between power cycles
## 6.   Repeat steps 4-5 for 6 hours
## 7.   Stop all threads
## 8.   Stop any disk scrub that might be running
## 9.   Stop firmware disk qualification
## 10.  Pulling BSAS link speed log##
##
## @param FILER        - optional for cluster setup
##                     - required for Dual path setup      Name of filer to be used
## @param FILERA       - required for cluster setup
##                     - optional for Dual path setup      Name of filer to be used
## @param FILERB       - required for cluster setup
##                     - optional for Dual path setup      Name of the partner node
## @param TEST_CONFIG  - optional Type of the setup, C for cluster, D for dual path (default)
## @param TEST_SETUP   - optional Default value set to 'SAS'
## @param FILER_CONN   - optional Type of connection, default is set to 'rsh' for this test
## @param MODE         - optional Filer mode maint/normal, default set to 'normal' for this test
## @param FILER_PROMPT - optional prompt of the filer
## @param LOGDIR       - optional This is required to generate 'END-LOG'. Default set to main log directory.
## @param EOE          - optional 'default' is set to default value
## @param BOOT_MODE    - optional Boot mode for filer , default set to '1' for this test
## @param BOOT_TYPE    - optional Boot type, default set to 'A' (automatic) for this test
## @param MAIL_TO      - Required. Destination email id.
## @param MAIL_FROM    - Required. Sender email id.
## @param EMAIL        - optional 'default' is set to 'Y'.
## @param FILER_TYPE   - required to differentiate between Boilermaker and Ironcity filers.
## @param POW_SLP_TIME - optional 'default' is set to default value.
## @param VOL_CREATE_CHOICE - optional 'default' is set to default value.
## @param ARMADILLO  -  This parameter will indicate if an ARMADILLO configuration is under testing (if yes, ARMADILLO=1 else ARMADILLO=2)
##
## @status  Automated
## @author  shiwani@netapp.com
## @burt    1205534
##
#############################################################################
#############################################################################
########################################
### Library functions
##########################################
use strict;

use File::Copy;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use Storage::SASUtils qw(disable_dbg_msgs);
use Storage::Sasadmin qw(sasadmin_dev_power_cycle);

#########################################
### Initialization/declaration
#########################################
use vars qw(
  $FILER
  $FILERA
  $FILERB
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $MODE
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $SATA_SETUP
  $MAIL_TO
  $MAIL_FROM
  $EMAIL
  $FILER_TYPE
  $SCRUB_PRIORITY_LEVEL
  $POW_SLP_TIME
  $HYBRID_AGGR
  $FIPS_SELECTED
);
my $params = NATE::ParamSet->new( global => 1 );
$LOGDIR               = $params->get( 'LOGDIR',               default => undef );
$FILER_CONN           = $params->get( 'FILER_CONN',           default => '#console' );
$EOE                  = $params->get( 'EOE',                  default => 'default' );
$EMAIL                = $params->get( 'EMAIL',                default => 'y' );
$MAIL_TO              = $params->get( 'MAIL_TO',              default => 'Email to' );
$MAIL_FROM            = $params->get( 'MAIL_FROM',            default => 'Email from' );
$FILER                = $params->get( 'FILER',                default => 'Filer' );
$FILERA               = $params->get( 'FILERA',               default => '' );
$FILERB               = $params->get( 'FILERB',               default => '' );
$FILER_PROMPT         = $params->get( 'FILER_PROMPT',         default => '\*>' );
$TEST_SETUP           = $params->get( 'TEST_SETUP',           default => 'SAS' );
$TEST_CONFIG          = $params->get( 'TEST_CONFIG',          default => 'D' );
$BOOT_MODE            = $params->get( 'BOOT_MODE',            default => '1' );
$BOOT_TYPE            = $params->get( 'BOOT_TYPE',            default => 'A' );
$SSD                  = $params->get( 'SSD',                  default => 'no' );
$HYBRID_AGGR          = $params->get( 'HYBRID_AGGR',          default => '0' );
$FILER_TYPE           = $params->get( 'FILER_TYPE',           default => 'BR' );
$SCRUB_PRIORITY_LEVEL = $params->get( 'SCRUB_PRIORITY_LEVEL', default => 'medium' );
$TEST_WAIT_TIME       = $params->get( 'TEST_WAIT_TIME',       default => '3' );
$MODE                 = $params->get( 'MODE',                 default => 'normal' );
$POW_SLP_TIME         = $params->get( 'POW_SLP_TIME',         default => '120' );
$FIPS_SELECTED        = $params->get( 'FIPS_SELECTED',        default => 'NO' );

#Common variable declaration
my ( $email_subject, $email_body );
my $TC_name;
my @Nodes;
my $filer_names;
my %nodes_filer = {};
my $BOAB_CONFIG = 0;
my $Home        = $ENV{HOME};

#Set test fail parameter to 0
my $test_status = 0;
my $Mode;
my $pre_post_flag = 0;
my $TC_name       = "328_NADQ02_NVMe_PwrCyc_FW_DL";

#my $test_wait_time = 43200;    ##equal to 12 hrs
my $test_wait_time = 21600;
logcomment("Default wait time is 6 hrs");
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 10% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 1");
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 2");
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( $test_wait_time * .95 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 4");
}

#########################################
## Pre-test processes
#########################################

my @Testcases = ( NVMe_PwrCyc_FW_DL => "PwrCyc_FW_DL" );

&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->nlog( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->nlog( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main

########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node objects used for test execution
    grep { logcomment( "Filers passed to the ntest are " . $_->name() ) } @Nodes;
    $Mode = $Nodes[0]->mode();

    version_test( node_present => \@Nodes, tc_name => $TC_name );
    logcomment("Checking for execution mode");
    $Test->nlog( "FILER- $filer_names : $TC_name : started, expected max completion time 12 hours : " . scalar( localtime() ) );
    $Test->nlog( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );
    logcomment("The user entered power cycle sleep time as $POW_SLP_TIME");
    return $TCD::PASS;
}

########## SETUP  ###################################################
# Setup automatically called before executing tests
#####################################################################
sub setup() {
    $Test->description("Setup the environment for the test exectuion ");
    $Test->nlog("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );
    ###########################################################################################
    #Stop the scrub process if already started
    foreach my $Node (@Nodes) {
        logcomment("Ensuring that the scrub and hammer process is stopped before the script starts execution");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->execute_command( 'command' => "disk scrub stop" );
        sleep 5;
        my $response = $Api_Set_Obj->hammer_abort();
        sleep 5;
    }
    return $TCD::PASS;
}

########## TEST 1 ###################################################
#NADQ02_PwrCyc
#####################################################################

sub NVMe_PwrCyc_FW_DL {
    my @subtests;
    logcomment( "Number of nodes are " . scalar @Nodes );
    my $partition = 0;
    logcomment("Checking whether the system is PARTITIONED or UNPARTITIONED");
    my $FILER_C = $Nodes[0]->name();
    my $disk_hash = disk_show_v( Node => $Nodes[0] );

    foreach ( keys %{$disk_hash} ) {
        if ( $_ =~ /P\d+$/ ) {
            $partition = 1;
            last;
        }
    }
    if ( $partition == 1 ) {
        my @Nodes_tmp = ();
        push( @Nodes_tmp, $Nodes[0] );
        my $Filer = $Nodes[0]->name();
        logcomment("SYSTEM is PARTITIONED, Test will run only on single node i.e., on $Filer node");
        @Nodes = ();
        @Nodes = @Nodes_tmp;
    } else {
        logcomment("SYSTEM is NOT PARTITIONED, Test will execute on both nodes");
    }
    logcomment("DEBUG : NODE PRESENT : @Nodes");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Executing Steps on Node : $FILER_C");
        push( @subtests, subtest( \&PwrCyc_Sub, -runid, "PwrCyc_$FILER_C", -bg, "--", $Node ) );
        logcomment("Completed Exeucting all steps on $FILER_C");
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $return = status_return( subtest_ref => [@subtests] );
    logcomment("Test status is $return");
    if ( $return == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

#######################
##Subtest_create
#######################
sub PwrCyc_Sub {
    my @Nodes;
    push( @Nodes, shift(@_) );

##########
## Test variables
##########
    my $sto_show_s2   = {};
    my $state_phy_s3  = {};
    my $sto_show_s11  = {};
    my $state_phy_s12 = {};
    my $sto_show_data = {};
    my %power_cyc     = ();
    my $phy_3         = {};
    my $phy_12        = {};
    my %filer_nvol    = ();
    my $state_phy_cur = {};
    my @NON_RT_AGG;
    my @NON_RT_VOLS;
    my %filer_nvol           = ();
    my $create_agg_n_vol     = 0;
    my $aggr_vol_found       = 0;
    my @list_of_disk_2_pwrcy = ();

    $Test->description("Power Cycle test");
    ###########################################################################################
    ## TEST STARTS
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Check for non-root aggregate, If already present, Proceed with test without creating new aggregate");
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        if (@non_rt_agg) {
            logcomment("Total non-root aggregate is $non_rt_status and Non -root aggregate are @non_rt_agg");
            logcomment("Check for non-root volume existance in available non root aggregate");
            vserver_delete( Node => $Node );    #Vserver deletion
            aggregate_volume_delete( Node => $Node );
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( command => "run local disk zero spares" );
            sleep 20;
            $Api_Set_Obj->execute_raw_command( command => "\013" );
        }
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("Non-root aggregate not found");
        sleep 30;
        my $spare_list  = nvme_sto_agg_show_spare_dsk( Node => $Node );
        my @spare_drvs  = keys %$spare_list;
        my $spare_count = @spare_drvs;
        logcomment("$FILER_C : Total spare are : $spare_count and Spare Drive are : @spare_drvs");
        logcomment("Create new vol");
        my $FILER_SIO = 20;
        logcomment("DEBUG :FILERSIO THREADS : $FILER_SIO ");
        sleep 30;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $return_filersio_thrds = nvme_multi_aggr_with_rd_wr_vol_n_fsio(
            API_con  => $Api_Set_Obj,
            filer    => $FILER_C,
            fsio     => $FILER_SIO,
            loc_drvs => \@spare_drvs,
            Node     => $Node,
        );
        my @loc_all_threads;

        foreach my $fil ( keys %$return_filersio_thrds ) {
            my $sio_th = $return_filersio_thrds->{$fil}->{'SIO_CMD'};
            @loc_all_threads = @$sio_th;
        }
        logcomment("Filer sio Started ");
        logcomment("@loc_all_threads");
        logcomment(" $FILER_C completed");
        sleep 30;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
    }
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        $API_obj->set_timeout( 'connectrec-timeout' => 7200 );
        my $FILER_C = $Node->name();
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        logcomment("Aggregate(s) found : @non_rt_agg");
        logcomment("Get Drives of all non-root aggregates : @non_rt_agg");
        foreach my $agr (@non_rt_agg) {
            logcomment("Get drives of $agr");
            $API_obj->set_timeout( 'connectrec-timeout' => 7200 );
            my $after = $API_obj->execute_command( 'command' => "aggr status -r $agr" );
            foreach ( split( /\n/, $after ) ) {
                next if ( $_ =~ /parity uninit/ );
                if ( $_ =~ /(data|dparity|parity)\s*(\S+)/ ) {
                    my $disk = $2;
                    $disk =~ s/(P|L)\d+$//g;
                    push( @list_of_disk_2_pwrcy, $disk );
                }
            }
        }
        logcomment("Drives : @list_of_disk_2_pwrcy will be powercycled , where IO are running");
    }
###########################################################################################
## Step 1 - Set privilege level to Test
## Pass/Fail criteria: : N/A
###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 1 of 10 : $FILER_C : Set privilege level to Test" );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    }

    #######################################################################
    # Step 1a -
    #######################################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 1a of 10 : Update to lower disk firmware" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Downgrade firmware for filer :$FILER_C using firmware files from DISKFW/LOW directory");
        my $low_dir = "$Home/NDATE/FIRMWARE/DISKFW/LOW";
        my @fw_dwmld_drvs = nvme_disk_fw_update_low_or_high( Node => $Node, FW_DIR_LOC => $low_dir, FORCE_UPDTE => "yes" );
        logcomment("Filer $FILER_C firmware downgrade compelted");
        logcomment("Filer $FILER_C : Drives @fw_dwmld_drvs , Firmware downgraded to version present in $low_dir directory");
    }

    #######################################################################
    # Step 1b
    #######################################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 1b of 10 : Check background firmware update option" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Check background firmware update option");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable" );
    }
    #######################################################################
    # Step 1c
    #######################################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 1c of 10: Copy high firmware files and statrte disk_fw_update -Q" );
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $high_dir    = "$Home/NDATE/FIRMWARE/DISKFW/HIGH";
        logcomment("Copying firmware files from : $high_dir");
        my @fw_dwmld_drvs = nvme_disk_fw_update_low_or_high( Node => $Node, FW_DIR_LOC => $high_dir, FORCE_UPDTE => "no" );
        logcomment("$FILER_C : Firmware files copied from $high_dir ");
    }

    #######################################################################
    # Step 1d
    #######################################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 1d of 10 : Start background firmware download in qualification mode" );
    foreach my $Node (@Nodes) {
        my $FILER_C         = $Node->name();
        my $Api_Set_Obj     = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=1" );
        logcomment("$FILER_C : Enabling firmware qualification ");
        my $prompts_a = [ "Are you sure you want to continue with firmware update.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "run local disk_fw_update -Q", 'connectrec-match_table' => $prompts_a );
        logcomment("firmware qualification started on $FILER_C");
        sleep 30;
        $Api_Set_Obj->execute_raw_command( 'command' => " " );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=0" );
    }
    ###########################################################################################
    # Step 2 - Get Nvme drives 
    # Pass/Fail criteria: N/A
    ###########################################################################################
    my @list_of_disk;
    my @disk_list;
    my @no_phy_cnt_2;
    my @hammer_run;
   nvme_eye_catcher( Test => $Test, string => "STEP 2 of 10 : Get Nvme drives" ); 
   foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $disk1   = disk_show_drive_maint( API_Obj => $API_obj, Filer => $FILER_C );
        @list_of_disk = @{ $disk1->{'driveA'} };
        logcomment("Disks present in the filer $FILER_C are @list_of_disk") if (@list_of_disk);
        @disk_list = @list_of_disk; 
   }

    ####################################################################################################
    ## Step 3 - Verify IO on drives
    ####################################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 3 of 10 : $FILER_C : Verify IO instances " );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $out = $Api_Set_Obj->execute_raw_command( command => "run local filersio status" );
        if ( $out =~ /no instances currently running/ ) {
            logcomment("**FATAL** : $FILER_C : NO IO Instances are currently running");
        } else {
            logcomment("Filersio isnstances are executing on $FILER_C");
        }
        sleep(10);
    }
    ###########################################################################################
    ## Step XX - Set scrub priority level.
    ## Pass/Fail criteria: options raid.scrub.perf_impact nnnn
    ## nnnn can be either low, mediun or high - setting to high
    ## would provide more CPU utilization to scrub process.
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("$FILER_C: Set scrub priority level");
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $API_obj->execute_raw_command( 'command' => "storage raid-options modify -node $FILER_C -name raid.scrub.perf_impact -value medium" );
    }

    ######################################################################################################
    ## Step 4 - Power cycle each drive in succession in the file-system that IO is running on
    ######################################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 4 of 10 : $FILER_C :Power cycle each drive in succession in the filesystem that IO is running on" );
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment( "FILER  $FILER_C : Power cycling all the drives on which hammer is running - " . scalar(@list_of_disk_2_pwrcy) . " drives @list_of_disk_2_pwrcy" );
        if ( scalar(@list_of_disk_2_pwrcy) == 0 ) {
            logcomment("**FATAL**: $FILER_C Test case will not be proceed because NON ROOT VOLUMES NOT EXISTS OR IO IS NOT RUNNING ON THE DISKS - STEP : 4");
            return logresult( "INFO", msg => 1 );
        }

        #Get the adapter and shelf ids
        #######my @ada_shlfids = extract_ada_shlf(@hammer_run);
        my @ada_shlfids = extract_ada_shlf(@list_of_disk_2_pwrcy);
        logcomment( "$FILER_C : Power cycling started at " . scalar( localtime() ) );
        my $loop  = 1;
        my $count = 1;
        my ( $s, $m ) = gmtime($POW_SLP_TIME);
        logcomment("Alotted Sleep Time : $POW_SLP_TIME seconds (i.e) $m MINS $s SECS\n");

        my $kill_time = time + $test_wait_time;
        while ( $kill_time > time ) {
            $Node->refresh_command_interface();
            my $step_6 = 1;
            nvme_eye_catcher( Test => $Test, string => "STEP 4($loop) of 10 : $FILER_C :Power cycle each drive in succession in the filesystem that IO is running on" ) if ( $loop > 1 );
            my $first = 1;
            my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
            $API_obj->execute_raw_command( 'command' => "\013" );

            #FIPS CHECK
            if ( $FIPS_SELECTED =~ /yes/i ) {
                logcomment("FIPS option is been selected, checking for SED drive and data key lock state");
                my $ret = check_encry_okm_lock_state( Node => $Node );
                if ( $ret == 1 ) {
                    logcomment("**WARNING**: FILER - $FILER_C - SED Drives cannot be re-locked or modify");
                }
            }
            if ( $loop == 1 ) {
                nvme_eye_catcher( Test => $Test, string => "STEP 6 of 10 : $FILER_C :Repeat steps 4-5 for 6 hours" );
                my ( $sec, $min, $hrs ) = gmtime($test_wait_time);
                logcomment("User selected -w=$TEST_WAIT_TIME so the looping time will be $hrs HOURS $min MINUTES and $sec SECONDS");
                logcomment( "Start time for looping is " . scalar( localtime() ) );

            }
            if ( $loop > 1 ) {
                my $time_left = $kill_time - time;
                my ( $lsec, $lmin, $lhrs ) = gmtime($time_left) if ( $time_left > 0 );
                logcomment("Remaining time for looping is $lhrs HOURS $lmin MINUTES and $lsec SECONDS") if ( $time_left > 0 );
            }
            $API_obj->set_timeout( "connectrec-timeout" => 180000 );
            $API_obj->execute_raw_command( 'command' => "\013" );
            foreach my $sel_drive (@list_of_disk_2_pwrcy) {
                my $API_obj = $Node->get_7m_or_nodescope_apiset();
                $API_obj->execute_command( 'command' => "snvmadmin power_cycle $sel_drive", 'timeout' => 600000 );
                $power_cyc{$sel_drive}++;
                nvme_eye_catcher( Test => $Test, string => "STEP 5 of 10 : $FILER_C :Dwell time between power cycles " )        if ( ( $first == 1 ) && ( $loop == 1 ) );
                nvme_eye_catcher( Test => $Test, string => "STEP 5($loop) of 10 : $FILER_C :Dwell time between power cycles " ) if ( ( $first == 1 ) && ( $loop > 1 ) );
                my ( $s, $m ) = gmtime($POW_SLP_TIME);
                logcomment("Sleeping for $POW_SLP_TIME seconds (i.e) $m MINS $s SECS\n");
                sleep($POW_SLP_TIME);
                $API_obj->execute_raw_command( command => " " );
                sleep 20;
                $API_obj->execute_raw_command( command => " " );
                $API_obj->set_timeout( "connectrec-timeout" => 180000 );
                $API_obj->execute_command( 'command' => "\013" );
                $API_obj->execute_command( 'command' => "\013" );
                $API_obj->execute_command( 'command' => "\013" );
                $first++;
            } ## end of foreach $sel_drive (@hammer_run)
                #$start_time_sec = time;
            $loop++;
            $count++;
            $API_obj->execute_raw_command( 'command' => "\013" );
        } ## end of while ($start_time_sec < $end_time_sec)
        logcomment( "End time for looping is " . scalar( localtime() ) );
        logcomment( "Power cycling on all the drives completed on filer $FILER_C " . scalar( localtime() ) );
        logcomment("The Power Cycle count details after completion of looping is : ");
        foreach $_ ( keys %power_cyc ) {
            logcomment("Drive : $_ \t Power Cycled : $power_cyc{$_} times\n");
        }
    }
    @Nodes = node_new_obj( node_array => [@Nodes] );
#############################################################################################
    ## Step 7 - Stop all IO threads
    ## Pass/Fail criteria: N/A
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 7 of 10 : $FILER_C :Stop all threads" );
        my $ret;
        try {
            logcomment("$FILER_C : Aborting all IO");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( command => "run local filersio stop" );
            logcomment("IO thread stopped ");
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            my $exception_object = shift;
            my $resp             = $exception_object->get_response_object();
            $Test->nlog( "Caught a " . ref($exception_object) . "!" );
            $Test->nlog( "Error is: " . $exception_object->text() );
            $Test->nlog( "Raw output was:\n" . $resp->get_raw_output() );
            $Test->nlog("Hammer abort has not done properly");
        };
    }

    ###########################################################################################
    ## Step 8 - Stop any disk scrub that might be running.
    ## Pass/Fail criteria: N/A
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 8 of 10 : $FILER_C :Stop any disk scrub that might be running" );
        try {
            $API_obj->execute_command( 'command' => 'disk scrub stop' );
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            logcomment("Disk srub cannot be stopped");
        };
    }

    ###########################################################################################
    ## STEP 9 - Stop firmware disk qualification
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C         = $Node->name();
        my $Api_Set_Obj     = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        nvme_eye_catcher( Test => $Test, string => "STEP 9 of 10 : $FILER_C : Stop firmware disk qualification" );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        my $prompts_ans = [ ".*Do you want to stop .*" => 'y', "Are you sure you want to stop firmware update quali.*" => 'y', 'sure you want to stop firmware update qualification' => 'yes' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "run local disk_fw_update -S", 'connectrec-match_table' => $prompts_ans );
        logcomment("Stopped firmware qualification $FILER_C");
    }
    ###########################################################################################
    ## Step 10 - Pull BSAS link speed logs
    ## Pass/Fail criteria: These additional steps are only for BSAS drives
    ###########################################################################################
    @Nodes = node_new_obj( node_array => [@Nodes] );
    if ( $TEST_SETUP !~ /FC/ig ) {
        foreach my $Node (@Nodes) {
            my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
            my $FILER_C = $Node->name();
            nvme_eye_catcher( Test => $Test, string => "STEP 10 of 10 : $FILER_C :Pulling BSAS link speed logs" );
            my $Mode = $Node->mode();
            my $stsb_status = get_filer_check( Node => $Node );
            $sto_show_data = storage_show_disk_a_data( Node => $Node );

            #  Collect the SM/SA/SDB disks if it is BSAS
            my ( $ssm, $ssa, $sdb, $dir );
            ( $ssm, $ssa, $sdb ) = ( 0, 0, 0 );
            my %dr_id = ();

            my @temp = grep { !/P\d+$/ } @disk_list;
            logcomment("Checking Model number of the disks @temp");
            foreach my $disk (@temp) {
                my $mdl = $sto_show_data->{$disk}->{'Model'};
                logcomment("Filer : $FILER_C : Model of disk $disk is $mdl");
                if ( $mdl =~ /(SA)$/ ) {
                    $ssa = 1;
                    $dr_id{$disk} = $1;
                    logcomment("SA: $disk - $1");
                    logcomment("Filbert(SSA): Writing crash dump to file of the $disk - type $dr_id{$disk}");
                    $API_obj->execute_command( 'command' => "$stsb_status get_crash_dump  $disk" );
                } elsif ( $mdl =~ /(SM)$/ ) {
                    $ssm = 1;
                    $dr_id{$disk} = $1;
                    logcomment("SM: $disk - $1");
                    logcomment("Acorn(SSM):  Scsi modesense of the disk $disk - type $dr_id{$disk}");
                    $API_obj->execute_command( 'command' => "scsi modesense -p 0x3a $disk" );
                } elsif ( $mdl =~ /(SDB)$/ ) {
                    $sdb = 1;
                    $dr_id{$disk} = $1;
                    print "SSD: $disk - $1\n";
                    logcomment("Marvell(SDB): Acorn get status of the disk $disk - type $dr_id{$disk}");
                    $API_obj->execute_command( 'command' => "$stsb_status get_status  $disk" );
                }
            }
            if ( ( $ssm == 0 ) && ( $ssa == 0 ) && ( $sdb == 0 ) ) {
                logcomment("Step 13: No SM/SA/SDB drives found - Exiting the Step 13");
            }
        }    # end foreach my $Node (@Nodes)
    } else {
        nvme_eye_catcher( Test => $Test, string => "STEP 13 of 13 : Pulling BSAS link speed logs" );
        logcomment("These additional steps are only for BSAS and MSATA drives");
    }
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        if (@non_rt_agg) {
            logcomment("Total non-root aggregate is $non_rt_status and Non -root aggregate are @non_rt_agg");
            logcomment("Check for non-root volume existance in available non root aggregate");
            vserver_delete( Node => $Node );    #Vserver deletion
            aggregate_volume_delete( Node => $Node );
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( command => "run local disk zero spares" );
            sleep 20;
            $Api_Set_Obj->execute_raw_command( command => "\013" );
        }
    }
    ###########################################################################################
    ## Test log/results
    ###########################################################################################
    logresult( "INFO", msg => $test_status );
}

#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    @Nodes = node_new_obj( node_array => [@Nodes] );
    logcomment(" $filer_names - Clean up and post test process");

    #    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ###########################################################################################
    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );

    return $TCD::PASS;
}
####################################
## SUBROUTINES
####################################
# This function extracts adapter and shlf name of the disks and passes into array
sub extract_ada_shlf {
    my (@extract) = (@_);
    my %uniq;
    my ( @ada, $shlf );
    foreach $_ (@extract) {
        $shlf = $_;
        $shlf =~ s/(\S+\.)[0](\S+)\.\S+/$1$2/;
        $shlf =~ s/(\S+\.)(\S+)\.\S+/$1$2/;
        if ( ( ++$uniq{$shlf} ) == 1 ) {
            push( @ada, $shlf );
        }
    }
    logcomment("Adapters and shlf ids: @ada");
    return @ada;
} ## end of extract_ada_s