#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/BR/NADQ02_SAS_PowerCycle_Test.thpl#20 $
## @summary Mode
##  Verify that if a drive is power cycled while under heavy IO
##
##
## @description
## Verify that if a drive is power cycled while under heavy IO it can come back online in a
## reasonable time and tracks that it has gone through a power loss,
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  SATA :Dual Path
##  SAS  :Dual Path
##
## @usage
## The test can be run independently or with other tests as part of STEST.
## This test will be conducted on 6-48 drives(half loaded to 2 full shelves).
##
## @dependencies
##
## @steps
## The test will execute steps mentioned below:
##   1. Set privilege level to Test
##   2. Record all Phy Change Counts for all drives present
##   3. Record power cycle count and power cycle failures for each drive
##   4. Start 2 threads of hammer
##   5. Power cycle one drive in the filesystem that hammer is running on
##   6. Issue these commands once every 5 mins
##   7. Dwell time between power cycles
##   8. Repeat steps 7-8 for 12 hours
##   9. Stop hammer execution
##  10. Stop any disk scrub that might be running
##  11. Record all Phy Change Counts for all drives present
##  12. Record power cycle count and power cycle failures for each drive
##  13. pull BSAS link speed logs
##
##
## @param FILER        - optional for cluster setup
##                     - required for Dual path setup      Name of filer to be used
## @param FILERA       - required for cluster setup
##                     - optional for Dual path setup      Name of filer to be used
## @param FILERB       - required for cluster setup
##                     - optional for Dual path setup      Name of the partner node
## @param TEST_CONFIG  - optional Type of the setup, C for cluster, D for dual path (default)
## @param TEST_SETUP   - optional Default value set to 'SAS'
## @param FILER_CONN   - optional Type of connection, default is set to 'rsh' for this test
## @param MODE         - optional Filer mode maint/normal, default set to 'normal' for this test
## @param FILER_PROMPT - optional prompt of the filer
## @param LOGDIR       - optional This is required to generate 'END-LOG'. Default set to main log directory.
## @param EOE          - optional 'default' is set to default value
## @param BOOT_MODE    - optional Boot mode for filer , default set to '1' for this test
## @param BOOT_TYPE    - optional Boot type, default set to 'A' (automatic) for this test
## @param SSD          - optional Setup is SSD or not, default set to 'no'
## @param MAIL_TO      - Required. Destination email id.
## @param MAIL_FROM    - Required. Sender email id.
## @param EMAIL        - optional 'default' is set to 'Y'.
## @param FILER_TYPE   - required to differentiate between Boilermaker and Ironcity filers.
## @param MINI_CREATE  -  optional 'default' is set to default value.
## @param MINI_CREATE_SIZE - optional 'default' is set to default value.
## @param POW_SLP_TIME - optional 'default' is set to default value.
## @param VOL_CREATE_CHOICE - optional 'default' is set to default value.
## @param ARMADILLO  -  This parameter will indicate if an ARMADILLO configuration is under testing (if yes, ARMADILLO=1 else ARMADILLO=2)
##
## @status  Automated
## @author  arunak@netapp.com
## @burt    861150
##
#############################################################################
#############################################################################
########################################
### Library functions
##########################################
use strict;

use File::Copy;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use Storage::SASUtils qw(disable_dbg_msgs);
use Storage::Sasadmin qw(sasadmin_dev_power_cycle);

#########################################
### Initialization/declaration
#########################################
use vars qw(
  $FILER
  $FILERA
  $FILERB
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $MODE
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $SATA_SETUP
  $MAIL_TO
  $MAIL_FROM
  $EMAIL
  $FILER_TYPE
  $SCRUB_PRIORITY_LEVEL
  $POW_SLP_TIME
  $HYBRID_AGGR
  $FIPS_SELECTED
);

param( 'FILER',     -mesg, "hostspec of the filer" );
param( 'MAIL_TO',   -mesg, "Email To" );
param( 'MAIL_FROM', -mesg, "Email from" );

my $params = NATE::ParamSet->new( global => 1 );
$LOGDIR               = $params->get( 'LOGDIR',               default => undef );
$FILER_CONN           = $params->get( 'FILER_CONN',           default => '#console' );
$EOE                  = $params->get( 'EOE',                  default => 'default' );
$EMAIL                = $params->get( 'EMAIL',                default => 'y' );
$MAIL_TO              = $params->get( 'MAIL_TO',              default => 'Email to' );
$MAIL_FROM            = $params->get( 'MAIL_FROM',            default => 'Email from' );
$FILER                = $params->get( 'FILER',                default => 'Filer' );
$FILERA               = $params->get( 'FILERA',               default => '' );
$FILERB               = $params->get( 'FILERB',               default => '' );
$FILER_PROMPT         = $params->get( 'FILER_PROMPT',         default => '\*>' );
$TEST_SETUP           = $params->get( 'TEST_SETUP',           default => 'SAS' );
$TEST_CONFIG          = $params->get( 'TEST_CONFIG',          default => 'D' );
$BOOT_MODE            = $params->get( 'BOOT_MODE',            default => '1' );
$BOOT_TYPE            = $params->get( 'BOOT_TYPE',            default => 'A' );
$SSD                  = $params->get( 'SSD',                  default => 'no' );
$HYBRID_AGGR          = $params->get( 'HYBRID_AGGR',          default => '0' );
$FILER_TYPE           = $params->get( 'FILER_TYPE',           default => 'BR' );
$SCRUB_PRIORITY_LEVEL = $params->get( 'SCRUB_PRIORITY_LEVEL', default => 'medium' );
$TEST_WAIT_TIME       = $params->get( 'TEST_WAIT_TIME',       default => '3' );
$MODE                 = $params->get( 'MODE',                 default => 'normal' );
$POW_SLP_TIME         = $params->get( 'POW_SLP_TIME',         default => '120' );
$FIPS_SELECTED        = $params->get( 'FIPS_SELECTED',        default => 'NO' );

#Common variable declaration
my ( $email_subject, $email_body );
my $TC_name;
my @Nodes;
my $filer_names;
my %nodes_filer = {};
my $BOAB_CONFIG = 0;

#Set test fail parameter to 0
my $test_status = 0;
my $Mode;
my $pre_post_flag = 0;
my $TC_name       = "9999_NADQ02_NVMe_NSSR_Standalone";

my $test_wait_time = 21600;    #6hrs
logcomment("Default wait time is 6 hrs");
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 10% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 1");
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 2");
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( $test_wait_time * .95 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 4");
}

#########################################
## Pre-test processes
#########################################

my @Testcases = ( NVMe_PwrCyc => "PwrCyc" );

&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->nlog( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->nlog( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main

########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node objects used for test execution
    grep { logcomment( "Filers passed to the ntest are " . $_->name() ) } @Nodes;
    $Mode = $Nodes[0]->mode();

    #version_test( node_present => \@Nodes, tc_name => $TC_name );
    logcomment("Checking for execution mode");
    $Test->nlog( "FILER- $filer_names : $TC_name : started, expected max completion time 12 hours : " . scalar( localtime() ) );
    $Test->nlog( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );
    logcomment("The user entered power cycle sleep time as $POW_SLP_TIME");
    return $TCD::PASS;
}

########## SETUP  ###################################################
# Setup automatically called before executing tests
#####################################################################
sub setup() {
    $Test->description("Setup the environment for the test exectuion ");
    $Test->nlog("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
    ###########################################################################################
    #Stop the scrub process if already started
    foreach my $Node (@Nodes) {
        logcomment("Ensuring that the scrub and hammer process is stopped before the script starts execution");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->execute_command( 'command' => "disk scrub stop" );
        sleep 5;
        my $response = $Api_Set_Obj->hammer_abort();
        sleep 5;
    }
    return $TCD::PASS;
}

########## TEST 1 ###################################################
#NADQ02_PwrCyc
#####################################################################

sub NVMe_PwrCyc {
    my @subtests;
    logcomment( "Number of nodes are " . scalar @Nodes );
    my $partition = 0;
    logcomment("Checking whether the system is PARTITIONED or UNPARTITIONED");
    my $FILER_C = $Nodes[0]->name();
    my $disk_hash = disk_show_v( Node => $Nodes[0] );

    foreach ( keys %{$disk_hash} ) {
        if ( $_ =~ /P\d+$/ ) {
            $partition = 1;
            last;
        }
    }

    logcomment("DEBUG : NODE PRESENT : @Nodes");
    my $return;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Test Started on $FILER_C");
        push( @subtests, subtest( \&PwrCyc_Sub, -runid, "PwrCyc_$FILER_C", -bg, "--", $Node ) );
        Subtest::wait_finish( subtest => [@subtests] );
        $return = status_return( subtest_ref => [@subtests] );
        logcomment("Test status is $return");
    }
    if ( $return == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

#######################
##Subtest_create
#######################
sub PwrCyc_Sub {
    my @Nodes;
    push( @Nodes, shift(@_) );

##########
## Test variables
##########
    my $sto_show_s2   = {};
    my $state_phy_s3  = {};
    my $sto_show_s11  = {};
    my $state_phy_s12 = {};
    my $sto_show_data = {};
    my %power_cyc     = ();
    my $phy_3         = {};
    my $phy_12        = {};
    my %filer_nvol    = ();
    my $state_phy_cur = {};
    my @NON_RT_AGG;
    my @NON_RT_VOLS;
    my %filer_nvol           = ();
    my $create_agg_n_vol     = 0;
    my $aggr_vol_found       = 0;
    my @list_of_disk_2_pwrcy = ();

    $Test->description("Power Cycle test");
    ###########################################################################################
    ## TEST STARTS
    ###########################################################################################
    #------------------------------------------------------------
## Step 1 - Set privilege level to Test
## Pass/Fail criteria: : N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 1 of 14 : $FILER_C : Set privilege level to Test" );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    }

    #---------------------------------------------------------------------------------
## Step 2 - Record all Phy Change Counts for all drives present
## Pass/Fail criteria: N/A
    #---------------------------------------------------------------------------------
    my @list_of_disk;
    my @disk_list;
    my @no_phy_cnt_2;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $disk1   = disk_show_drive_maint( API_Obj => $API_obj, Filer => $FILER_C );
        @list_of_disk = @{ $disk1->{'driveA'} };
        logcomment("Disks present in the filer $FILER_C are @list_of_disk") if (@list_of_disk);
        @disk_list = @list_of_disk;
    }
####################################################################################################
## Step 4 - Create Aggregate , Volume and start filersio
####################################################################################################
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000000 );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $Host = host($FILER_C);
        logcomment("Check for non-root aggregate, If already present, Proceed with test without creating new aggregate");
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );

        if (@non_rt_agg) {
            logcomment("Total non-root aggregate is $non_rt_status and Non -root aggregate are @non_rt_agg");
            logcomment("Check for non-root volume existance in available non root aggregate");
            vserver_delete( Node => $Node );    #Vserver deletion
            aggregate_volume_delete( Node => $Node );
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( command => "run local disk zero spares" );
            sleep 20;
            $Api_Set_Obj->execute_raw_command( command => "\013" );
        }
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("Non-root aggregate cannot be found");
        sleep 30;
        my $FILER_SIO   = 20;
        my $spare_list  = nvme_sto_agg_show_spare_dsk( Node => $Node );
        my @spare_drvs  = keys %$spare_list;
        my $spare_count = @spare_drvs;
        logcomment("$FILER_C : Total spare are : $spare_count and Spare Drive are : @spare_drvs");
        logcomment("Create new vol");
        logcomment("DEBUG :FILERSIO THREADS : $FILER_SIO ");
        sleep 30;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $return_filersio_thrds = nvme_multi_aggr_with_rd_wr_vol_n_fsio(
            API_con  => $Api_Set_Obj,
            filer    => $FILER_C,
            fsio     => $FILER_SIO,
            loc_drvs => \@spare_drvs,
            Node     => $Node,
        );
        my @loc_all_threads;

        foreach my $fil ( keys %$return_filersio_thrds ) {
            my $sio_th = $return_filersio_thrds->{$fil}->{'SIO_CMD'};
            @loc_all_threads = @$sio_th;
        }
        logcomment("Filer sio Started ");
        logcomment("@loc_all_threads");
        logcomment(" $FILER_C completed");
        sleep 30;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
    }

### Get drives where IO are running.
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        $API_obj->set_timeout( 'connectrec-timeout' => 7200 );
        my $FILER_C = $Node->name();
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        logcomment("Aggregate(s) found : @non_rt_agg");
        logcomment("Get Drives of all non-root aggregates : @non_rt_agg");
        foreach my $agr (@non_rt_agg) {
            logcomment("Get drives of $agr");
            $API_obj->set_timeout( 'connectrec-timeout' => 7200 );
            my $after = $API_obj->execute_command( 'command' => "aggr status -r $agr" );
            foreach ( split( /\n/, $after ) ) {
                next if ( $_ =~ /parity uninit/ );
                if ( $_ =~ /(data)\s*(\S+)/ ) {
                    my $disk = $2;
                    $disk =~ s/(P|L)\d+$//g;
                    push( @list_of_disk_2_pwrcy, $disk );
                    logcomment("Execute NSSR on 1 data drive $disk");
                    last;
                }
            }
            last;
        }
        logcomment("Drives : @list_of_disk_2_pwrcy will be powercycled , where IO are running");
    }

    #--------------------------------------------------------------------------------
## Step XX - Set scrub priority level.
## Pass/Fail criteria: options raid.scrub.perf_impact nnnn
## nnnn can be either low, mediun or high - setting to high
## would provide more CPU utilization to scrub process.
    #--------------------------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("$FILER_C: Set scrub priority level");
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $API_obj->execute_raw_command( 'command' => "storage raid-options modify -node $FILER_C -name raid.scrub.perf_impact -value medium" );
    }
######################################################################################################
## Step 5 - Power cycle each drive in succession in the file-system that hammer is running on
######################################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 5 of 13 : $FILER_C :Power cycle each drive in succession in the filesystem that hammer is running on" );
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment( "FILER  $FILER_C : Power cycling all the drives on which IO is running - " . scalar(@list_of_disk_2_pwrcy) . " drives @list_of_disk_2_pwrcy" );
        if ( scalar(@list_of_disk_2_pwrcy) == 0 ) {
            logcomment("**FATAL**: $FILER_C Test case will not be proceed because NON ROOT VOLUMES NOT EXISTS OR HAMMER IS NOT RUNNING ON THE DISKS - STEP : 5");
            return logresult( "INFO", msg => 1 );
        }

        #Get the adapter and shelf ids
        #######my @ada_shlfids = extract_ada_shlf(@hammer_run);
        my @ada_shlfids    = extract_ada_shlf(@list_of_disk_2_pwrcy);
        my $loop           = 1;
        my $count          = 1;
        my $first          = 1;
        my $total_loops    = 1000;                                      #total loop count
        my $nssr_rest_time = 240;                                       #4 min between each NSSR.
        my $sleep_time     = 60;                                        #1 min wait after NSSR
        logcomment("NSSR will be executed only on one drive, Drive is : @list_of_disk_2_pwrcy");
        logcomment("4 min wait time is configured in between each NSSR");
        logcomment("1 min wait time after NSSR , Will Check path of drives and PSM links");
        logcomment( "$FILER_C : 1000 NSSR loop started at " . scalar( localtime() ) );

        for ( $loop = 1 ; $loop <= $total_loops ; $loop++ ) {
            $Test->nlog("################################# STARTING LOOP : $loop #########################################");
            logcomment( "$FILER_C : Start Time for loop $loop is : " . scalar( localtime() ) );
            logcomment("$FILER_C : Starting loop $loop - NSSR on Drive ::: :::: ");
            foreach my $sel_drive (@list_of_disk_2_pwrcy) {
                my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
                $API_obj->execute_command( 'command' => "snvmadmin nssr $sel_drive", 'timeout' => 600 );
                $power_cyc{$sel_drive}++;
                nvme_eye_catcher( Test => $Test, string => "STEP 6 of 13 : $FILER_C :Dwell time between power cycles " )        if ( ( $first == 1 ) && ( $loop == 1 ) );
                nvme_eye_catcher( Test => $Test, string => "STEP 6($loop) of 13 : $FILER_C :Dwell time between power cycles " ) if ( ( $first == 1 ) && ( $loop > 1 ) );
                logcomment("wait for $sleep_time seconds (i.e) 1 MIN \n");
                sleep($sleep_time);
                $API_obj->set_timeout( "connectrec-timeout" => 18000 );
                $API_obj->execute_raw_command( 'command' => "\013" );
                $API_obj->execute_raw_command( 'command' => "\013" );
                $API_obj->execute_raw_command( 'command' => "\013" );
                $first++;

                #check IO
                logcomment("$FILER_C : Check IO status");
                $API_obj->execute_raw_command( 'command' => "\013" );
                $API_obj->execute_command( 'command' => "filersio status" );

                logcomment("$FILER_C : Check for drive path");

                #check Drive Path
                my @Nodes = NACL::C::Node->find();
                foreach my $Node (@Nodes) {
                    my $FILER_C = $Node->name();
                    my $Host    = host($FILER_C);
                    logcomment("$FILER_C : Check storage failover status");
                    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" );
                    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" ) if ( $out eq '' );
                    my @miss_drv;
                    my $FILER_A = $Nodes[0]->name();
                    my $FILER_B = $Nodes[1]->name();
                    logcomment("FILER A : $FILER_A and FILER_B : $FILER_B");

                    foreach my $line ( split /\n/, $out ) {
                        my ( $drv, $path );
                        next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change|\[\S+\]/ );
                        if ( ( $line =~ /$FILER_A/ ) && ( $line =~ /$FILER_B/ ) ) {
                            my ( $drv, $path ) = $line =~ /(\S+)\s+(\S+)/;
                            logcomment("Drive $drv has path $path");
                        } else {
                            my ( $drv, $path ) = $line =~ /\S+\:(\S+)\s+(\S+)/;
                            logcomment("Drive $drv path is incomplete");
                            push( @miss_drv, $drv );
                        }
                    }
                    if (@miss_drv) {
                        logcomment("!!!! DRIVE PATH MISSING  : @miss_drv");
                        logcomment("**FATAL** : DRIVE PATH MISSING  : @miss_drv");
                        logresult( "FATAL", "DRIVE PATH MISSING  : @miss_drv" );
                    }
                    sleep 30;
                }

                logcomment("$FILER_C : Check for PSM link");

                #check psm link
                logcomment("$FILER_C  : Check port link state for all drives");
                my @link_dis_drivs;
                my $out = $API_obj->execute_raw_command( 'command' => "set -rows 0;run local storage show psm" );
                foreach my $line ( split /\n/, $out ) {
                    next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change/ );
                    if ( $line =~ /\[.*]/ ) {
                        if ( $line =~ /DIS\/LNK|LNK|DIS/ ) {
                            my ( $drv, $state, $lnk_lan, $max_lnk, $neg_lnk, $med_mx_lnk, $dev_mx_la, $Neg_max, $lan_wd ) = ( $line =~ /(\[.*\])\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ );
                            logcomment("Drive ID : $drv State is $state !!!!!!!!! ");
                            logcomment("Drive port link states are : Disk ID : $drv\n Port Max Link Speed:$lnk_lan\n Negotiated Max Link Speed:$max_lnk\n Medium Link Speed:$neg_lnk\n Device Max Link Speed:$med_mx_lnk\n Port Max Lane Width:$dev_mx_la\n Negotiated Max Lanewidth:$Neg_max\n Lanewidth:$lan_wd\n");
                            logcomment("Drive ID : $drv State is $state");
                            push( @link_dis_drivs, $drv );
                        }
                    }
                }
                if (@link_dis_drivs) {
                    logcomment("***FATAL*** : $FILER_C : Port State for following drives are DISABLED or OFFLINE, Drive ID(s) are: @link_dis_drivs");
                    logresult( "FATAL", "Port State for following drives are DISABLED or OFFLINE, Drive ID(s) are: @link_dis_drivs" );
                }

                logcomment("Wait for 4 min, before issuing NSSR on same drive - $sel_drive");
            } ## end of foreach $sel_drive (@hammer_run)
            my $start_time_sec = time;
            $count++;
            logcomment("Wait for 4 min, before issuing NSSR on same drive");
            sleep($nssr_rest_time);
            logcomment( "End time for loop $loop is : " . scalar( localtime() ) );
            $Test->nlog("################################# ENDING LOOP : $loop #########################################");
            $Test->nlog(" ");
        } ## end of for loop
        logcomment( "$FILER_C : 1000 NSSR loop End time is " . scalar( localtime() ) );
        logcomment( "NSSR on Drive for $total_loops loops completed on filer $FILER_C " . scalar( localtime() ) );
        logcomment("The Power Cycle count details after completion of looping is : ");
        foreach $_ ( keys %power_cyc ) {
            logcomment("Drive : $_ \t Power Cycled : $power_cyc{$_} times\n");
        }
    }
    @Nodes = node_new_obj( node_array => [@Nodes] );

#############################################################################################
## Step 9 - Stop all threads of kernel tool Hammer
## Pass/Fail criteria: N/A
###########################################################################################

    foreach my $Node (@Nodes) {
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 9 of 13 : $FILER_C :Stop all IO threads " );
        my $ret;
        try {
            logcomment("$FILER_C : Aborting all IO");
            $ret = $API_obj->execute_command( 'command' => "filersio stop" );
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            my $exception_object = shift;
            my $resp             = $exception_object->get_response_object();
            $Test->nlog( "Caught a " . ref($exception_object) . "!" );
            $Test->nlog( "Error is: " . $exception_object->text() );
            $Test->nlog( "Raw output was:\n" . $resp->get_raw_output() );
            $Test->nlog("Hammer abort has not done properly");
        };
    }

###########################################################################################
## Step 10 - Stop any disk scrub that might be running.
## Pass/Fail criteria: N/A
###########################################################################################
    foreach my $Node (@Nodes) {
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 10 of 13 : $FILER_C :Stop any disk scrub that might be running" );
        try {
            $API_obj->execute_command( 'command' => 'disk scrub stop' );
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            logcomment("Disk srub cannot be stopped");
        };
    }

    #---------------------------------------------------------------------------------
## Step 13 - Pull BSAS link speed logs
## Pass/Fail criteria: These additional steps are only for BSAS drives
    #---------------------------------------------------------------------------------
    @Nodes = node_new_obj( node_array => [@Nodes] );
    if ( $TEST_SETUP !~ /FC/ig ) {
        foreach my $Node (@Nodes) {
            my $API_obj = $Node->get_7m_or_nodescope_apiset();
            my $FILER_C = $Node->name();
            nvme_eye_catcher( Test => $Test, string => "STEP 13 of 13 : $FILER_C :Pulling BSAS link speed logs" );
            my $Mode = $Node->mode();
            my $stsb_status = get_filer_check( Node => $Node );
            $sto_show_data = storage_show_disk_a_data( Node => $Node );

            #  Collect the SM/SA/SDB disks if it is BSAS
            my ( $ssm, $ssa, $sdb, $dir );
            ( $ssm, $ssa, $sdb ) = ( 0, 0, 0 );
            my %dr_id = ();

            my @temp = grep { !/P\d+$/ } @disk_list;
            logcomment("Checking Model number of the disks @temp");
            foreach my $disk (@temp) {
                my $mdl = $sto_show_data->{$disk}->{'Model'};
                logcomment("Filer : $FILER_C : Model of disk $disk is $mdl");
                if ( $mdl =~ /(SA)$/ ) {
                    $ssa = 1;
                    $dr_id{$disk} = $1;
                    logcomment("SA: $disk - $1");
                    logcomment("Filbert(SSA): Writing crash dump to file of the $disk - type $dr_id{$disk}");
                    $API_obj->execute_command( 'command' => "$stsb_status get_crash_dump  $disk" );
                } elsif ( $mdl =~ /(SM)$/ ) {
                    $ssm = 1;
                    $dr_id{$disk} = $1;
                    logcomment("SM: $disk - $1");
                    logcomment("Acorn(SSM):  Scsi modesense of the disk $disk - type $dr_id{$disk}");
                    $API_obj->execute_command( 'command' => "scsi modesense -p 0x3a $disk" );
                } elsif ( $mdl =~ /(SDB)$/ ) {
                    $sdb = 1;
                    $dr_id{$disk} = $1;
                    print "SSD: $disk - $1\n";
                    logcomment("Marvell(SDB): Acorn get status of the disk $disk - type $dr_id{$disk}");
                    $API_obj->execute_command( 'command' => "$stsb_status get_status  $disk" );
                }
            }
            if ( ( $ssm == 0 ) && ( $ssa == 0 ) && ( $sdb == 0 ) ) {
                logcomment("Step 13: No SM/SA/SDB drives found - Exiting the Step 13");
            }
        }    # end foreach my $Node (@Nodes)
    } else {
        nvme_eye_catcher( Test => $Test, string => "STEP 13 of 13 : Pulling BSAS link speed logs" );
        logcomment("These additional steps are only for BSAS and MSATA drives");
    }
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        if (@non_rt_agg) {
            logcomment("Total non-root aggregate is $non_rt_status and Non -root aggregate are @non_rt_agg");
            logcomment("Check for non-root volume existance in available non root aggregate");
            vserver_delete( Node => $Node );    #Vserver deletion
            aggregate_volume_delete( Node => $Node );
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( command => "run local disk zero spares" );
            sleep 20;
            $Api_Set_Obj->execute_raw_command( command => "\013" );
        }
    }
    ###########################################################################################
    ## Test log/results
    ###########################################################################################
    logresult( "INFO", msg => $test_status );
}

#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    @Nodes = node_new_obj( node_array => [@Nodes] );
    logcomment(" $filer_names - Clean up and post test process");

    #    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ###########################################################################################

    return $TCD::PASS;
}

####################################
## SUBROUTINES
####################################
# This function extracts adapter and shlf name of the disks and passes into array
sub extract_ada_shlf {
    my (@extract) = (@_);
    my %uniq;
    my ( @ada, $shlf );
    foreach $_ (@extract) {
        $shlf = $_;
        $shlf =~ s/(\S+\.)[0](\S+)\.\S+/$1$2/;
        $shlf =~ s/(\S+\.)(\S+)\.\S+/$1$2/;
        if ( ( ++$uniq{$shlf} ) == 1 ) {
            push( @ada, $shlf );
        }
    }
    logcomment("Adapters and shlf ids: @ada");
    return @ada;
} ## end of extract_ada_s