#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/BR/NADQ02_SAS_PowerCycle_Test.thpl#20 $
## @summary Mode
##  Verify that if a drive is power cycled while under heavy IO
##
##
## @description
## Verify that if a drive is power cycled while under heavy IO it can come back online in a
## reasonable time and tracks that it has gone through a power loss,
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  SATA :Dual Path
##  SAS  :Dual Path
##
## @usage
## The test can be run independently or with other tests as part of STEST.
## This test will be conducted on 6-48 drives(half loaded to 2 full shelves).
##
## @dependencies
##
## @steps
## The test will execute steps mentioned below:
##   1. Set privilege level to Test
##   2. Record all Phy Change Counts for all drives present
##   3. Record power cycle count and power cycle failures for each drive
##   4. Start 2 threads of hammer
##   5. Power cycle one drive in the filesystem that hammer is running on
##   6. Issue these commands once every 5 mins
##   7. Dwell time between power cycles
##   8. Repeat steps 7-8 for 12 hours
##   9. Stop hammer execution
##  10. Stop any disk scrub that might be running
##  11. Record all Phy Change Counts for all drives present
##  12. Record power cycle count and power cycle failures for each drive
##  13. pull BSAS link speed logs
##
##
## @param FILER        - optional for cluster setup
##                     - required for Dual path setup      Name of filer to be used
## @param FILERA       - required for cluster setup
##                     - optional for Dual path setup      Name of filer to be used
## @param FILERB       - required for cluster setup
##                     - optional for Dual path setup      Name of the partner node
## @param TEST_CONFIG  - optional Type of the setup, C for cluster, D for dual path (default)
## @param TEST_SETUP   - optional Default value set to 'SAS'
## @param FILER_CONN   - optional Type of connection, default is set to 'rsh' for this test
## @param MODE         - optional Filer mode maint/normal, default set to 'normal' for this test
## @param FILER_PROMPT - optional prompt of the filer
## @param LOGDIR       - optional This is required to generate 'END-LOG'. Default set to main log directory.
## @param EOE          - optional 'default' is set to default value
## @param BOOT_MODE    - optional Boot mode for filer , default set to '1' for this test
## @param BOOT_TYPE    - optional Boot type, default set to 'A' (automatic) for this test
## @param SSD          - optional Setup is SSD or not, default set to 'no'
## @param MAIL_TO      - Required. Destination email id.
## @param MAIL_FROM    - Required. Sender email id.
## @param EMAIL        - optional 'default' is set to 'Y'.
## @param FILER_TYPE   - required to differentiate between Boilermaker and Ironcity filers.
## @param MINI_CREATE  -  optional 'default' is set to default value.
## @param MINI_CREATE_SIZE - optional 'default' is set to default value.
## @param POW_SLP_TIME - optional 'default' is set to default value.
## @param VOL_CREATE_CHOICE - optional 'default' is set to default value.
## @param ARMADILLO  -  This parameter will indicate if an ARMADILLO configuration is under testing (if yes, ARMADILLO=1 else ARMADILLO=2)
##
## @status  Automated
## @author  himangin@netapp.com
## @burt    861150
##
#############################################################################
#############################################################################
########################################
### Library functions
##########################################
use strict;

use File::Copy;
use Storage::Common_Lib;
use Storage::SASUtils qw(disable_dbg_msgs);
use Storage::Sasadmin qw(sasadmin_dev_power_cycle);
use Storage::Ems qw(ems_log_get_mark ems_log_dump_by_id ems_error_monitor);

#########################################
### Initialization/declaration
#########################################
use vars qw(
  $FILER
  $FILERA
  $FILERB
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $MODE
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $SATA_SETUP
  $MAIL_TO
  $MAIL_FROM
  $EMAIL
  $FILER_TYPE
  $SCRUB_PRIORITY_LEVEL
  $POW_SLP_TIME
  $HYBRID_AGGR
);
my $params = NATE::ParamSet->new( global => 1 );
$LOGDIR               = $params->get( 'LOGDIR',               default => undef );
$FILER_CONN           = $params->get( 'FILER_CONN',           default => '#console' );
$EOE                  = $params->get( 'EOE',                  default => 'default' );
$EMAIL                = $params->get( 'EMAIL',                default => 'y' );
$MAIL_TO              = $params->get( 'MAIL_TO',              default => 'Email to' );
$MAIL_FROM            = $params->get( 'MAIL_FROM',            default => 'Email from' );
$FILER                = $params->get( 'FILER',                default => 'Filer' );
$FILERA               = $params->get( 'FILERA',               default => '' );
$FILERB               = $params->get( 'FILERB',               default => '' );
$FILER_PROMPT         = $params->get( 'FILER_PROMPT',         default => '\*>' );
$TEST_SETUP           = $params->get( 'TEST_SETUP',           default => 'SAS' );
$TEST_CONFIG          = $params->get( 'TEST_CONFIG',          default => 'D' );
$BOOT_MODE            = $params->get( 'BOOT_MODE',            default => '1' );
$BOOT_TYPE            = $params->get( 'BOOT_TYPE',            default => 'A' );
$SSD                  = $params->get( 'SSD',                  default => 'no' );
$HYBRID_AGGR          = $params->get( 'HYBRID_AGGR',          default => '0' );
$FILER_TYPE           = $params->get( 'FILER_TYPE',           default => 'BR' );
$SCRUB_PRIORITY_LEVEL = $params->get( 'SCRUB_PRIORITY_LEVEL', default => 'medium' );
$TEST_WAIT_TIME       = $params->get( 'TEST_WAIT_TIME',       default => '3' );
$MODE                 = $params->get( 'MODE',                 default => 'normal' );
$POW_SLP_TIME         = $params->get( 'POW_SLP_TIME',         default => '30' );

#Common variable declaration
my ( $email_subject, $email_body );
my $TC_name;
my @Nodes;
my $filer_names;
my %nodes_filer = {};
my $BOAB_CONFIG = 0;
#Set test fail parameter to 0
my $test_status = 0;
my $Mode;

my $TC_name = "504_NADQ02_PwrCyc";

my $test_wait_time = 43200;    ##equal to 12 hrs
logcomment("Default wait time is 12 hrs");
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 10% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 1");
}
elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 2");
}
elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( $test_wait_time * .95 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 4");
}

#########################################
## Pre-test processes
#########################################

my @Testcases = ( PwrCyc => "PwrCyc" );

&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->nlog( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->nlog( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main

########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node objects used for test execution
    grep { logcomment( "Filers passed to the ntest are " . $_->name() ) } @Nodes;
    $Mode = $Nodes[0]->mode();

    version_test( node_present => \@Nodes, tc_name => $TC_name );
    logcomment("Checking for execution mode");
    $Test->nlog( "FILER- $filer_names : $TC_name : started, expected max completion time 12 hours : " . scalar( localtime() ) );
    $Test->nlog( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );
	logcomment("**WARNING** : Power cycle sleep time will be set to 30");
	$POW_SLP_TIME = 30;
    logcomment("The user entered power cycle sleep time as $POW_SLP_TIME");
    return $TCD::PASS;
}

########## SETUP  ###################################################
# Setup automatically called before executing tests
#####################################################################
sub setup() {
    $Test->description("Setup the environment for the test exectuion ");
    $Test->nlog("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

    ###########################################################################################
    ## Checking for the Flexible Volumes if present, If Flexible volumes are not available it
    ## will create one Flexible volume other than root volume provided spare drives are more
    ## than 3. Else Test case will terminate
    ###########################################################################################
    #Checking for non root volumes
    my @subtests;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&vol_creation_sub, -runid, "vol_creation_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $return = status_return( subtest_ref => [@subtests] );
    if ( $return == 1 ) {
        logcomment("**FATAL**: Cannot create volumes which are other than Root Volumes.");
        logresult( "INFO", msg => 1 );
    }

    sub vol_creation_sub {
        my @Nodes;
        push( @Nodes, shift(@_) );
        ##do this in subtest
        logcomment( "Filer passed to the subtest :: " . $Nodes[0]->name() );

        #Checking for non root volumes
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            create_flex_vol( Node => $Node );    ##Create flex volumes as much as possible
            logcomment("Checking the non-root volumes on the filer $FILER_C");
            my @vol_created = check_non_root_volumes( Node => $Node );
            if (@vol_created) {
                logcomment("$FILER_C : Volumes which are available in the filer are :: @vol_created");
            }
            else {
                $test_status = 1;                #"**FATAL**: $FILER_C: Cannot create volumes which are other than Root volumes";
            }
        }
        logresult( "INFO", msg => $test_status );
    }    ##End of vol creation
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
=head
    execute_pre_test(
        node_present => $node_ref,
        Test         => $Test,
        filer_mode   => $Mode,
        LOGDIR       => $LOGDIR,
        BOOT_TYPE    => $BOOT_TYPE,
        test_setup   => $TEST_SETUP,
    );
=cut
    ###########################################################################################
    #Stop the scrub process if already started
    foreach my $Node (@Nodes) {
        logcomment("Ensuring that the scrub and hammer process is stopped before the script starts execution");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->execute_command( 'command' => "disk scrub stop" );
        sleep 5;
        $Api_Set_Obj->execute_command( 'command' => "hammer abort" );
        sleep 5;
    }

    return $TCD::PASS;
}

########## TEST 1 ###################################################
#NADQ02_PwrCyc
#####################################################################

sub PwrCyc {
    my @subtests;
    logcomment( "Number of nodes are " . scalar @Nodes );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&PwrCyc_Sub, -runid, "PwrCyc_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $return = status_return( subtest_ref => [@subtests] );
    logcomment("Test status is $return");
    if ( $return == 0 ) {
        return $TCD::PASS;
    }
    else {
        return $TCD::FAIL;
    }
}

#######################
##Subtest_create
#######################
sub PwrCyc_Sub {
    my @Nodes;
    push( @Nodes, shift(@_) );

##########
## Test variables
##########
    my $sto_show_s2   = {};
    my $state_phy_s3  = {};
    my $sto_show_s11  = {};
    my $state_phy_s12 = {};
    my $sto_show_data = {};
    my %power_cyc     = ();
    my $phy_3         = {};
    my $phy_12        = {};

    $Test->description("Power Cycle test");
    ###########################################################################################
    ## TEST STARTS
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my @vol_created = check_non_root_volumes( Node => $Node );
        if (@vol_created) {
            logcomment("$FILER_C : Volumes which are available in the filer are :: @vol_created");
        }
        else {
            $test_status = 1;
            logcomment("**FATAL** : Volumes are not present on the filer");
            return logresult( "INFO", msg => $test_status );
        }
    }

    #------------------------------------------------------------
## Step 1 - Set privilege level to Test
## Pass/Fail criteria: : N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 1 of 14 : $FILER_C : Set privilege level to Test" );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    }

    #---------------------------------------------------------------------------------
## Step 2 - Record all Phy Change Counts for all drives present
## Pass/Fail criteria: N/A
    #---------------------------------------------------------------------------------
    my @list_of_disk;
    my @disk_list;
    my @no_phy_cnt_2;
    my @hammer_run;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        my $disk1   = disk_show_drive_maint( API_Obj => $API_obj, Filer => $FILER_C );
        @list_of_disk = @{ $disk1->{'driveA'} };
        logcomment("Disks present in the filer $FILER_C are @list_of_disk") if (@list_of_disk);
        @disk_list = @list_of_disk;

        # remove root volume disks and spare drives
        my @aggr;
        my $filter = 'vsroot';
        try {
            ##Checking for Non Rooot Volumes
            my @non_root;
            my @volu = NACL::C::Volume->find(
                command_interface => $Node,
                filter            => { node => $FILER_C, $filter => 'false' }
            );
            foreach my $vol (@volu) {
                push( @aggr,     $vol->get_one_state_attribute('aggregate') );
                push( @non_root, $vol->volume() );
            }
            logcomment("$FILER_C : Non Root volumes are - @non_root exits");
        }
        catch NACL::Exceptions::NoElementsFound with {
            logcomment("$FILER_C : Non Root volumes are not found");
        };
        logcomment("Non Root Aggregates present in the filer : $FILER_C : are @aggr");

        # Finding the disks in the aggregates
        foreach my $raid_disks (@aggr) {
            my $API_obj = $Node->get_7m_or_nodescope_apiset();
            my $after;
            $after = $API_obj->execute_command( 'command' => "aggr status -r $raid_disks" );
            foreach $_ ( split( /\n/, $after ) ) {

                #Do not match the uninitialised
                next if ( $_ =~ /parity uninit/ );
                if ( $_ =~ /(data|dparity|parity)\s*(\S+)/ ) {

                    # copy the raid disks
                    my $disk = $2;
                    if ( $disk =~ /\w+\.\S+/ ) {
                        push( @hammer_run, $disk );
                    }
                }
            }
        }
        my @temp;

	### Checking for partition disks or boab setup 
        foreach (@hammer_run) {
        if ($_ =~ /P\d+$/) {
                (my $dsk = $_) =~ s/P\d+$//;
                push (@temp, $dsk);
        } elsif ($_=~ /L\d+$/) {
		(my $dsk = $_) =~ s/L\d+$//;	
		$BOAB_CONFIG = 1;
		push (@temp, $dsk);
	} else {
        push (@temp, $_);
        }
        }
        @list_of_disk = @temp;
    }

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Host    = host($FILER_C);
        my %disk_hash;
        eye_catcher( Test => $Test, string => "STEP 2 of 13 : $FILER_C :Record all Phy Change Counts for all drives present" );
        $sto_show_s2 = get_phy_state( Node => $Node );
        grep { $disk_hash{$_} = 1 } @list_of_disk;
        logcomment("For filer $FILER_C Phy Change Count details for all drives present are following");
        my @keys_sto = keys(%$sto_show_s2);

        foreach my $key (@keys_sto) {
            my @keys_inside = keys %{ $sto_show_s2->{$key} };
            foreach my $key1 (@keys_inside) {
                my $disk_inside = $key . '.' . $key1;

                if ( exists $disk_hash{$disk_inside} ) {
                    my $phy_cnt = $sto_show_s2->{$key}->{$key1}->{'Phy change count'};
                    if ( $phy_cnt =~ /^$/ ) {
                        push( @no_phy_cnt_2, $disk_inside );
                    }
                    else {
                        logcomment( "$FILER_C :  Phy Change Count for drive : $disk_inside : " . $phy_cnt );
                    }
                }
            }
        }
        logcomment( "**WARNING**: $FILER_C There are " . scalar(@no_phy_cnt_2) . " drives for which Phy Change count value didn't occur : @no_phy_cnt_2 - STEP : 2 \n These drives will not be validated for Power Cycle count changes in Step 12" )
          if (@no_phy_cnt_2);
    }

    #---------------------------------------------------------------------------------
## Step 3 - Record power cycle count and power cycle failures for each drive
## Pass/Fail criteria: N/A
    #---------------------------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 3 of 13 : $FILER_C :Record power cycle count and power cycle failures for each drive" );
        $state_phy_s3 = get_phy_state( Node => $Node );
        my %disk_hash;
        grep { $disk_hash{$_} = 1 } @list_of_disk;
        logcomment("For filer $FILER_C Power Cycle Count details for all drives present are following");
        my @keys_phy = keys(%$state_phy_s3);
        foreach my $key (@keys_phy) {
            my @keys_inside = keys %{ $state_phy_s3->{$key} };
            foreach my $key1 (@keys_inside) {
                my $disk_inside = $key . '.' . $key1;

                if ( exists $disk_hash{$disk_inside} ) {
                    my $pwr_cnt = $state_phy_s3->{$key}->{$key1}->{'Power cycle count'};
                    logcomment( "$FILER_C :  Power Cycle Count for drive : $disk_inside : " . $pwr_cnt );
                }
            }
        }
    }
####################################################################################################
## Step 4 - Start 2 threads of hammer
####################################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 4 of 13 : $FILER_C :Start 2 threads of hammer" );
        my @non_root = check_non_root_volumes( Node => $Node );    #Checking for non root volumes
        logcomment("$FILER_C : Total non root volumes are @non_root");
        foreach my $vol (@non_root) {
            logcomment("Step 4(1) of 21: $FILER_C : Start a thread of kernel tool Hammer with the file name /vol/$vol/f1 and size 123456");
            my $return = hammer_start(
                Node      => $Node,
                file_name => "/vol/$vol/f1",
                file_size => '123456',
                Test      => $Test
            );
            if ( $return == 0 ) {
                logcomment("Hammer Started");
            }
            else {
                $test_status = 1;
                logcomment("**FAIL**: $FILER_C : Could not Start HAMMER with the file name /vol/$vol/f1 and size 123456 - STEP : 4");
            }
            logcomment("Step 4(2) of 21: $FILER_C : Start a thread of kernel tool Hammer with the file name /vol/$vol/f2 and size 123456");
            my $return1 = hammer_start(
                Node      => $Node,
                file_name => "/vol/$vol/f2",
                file_size => '123456',
                Test      => $Test
            );
            if ( $return1 == 0 ) {
                logcomment("Hammer Started");
            }
            else {
                $test_status = 1;
                logcomment("**FAIL**: $FILER_C : Could not Start HAMMER with the file name /vol/$vol/f2 and size 123456 - STEP : 4");
            }
        }
    }

    #--------------------------------------------------------------------------------
## Step XX - Set scrub priority level.
## Pass/Fail criteria: options raid.scrub.perf_impact nnnn
## nnnn can be either low, mediun or high - setting to high
## would provide more CPU utilization to scrub process.
    #--------------------------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        $Test->nlog("$FILER_C: Set scrub priority level");
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        $API_obj->execute_command( 'command' => 'options raid.scrub.perf_impact medium' );
    }
#####################################################################################################
## Calcualte time out 
#####################################################################################################
	my $TIME_OUT_MIN;
	my $TIME_OUT_MAX;
	my $timeout_value;
    my $max_disk_size = get_max_disk_size(
        node_present => \@Nodes
    );
	logcomment("Max disk size : $max_disk_size GB");
	$max_disk_size = ($max_disk_size * 2147483648)/1000;
	if ( $max_disk_size <= 20971520 ) {
        $TIME_OUT_MIN  = 300;
        $TIME_OUT_MAX  = 14400;
        $timeout_value = 43200;
    }elsif ( $max_disk_size <= 2147483648 ) {
        $TIME_OUT_MIN  = 300 * 5;
        $TIME_OUT_MAX  = 14400 * 5;
        $timeout_value = 64800;
    }else {
        $TIME_OUT_MIN  = 300 * 25;
        $TIME_OUT_MAX  = 14400 * 25;
        $timeout_value = 72000;
    }

#####################################################################################################
## EXECUT 20 FISERSIO WORK LOAD TO THE SCRIPT. 
#####################################################################################################
	foreach my $Node (@Nodes) {
		my $FILER_C = $Node->name();
		my $host = host($FILER_C);
		eye_catcher( Test => $Test, string => "CHECK FOR NON ROOT AGGREGATE, AND INITATE FILERSIO WORKLOAD");
		my @NON_ROOT_VOLS = check_non_root_volumes( Node => $Node );
		if (@NON_ROOT_VOLS) {
	        logcomment("$FILER_C : Non-root volume used for this test is : @NON_ROOT_VOLS");
        }else{
	        logcomment("**FATAL**: $FILER_C : No non-root volume is not present for testing - STEP : 5");
    	    return logresult( "INFO", msg => 1);
        }
		#####################################################################################################
		## Create 20 files 
		#####################################################################################################
	    my $vol_size;
	    my $file_size;
	    eye_catcher(Test => $Test, string => "Create Files to initiate filersio work load");
		foreach my $vol_name(@NON_ROOT_VOLS){
			my $Api_Set_Obj  = $Node->get_7m_or_nodescope_apiset();
			my $output=$Api_Set_Obj->execute_command( 'command' => "df $vol_name");
=head
			foreach my $line ( split( /\n/, $output)){
    	        if ( $line =~ /^\/vol\/$vol_name\/\s+(\d+)\s+(\d+)/ ) {
	                $vol_size = $1;
    	            logcomment("Actual size of volume $vol_name is $vol_size kbytes");
	                $file_size = int(($vol_size * 0.95) / 21);
    	            $file_size = $file_size."k";
        	    }
	        }	
	        for ( my $no_files = 0 ; $no_files <= 19 ; $no_files++ ) {
#            	$Api_Set_Obj->execute_command( 'command' => "mkfile $file_size /vol/$vol_name/file-$no_files", 'timeout' => $TIME_OUT_MAX );
        	}
		    logcomment("Creation of 20 files on vol1 of $FILER_C  successful");
=cut
		}
		######################################################################################################
		## Initiate work load using filer sio
		######################################################################################################
		eye_catcher(Test => $Test, string => "Initiate work load");
		my $Api_Set_Obj  = $Node->get_7m_or_nodescope_apiset();
	    foreach my $vol_name(@NON_ROOT_VOLS){
    	    for ( my $no_files = 0 ; $no_files <= 19 ; $no_files++ ) {
        	    $Api_Set_Obj->execute_command( 'command' => "filersio asyncio_active 80 -r 50 4k 0 12g 48200 10 /vol/$vol_name/file-$no_files -create", 'timeout' => $TIME_OUT_MAX );
	        }
		}
		logcomment("Work load initiated on $FILER_C");
	}


######################################################################################################
## Step 5 - Power cycle each drive in succession in the file-system that hammer is running on
######################################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
		my $host = host($FILER_C);
        eye_catcher( Test => $Test, string => "STEP 5 of 13 : $FILER_C :Power cycle each drive in succession in the filesystem that hammer is running on" );
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        #logcomment( "FILER  $FILER_C : Power cycling all the drives on which hammer is running - " . scalar(@hammer_run) . " drives @hammer_run" );
	logcomment("FILER  $FILER_C : Power cycling all the drives on which hammer is running - " . scalar(@list_of_disk). " drives @list_of_disk");
        #if ( scalar(@hammer_run) == 0 ) {
	if ( scalar(@list_of_disk) == 0) {
            logcomment("**FATAL**: $FILER_C Test case will not be proceed because non root volumes not exists or hammer is not running on the disks - STEP : 5");
            return logresult( "INFO", msg => 1 );
        }

        #Get the adapter and shelf ids
        #######my @ada_shlfids = extract_ada_shlf(@hammer_run);
	my @ada_shlfids = extract_ada_shlf(@list_of_disk);
        logcomment( "$FILER_C : Power cycling started at " . scalar( localtime() ) );

        #my $five_mins   = 0;
        my $loop = 1;
        logcomment("Alotted Sleep Time : $POW_SLP_TIME seconds\n");
        my $start_time_sec = time;
        my $end_time_sec   = $start_time_sec + $test_wait_time;

        while ( $start_time_sec < $end_time_sec ) {
            my $step_6 = 1;
            eye_catcher( Test => $Test, string => "STEP 5($loop) of 13 : $FILER_C :Power cycle each drive in succession in the filesystem that hammer is running on" ) if ( $loop > 1 );
            my $first = 1;
            foreach my $sel_drive (@list_of_disk) { #(@hammer_run) {
                my $API_obj = $Node->get_7m_or_nodescope_apiset(connid=> "console");
				logcomment("Sleep time between each powercycle - $POW_SLP_TIME");
				logcomment("Check for Drive rediness");
				my $outop = $API_obj->execute_raw_command('command' =>"run local scsi test $sel_drive", 'timeout' => 600 );
				logcomment("DEBUG - Output is -$outop");
				my $power_c = 0;
				if($outop =~ /drive ready/){
					logcomment("Drive $sel_drive is ready for power cycling");
					$power_c = 1;
				}else{
					logcomment("Drive $sel_drive is not ready for power cycling");
				}
				if ($power_c == 1) {
					logcomment("Power cycling drive - $sel_drive");
	                $API_obj->execute_command( 'command' => "sasadmin dev_power_cycle $sel_drive", 'timeout' => 600 );
	                $power_cyc{$sel_drive}++;
	                eye_catcher( Test => $Test, string => "STEP 6 of 13 : $FILER_C :Dwell time between power cycles " ) if(($first == 1) && ($loop == 1));
	                eye_catcher( Test => $Test, string => "STEP 6($loop) of 13 : $FILER_C :Dwell time between power cycles " ) if(($first == 1) && ($loop > 1));
    	            logcomment("Sleeping for $POW_SLP_TIME seconds");
	                sleep($POW_SLP_TIME);
					logcomment("Check for Drive status");
					my $sc_ot = $API_obj->execute_raw_command('command' =>"run local scsi test $sel_drive", 'timeout' => 600 );
					if($sc_ot =~ /drive ready/){
	                    logcomment("Drive $sel_drive is good state");
					}else{
						logcomment("**FATAL** : SCSI test returned error after power cycling on - $sel_drive");
						logresult("FATAL","SCSI test returned error after power cycling on - $sel_drive");
					}
				}else{
					logcomment("**FATAL** : SCSI test returned error on - $sel_drive");
					logresult("FATAL","SCSI test returned error on - $sel_drive");
				}
				$API_obj->set_timeout("connectrec-timeout"=>18000);
                $API_obj->execute_raw_command('command' =>"\013");
                $API_obj->execute_raw_command('command' =>"\013");
                $API_obj->execute_raw_command('command' =>"\013");
                my $current_time = time;
                my $elapsed_time = $current_time - $start_time_sec;
                #$start_time_sec = $current_time;
                #$five_mins      = $five_mins + $elapsed_time;
                last if ( $start_time_sec > $end_time_sec );
                if ( $elapsed_time >= ( 300 * $step_6 ) ) {    # || ($step_6 == 0))
                    eye_catcher( Test => $Test, string => "STEP 7 of 13 : $FILER_C :Verifying interface speed after every 5 mins on all expanders @ada_shlfids" ) if ( ( $loop == 1 ) && ( $step_6 == 1 ) );
                    logcomment("Verifying interface speed after every 5 mins on all expanders @ada_shlfids in between power cycling") if ( $step_6 > 1 );
                    eye_catcher( Test => $Test, string => "STEP 7($loop) of 13 : $FILER_C :Verifying interface speed after every 5 mins on all expanders @ada_shlfids" ) if ( ( $loop > 1 ) && ( $step_6 == 1 ) );
                    $step_6++;
                    foreach my $count (@ada_shlfids) {
                        logcomment("Verifying interface expander $count");
                        $API_obj->execute_command( 'command' => "sasadmin expander_cli $count \"phydump\"" );
                    }
                }

                $first++;
            } ## end of foreach $sel_drive (@hammer_run)
            if ( $loop == 1 ) {
                eye_catcher( Test => $Test, string => "STEP 8 of 13 : $FILER_C :Repeat steps 5-8 for 12 hours" );
                my ( $sec, $min, $hrs ) = gmtime($test_wait_time);
                logcomment("User selected -w=$TEST_WAIT_TIME so the looping time will be $hrs HOURS $min MINUTES and $sec SECONDS");
                logcomment( "Start time for looping is " . scalar( localtime() ) );
                $end_time_sec = time + $test_wait_time;
            }
            $start_time_sec = time;
            if ( $loop > 1 ) {
                my $time_left = $end_time_sec - $start_time_sec;
                my ( $lsec, $lmin, $lhrs ) = gmtime($time_left) if ( $time_left > 0 );
                logcomment("Remaining time for looping is $lhrs HOURS $lmin MINUTES and $lsec SECONDS") if ( $time_left > 0 );
            }
            $loop++;
        } ## end of while ($start_time_sec < $end_time_sec)
        logcomment( "End time for looping is " . scalar( localtime() ) );
        logcomment( "Power cycling on all the drives completed on filer $FILER_C " . scalar( localtime() ) );
        foreach $_ ( keys %power_cyc ) {
            logcomment("Drive : $_ \t Power Cycled : $power_cyc{$_} times\n");
        }
    }
    @Nodes = node_new_obj( node_array => [@Nodes] );
#############################################################################################
## Step 9 - Stop all threads of kernel tool Hammer
## Pass/Fail criteria: N/A
###########################################################################################

    foreach my $Node (@Nodes) {
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 9 of 13 : $FILER_C :Stop all threads of kernel tool Hammer" );
        my $ret;
        try {
            logcomment("$FILER_C : Aborting hammer");
            $ret = $API_obj->hammer_abort();
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            my $exception_object = shift;
            my $resp             = $exception_object->get_response_object();
            $Test->nlog( "Caught a " . ref($exception_object) . "!" );
            $Test->nlog( "Error is: " . $exception_object->text() );
            $Test->nlog( "Raw output was:\n" . $resp->get_raw_output() );
            $Test->nlog("Hammer abort has not done properly");
        };
    }

###########################################################################################
## Step 10 - Stop any disk scrub that might be running.
## Pass/Fail criteria: N/A
###########################################################################################
    foreach my $Node (@Nodes) {
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 10 of 13 : $FILER_C :Stop any disk scrub that might be running" );
        try {
            $API_obj->execute_command( 'command' => 'disk scrub stop' );
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            logcomment("Disk srub cannot be stopped");
        };
    }

    #---------------------------------------------------------------------------------
## Step 11 - Record all Phy Change Counts for all drives present
## Pass/Fail criteria: N/A
    #---------------------------------------------------------------------------------
    my @no_phy_cnt_11;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Host    = host($FILER_C);
        my %disk_hash;
        eye_catcher( Test => $Test, string => "STEP 11 of 13 : $FILER_C :Record all Phy Change Counts for all drives present" );
        $sto_show_s11 = get_phy_state( Node => $Node );
        grep { $disk_hash{$_} = 1 } @list_of_disk;
        logcomment("For filer $FILER_C Phy Change Count details for all drives present are following");
        my @keys_sto = keys(%$sto_show_s11);

        foreach my $key (@keys_sto) {
            my @keys_inside = keys %{ $sto_show_s11->{$key} };
            foreach my $key1 (@keys_inside) {
                my $disk_inside = $key . '.' . $key1;

                if ( exists $disk_hash{$disk_inside} ) {
                    my $phy_cnt = $sto_show_s11->{$key}->{$key1}->{'Phy change count'};
                    if ( $phy_cnt =~ /^$/ ) {
                        push( @no_phy_cnt_11, $disk_inside );
                    }
                    else {
                        logcomment( "$FILER_C :  Phy Change Count for drive : $disk_inside : " . $phy_cnt );
                    }
                }
            }
        }
        logcomment( "**WARNING**: $FILER_C There are " . scalar(@no_phy_cnt_11) . " drives for which Phy Change count value didn't occur : @no_phy_cnt_11 - STEP : 11 \n These drives will not be validated for Power Cycle count changes in Step 12" )
          if (@no_phy_cnt_11);
    }

    #----------------------------------------------------------------------------------------------
## Step 12 - Record power cycle count and power cycle failures for each drive
## Pass/Fail criteria: Make sure the phy change count has incremented correctly for each drive
    #----------------------------------------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 12 of 13: $FILER_C :Record power cycle count and power cycle failures for each drive" );
        $state_phy_s12 = get_phy_state( Node => $Node );
        my %disk_hash;
        grep { $disk_hash{$_} = 1 } @list_of_disk;
        logcomment("For filer $FILER_C Power Cycle Count details for all drives present are following");
        my @keys_phy = keys(%$state_phy_s12);
        foreach my $key (@keys_phy) {
            my @keys_inside = keys %{ $state_phy_s12->{$key} };
            foreach my $key1 (@keys_inside) {
                my $disk_inside = $key . '.' . $key1;

                if ( exists $disk_hash{$disk_inside} ) {
                    my $pwr_cnt = $state_phy_s12->{$key}->{$key1}->{'Power cycle count'};
                    logcomment( "$FILER_C :  Power Cycle Count for drive : $disk_inside : " . $pwr_cnt );
                }
            }
        }
    }
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        sort (@list_of_disk);
        sort (@no_phy_cnt_2);
        @list_of_disk = array_diff( [@list_of_disk], [@no_phy_cnt_2] );
        sort (@list_of_disk);
        sort (@no_phy_cnt_11);
        @list_of_disk = array_diff( [@list_of_disk], [@no_phy_cnt_11] );
        my $disks = \@list_of_disk;
        my $result = compare_phy_state( before => $state_phy_s3, after => $state_phy_s12, disk_ref => $disks, Node => $Node, compare => 'y', boab => $BOAB_CONFIG );

        if ( $result !~ /true/i ) {
            $test_status = 1;
	    my @fail_drives = @$result;
            logcomment("**FAIL**: $FILER_C - The increment in the Power Cycle count is incorrect for drives : @fail_drives - STEP : 12");
        }
        else {
            logcomment("$FILER_C - The Power Cycle count has incremented correctly for each drive");
        }
    }

    #---------------------------------------------------------------------------------
## Step 13 - Pull BSAS link speed logs
## Pass/Fail criteria: These additional steps are only for BSAS drives
    #---------------------------------------------------------------------------------
	@Nodes = node_new_obj( node_array => [@Nodes] );
    if ( $TEST_SETUP !~ /FC/ig ) {
        foreach my $Node (@Nodes) {
            my $API_obj = $Node->get_7m_or_nodescope_apiset();
            my $FILER_C = $Node->name();
            eye_catcher( Test => $Test, string => "STEP 13 of 13 : $FILER_C :Pulling BSAS link speed logs" );
            my $Mode = $Node->mode();
            my $stsb_status = get_filer_check( Node => $Node );
            $sto_show_data = storage_show_disk_a_data( Node => $Node );

            #  Collect the SM/SA/SDB disks if it is BSAS
            my ( $ssm, $ssa, $sdb, $dir );
            ( $ssm, $ssa, $sdb ) = ( 0, 0, 0 );
            my %dr_id = ();

            my @temp = grep { !/P\d+$/ } @disk_list;
            logcomment("Checking Model number of the disks @temp");
            foreach my $disk (@temp) {
                my $mdl = $sto_show_data->{$disk}->{'Model'};
                logcomment("Filer : $FILER_C : Model of disk $disk is $mdl");
                if ( $mdl =~ /(SA)$/ ) {
                    $ssa = 1;
                    $dr_id{$disk} = $1;
                    logcomment("SA: $disk - $1");
                    logcomment("Filbert(SSA): Writing crash dump to file of the $disk - type $dr_id{$disk}");
                    $API_obj->execute_command( 'command' => "$stsb_status get_crash_dump  $disk" );
                }
                elsif ( $mdl =~ /(SM)$/ ) {
                    $ssm = 1;
                    $dr_id{$disk} = $1;
                    logcomment("SM: $disk - $1");
                    logcomment("Acorn(SSM):  Scsi modesense of the disk $disk - type $dr_id{$disk}");
                    $API_obj->execute_command( 'command' => "scsi modesense -p 0x3a $disk" );
                }
                elsif ( $mdl =~ /(SDB)$/ ) {
                    $sdb = 1;
                    $dr_id{$disk} = $1;
                    print "SSD: $disk - $1\n";
                    logcomment("Marvell(SDB): Acorn get status of the disk $disk - type $dr_id{$disk}");
                    $API_obj->execute_command( 'command' => "$stsb_status get_status  $disk" );
                }
            }
            if ( ( $ssm == 0 ) && ( $ssa == 0 ) && ( $sdb == 0 ) ) {
                logcomment("Step 13: No SM/SA/SDB drives found - Exiting the Step 13");
            }
        }    # end foreach my $Node (@Nodes)
    }
    else {
        eye_catcher( Test => $Test, string => "STEP 13 of 13 : Pulling BSAS link speed logs" );
        logcomment("These additional steps are only for BSAS and MSATA drives");
    }

    ###########################################################################################
    ## Test log/results
    ###########################################################################################
    logresult( "INFO", msg => $test_status );
}

#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
	@Nodes = node_new_obj( node_array => [@Nodes] );
    logcomment(" $filer_names - Clean up and post test process");
#    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ###########################################################################################
=head
    execute_post_test(
        filer_mode      => $Mode,
        node_present    => [@Nodes],
        Test            => $Test,
        change_state_to => "CLI",
        LOGDIR          => $LOGDIR,
        test_setup      => $TEST_SETUP,
        BOOT_TYPE       => $BOOT_TYPE,
    );
=cut
    return $TCD::PASS;
}

####################################
## SUBROUTINES
####################################
# This function extracts adapter and shlf name of the disks and passes into array
sub extract_ada_shlf {
    my (@extract) = (@_);
    my %uniq;
    my ( @ada, $shlf );
    foreach $_ (@extract) {
        $shlf = $_;
        $shlf =~ s/(\S+\.)[0](\S+)\.\S+/$1$2/;
        $shlf =~ s/(\S+\.)(\S+)\.\S+/$1$2/;
        if ( ( ++$uniq{$shlf} ) == 1 ) {
            push( @ada, $shlf );
        }
    }
    logcomment("Adapters and shlf ids: @ada");
    return @ada;
} ## end of extract_ada_s

###EMS CEHCK
sub check_ems_msg{
    my $time_stamp = $_[0];
    my $disk = $_[1];
    my $host = $_[2];
    my $found_abort = 0;
    logcomment("Checking for iSenseKey=\"0x4\" and iASC=\"0x3e\" on Disk $disk in the ems logs");
    my $event_by_type = ems_log_dump_by_id(host=>$host, timestamp=>$time_stamp);
    my @time_stamps = keys %$event_by_type;
    foreach my $time(@time_stamps){
        if ((($event_by_type->{"$time"}->{"sSenseKey"} eq "SCSI:hardware error") && ( $event_by_type->{"$time"}->{"deviceName"} eq "$disk") && ($event_by_type->{"$time"}->{"iSenseKey"} eq "0x4") && ($event_by_type->{"$time"}->{"iASC"} eq "0x3e")) && ($found_abort == 0)) {
            logcomment("FOUND - iSenseKey=\"0x4\" and iASC=\"0x3e\" on Disk $disk with sSenseKey as \"SCSI:hardware error\"");
            $found_abort++;
            return $found_abort;
        }else{
            my $isenseKey = $event_by_type->{"$time"}->{"iSenseKey"};
            my $ssenseKey = $event_by_type->{"$time"}->{"sSenseKey"};
            my $iASC = $event_by_type->{"$time"}->{"iASC"};
            my $dsk = $event_by_type->{"$time"}->{"deviceName"};
            if ($ssenseKey ne ""){
                logcomment("For Disk - $dsk - sSenseKey:$ssenseKey | iSenseKey:$isenseKey | iASC:$iASC");
            }
        }
    }# end of foreach

    if ($found_abort ==0 ){
        logcomment("NOT FOUND - iSenseKey=\"0x4\" and iASC=\"0x3e\" on Disk $disk with sSenseKey as \"SCSI:hardware error\"");
        return $found_abort;
    }
}# end of sub