#!/usr/suftware/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/BR/NADQ02_FC_Disk_Qual_WRV.thpl#17 $
#
## @summary Mode
## disk_qual Write/Read/Verify
#
##
## @description
##  Verify that the drive can handle data properly, using the pass-thru driver layer.
##
## @Test Mode
##  Maintenance mode
##
## @Test bed setup
##  SATA  :Cluster Fabric
##  SAS   :Cluster
##  Cmode :SETUP
##
##  Note: A Finisar Analyzer with SanMetrics SW must be used, and a test summary for each step below must
##  be included in the results for the test pass conducted .
##
## @usage
## The test can be run independently or with other tests as part of an STEST.
##
## @steps
##  The test will execute commands mentioned below:
##   1.  Output user remarks
##   2.  Set disk_qual_time_limit to 50000.
##   3.  Set SyncScripting parameter to 5.
##   4.  Executes the default disk_qual test script on all drives. Record the results.
##   5.  Record summary of script results
##   6.  Record the log sense data from each drive.
##   7.  Force disk_qual to report but not stop on media errors.
##   8.  Sequential write noverify for > 4 hrs
##   9.  Wait for all disks to report results and stop testing.
##  10.  Compare the log sense data from each drive. Note any increase in errors since step 6.
##  11.  Start MIXED TEST using 67% Random Reads, execute more than 4 hrs
##  12.  Wait for all disks to report results and stop testing.
##  13.  Start Random Read Noverify Test, execute for more than 4 hrs
##  14.  Wait for all disks to report results and stop testing
##  15.  Start Sequential Verify Test, execute for more than 4 hrs
##  16.  Wait for all disks to report results and stop testing
##
##
## @param FILER       - optional for cluster setup
##                    - required for Dual path setup      Name of filer to be used
## @param FILERA      - required for cluster setup
##                    - optional for Dual path setup      Name of filer to be used
## @param FILERB      - required for cluster setup
##                    - optional for Dual path setup      Name of the partner node
## @param TEST_CONFIG - optional Type of the setup, C for cluster, D for dual path (default)
## @param TEST_SETUP  - optional Default value set to 'FC'
## @param FILER_CONN  - optional Type of connection, default is set to 'console' for this test
## @param Mode        - optional Filer mode maint/normal, default set to 'maint' for this test
## @param FILER_PROMPT - optional prompt of the filer
## @param LOGDIR      - optional This is required to generate 'END-LOG'. Default set to main log directory.
## @param EOE         - optional 'default' is set to default value
## @param BOOT_MODE   - optional Boot mode for filer , default set to '5' for this test
## @param BOOT_TYPE   - optional Boot type, default set to 'A' (automatic) for this test
## @param SSD         - optional Setup is SSD or not, default set to 'no'
## @param EMAIL       -  optional Default value set to 'Y'
## @param EMAIL_TO    - required for setting to address sending email to recipient
## @param EMAIL_FROM  - required for setting  from address for sending email
## @param FILER_TYPE -  required to differentiate between Boilermaker and Ironcity filers.
## @param ARMADILLO  -  This parameter will indicate if an ARMADILLO configuration is under testing (if yes, ARMADILLO=1 else ARMADILLO=2)
##
## @status  Automated
## @auther  arunak@netapp.com
## @burt    
## @change  YY/MM/DD from auther: Description of the change
##
##
#############################################################################
########################################
### Library functions
##########################################
# Compiler directives.
use strict;

# Module imports
use TCD;
use NACL::APISet;
use NACL::C::Node;
use NACL::APISet::Host;
use NACL::C::Disk;
use NACL::Transit;
use Storage::Common_Lib;
use Storage::NVMe_Common_Lib;
use Storage::SASUtils qw(disable_dbg_msgs);

use vars qw(
  $FILER
  $FILERA
  $FILERB
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $MODE
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $MAIL_TO
  $MAIL_FROM
  $EMAIL
  $FILER_TYPE
  $ARMADILLO
);

my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$FILERA         = $params->get( 'FILERA',         default => '' );
$FILERB         = $params->get( 'FILERB',         default => '' );
$FILER_PROMPT   = $params->get( 'FILER_PROMPT',   default => '\*>' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 'default' );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '5' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'D' );
$FILER_CONN     = $params->get( 'FILER_CONN',     default => '#console' );
$MODE           = $params->get( 'MODE',           default => 'maint' );

################
# Testcase name
################
my $TC_name = "201_NADQ02_NVMe_Disk_Qual_WRV";

#########################################
### Initialization/declaration
#########################################
my $test_status = 0;
my %FILER_API   = ();
my @Nodes;
my $filer_names;
my $Mode;
my $FILER_STATE;
my $API_Object;

my ( $email_subject, $email_body );

########################################
# Testcase available for execution
#########################################
my @Testcases = ( Disk_Qual_WRV => "Disk_Qual_WRV  - Maintenance mode" );

#########################################
## Pre-test processes
##########################################
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );    # Testcases will be executed using TCD object.
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main

########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes  = NACL::C::Node->find();     # Find Nodes/Filers used in the test, Based on FILER param.
    my %nodes_filer = {};
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node objects used for test execution.
    grep { logcomment( "Filers passed to the script are " . $_->name() ) } @Nodes;
    $Mode = $Nodes[0]->mode();                                      # gets mode(7mode/cmode) of the nodes used in test execution.
    logcomment("Checking for execution mode");

    version_test( node_present => \@Nodes, tc_name => $TC_name );

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );

        #$FILER_STATE = $Transit_obj->get_state();
        $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        $Test->nlog("FILER- $FILER_C - Is in $FILER_STATE state");
        if ( $FILER_STATE !~ /MAINT/ ) {
            $Transit_obj->change_state( to => "MAINT" );
            $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        }
        if ( $FILER_STATE =~ /MAINT/ ) {
            my $host = host($FILER_C);
            $API_Object = NACL::APISet->new(
                hostobj   => $host,
                category  => "Node",
                interface => "CLI",
                set       => "Maintenance"
            );
            $FILER_API{$FILER_C} = $API_Object;
        }
    }
    $Test->nlog( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    $Test->nlog( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}

########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test exectuion ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
	#####################################################################
    # Pre test proces : call for pre_n_post test process
	#####################################################################
	nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );

    my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
    my $disk_qual_path = $actual_log_dir . "/disk_qual_cnt_$TC_name.log";
    logcomment("**DEBUG** : log path : $disk_qual_path");
    open( FH, ">>$disk_qual_path" ) || die "Failed to open the write file $!";
    close FH;
    return $TCD::PASS;
}

########## TEST 1 ###################################################
#Disk_Qual_WRV
####################################################################
sub Disk_Qual_WRV {
    my @subtests;
    logcomment("Number of nodes are @Nodes");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&Disk_Qual_WRV_sub, -runid, "Disk_Qual_WRV_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $return = status_return( subtest_ref => [@subtests] );
    logcomment("Test status is $return");
    if ( $return == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub Disk_Qual_WRV_sub {
    my ( $driveA, $driveB, @Nodes, @disk_arr_driveA, @disk_arr_driveB, @subtests, $fail_cnt, $fatal_cnt );
    my $TIME_OUT_5   = 300;
    my $disk_max_sup = 40;
    my $TIME_OUT_30  = 1800;
    push( @Nodes, shift(@_) );
    my %disk_qual_cnt;
    $Test->description("Disk_Qual_WRV  Maintenance mode ");
    my $test_wait_time = 1800;
    if ( $TEST_WAIT_TIME == 1 ) {
        $test_wait_time = ( $test_wait_time / 10 );
    } elsif ( $TEST_WAIT_TIME == 2 ) {
        $test_wait_time = ( $test_wait_time / 2 );
    }
    my $test_time_step    = ( 8 * $test_wait_time );
    my $test_time_step_hr = ( $test_time_step / 3600 );
    my ( $sec, $min, $hr ) = gmtime($test_time_step);
    my $expected_comp_time = ( ( 32 * ($test_wait_time) / 3600 ) + 2 );

    #------------------------------------------------------------
	## Step 1 - Output user remarks
	## Pass/Fail criteria: N/A
    #------------------------------------------------------------

    my @disk_array = get_disks_list( Node => $Nodes[0] );
	my $drv_list_all = nvme_list_only_nvme( node_present => [$Nodes[0]] );
	my @disk_array = keys(%$drv_list_all);
    my $disk_length = @disk_array;
    logcomment("Disk Length= $disk_length");
    my $FILER_C          = $Nodes[0]->name();
    my $API              = $FILER_API{$FILER_C};
    my $disk_show_output = disk_show_drive_maint(
        API_Obj => $API,
        Filer   => $FILER_C
    );
    $driveA          = $disk_show_output->{driveA};
    $driveB          = $disk_show_output->{driveB};
    @disk_arr_driveA = @$driveA;
    @disk_arr_driveB = @$driveB;
    logcomment("disk_arr_driveA = @disk_arr_driveA and disk_arr_driveB = @disk_arr_driveB");

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 1 of 16 :$FILER_C Output user remarks",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my @disk_list_q = ();

        while (@drive_array) {
            my $temp_disk;
            my @disk_list_q = ();
            my $loop_no     = 0;
            while ( $loop_no < $disk_max_sup ) {
                $temp_disk = pop(@drive_array);
                push( @disk_list_q, $temp_disk );
                $loop_no++;
                last if ( !(@drive_array) );
            }
            $API->execute_raw_command( command => "disk_qual remarks -d @disk_list_q -o 'run disk_qual script'" );
            sleep(100);
	        logcomment("Execute defined default disk_qual run script on node :$FILER_C");
	        run_default_dsk_ql_scpt( Node => $Node, disk_list_q => \@disk_list_q , drv_typ => "NVMe");          #1031605 #1063742
    	    logcomment("Completed default disk_qual runscript");
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
        }
        logcomment("$FILER_C - Output user remarks completed");
        disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
    }

    #------------------------------------------------------------
	## Step 2 - Set disk_qual_time_limit to 50000
	## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 2 of 16 :$FILER_C disk_qual setparam TimeoutValue to 50000",
        );
        $API->execute_raw_command( command => "disk_qual setparam TimeoutValue 50000" );
        logcomment("$FILER_C - disk_qual_time_limit set to 50000");
    }

    #------------------------------------------------------------
	## Step 3 - Set SyncScripting parameter to 5
	## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 3 of 16 :$FILER_C Set SyncScripting parameter to 5",
        );
        $API->execute_raw_command( command => "disk_qual setparam SyncScripting 5" );
        logcomment("$FILER_C - SyncScripting parameter set to 5");
    }

    #------------------------------------------------------------
	## Step 4 - Executes the default disk_qual test script on all drives. Record the results.
	## Pass/Fail criteria: N/A	
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 4 of 16 :$FILER_C Executes the default disk_qual test script on all drives. Record the results.",
        );
		my @drive_array_runscript = ();
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my @disk_list_q = ();
        @drive_array_runscript = @drive_array;

        while (@drive_array) {
            my $temp_disk;
            my $loop_no = 0;
            while ( $loop_no < $disk_max_sup ) {
                $temp_disk = pop(@drive_array);
                push( @disk_list_q, $temp_disk );
                $loop_no++;
                last if ( !(@drive_array) );
            }
            grep { $disk_qual_cnt{$_}++ } @disk_list_q;
            push( @subtests, subtest( \&_disk_qual_run_script, -runid, "_disk_qual_run_script_$FILER_C", -bg, "--", $API, \@disk_list_q, $Node, -onexit, \&clean_up ) );
            Subtest::wait_finish( subtest => [@subtests] );
            foreach my $subtest (@subtests) {
                my @results = $subtest->rslts();
                foreach my $result (@results) {
                    if ( $result->type() =~ /INFO/ ) {
                        logcomment("subtest _disk_qual_run_script completed successfully");
                    } elsif ( $result->severity() >= 0 ) {
                        logcomment("subtest _disk_qual_run_script not completed successfully");
                        if ( $result->type() =~ /FAIL/i ) {
                            $fail_cnt++;
                        }
                        if ( $result->type() =~ /FATAL|SCRIPT|CONFIG/i ) {
                            $fatal_cnt++;
                        }

                    } ## end elsif ($result->severity(...
                } ## end foreach my $result (@results)
            } ## end foreach my $subtest (@subtests)
            if ( $fail_cnt == 0 ) {
                $Test->nlog( "$FILER_C : $TC_name : Disk qual run scripts test completed successfully " . scalar( localtime() ) );
            } else {
                $test_status = 1;
#                logcomment("**FAIL**: $FILER_C : Disk qual run scripts test failed to complete successfully - STEP : 4");
logcomment("**FAIL**: $FILER_C :DISK QUAL RUN SCRIPTS TEST FAILED TO COMPLETE SUCCESSFULLY - STEP : 4");
            }
        }
        disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
    }

    #------------------------------------------------------------
## Step 5 - Record summary of script results.
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 5 of 16 :$FILER_C Record summary of script results.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my $max_time_out = get_max_timeout( node_present => \@Nodes, Test => $Test );
        $API->set_timeout( "connectrec-timeout" => $max_time_out );

        while (@drive_array) {
            my $temp_disk;
            my $loop_no     = 0;
            my @disk_list_q = ();
            while ( $loop_no < $disk_max_sup ) {
                $temp_disk = pop(@drive_array);
                push( @disk_list_q, $temp_disk );
                $loop_no++;
                last if ( !(@drive_array) );
            }
            my $disk_attached = \@disk_list_q;
            $API->execute_raw_command( command => "disk_qual summary -d @disk_list_q" );
            sleep(200);
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
        }
        $Test->nlog("$FILER_C - Record summary of script results completed");
        disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
    }

    #------------------------------------------------------------
## Step 6 - Record the log sense data from each drive.
## Pass/Fail criteria:
    #------------------------------------------------------------
    my %disk_qual_s6 = {};
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 6 of 16 :$FILER_C Record the log sense data from each drive.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my $max_time_out = get_max_timeout( node_present => \@Nodes, Test => $Test );
        $API->set_timeout( "connectrec-timeout" => $max_time_out );

        while (@drive_array) {
            my $temp_disk;
            my $loop_no     = 0;
            my @disk_list_q = ();
            while ( $loop_no < $disk_max_sup ) {
                $temp_disk = pop(@drive_array);
                push( @disk_list_q, $temp_disk );
                $loop_no++;
                last if ( !(@drive_array) );
            }
            my $disk_attached = \@disk_list_q;
            grep { $disk_qual_cnt{$_}++ } @disk_list_q;
            $disk_qual_s6{$Node} = run_disk_qual_t12( API_Obj => $API, disk_list_local => $disk_attached );

            # logcomment("Keys returned are : ".keys$disk_qual_s6{$Node});
            # logcomment ("Values are :".values$disk_qual_s6{$Node});
            sleep(200);
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
        }
        $Test->nlog("$FILER_C - Record the log sense data completed");
        disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
    }

    #------------------------------------------------------------
## Step 7 - Force disk_qual to report but not stop on media errors.
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 7 of 16 :$FILER_C Force disk_qual to report but not stop on media errors.",
        );
        $API->execute_raw_command( command => "disk_qual setparam ContinueOnError 1" );
        $Test->nlog("$FILER_C - Force disk_qual to report but not stop on media errors completed");
    }

    #------------------------------------------------------------
## Step 8 - Sequential write noverify
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 8 of 16 :$FILER_C Executing Sequential write noverify test.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my @disk_list_q = ();
        my $max_time_out = get_max_timeout( node_present => \@Nodes, Test => $Test );
        $API->set_timeout( "connectrec-timeout" => $max_time_out );

        while (@drive_array) {
            my $temp_disk;
            my $loop_no = 0;
            while ( $loop_no < $disk_max_sup ) {
                $temp_disk = pop(@drive_array);
                push( @disk_list_q, $temp_disk );
                $loop_no++;
                last if ( !(@drive_array) );
            }
            grep { $disk_qual_cnt{$_}++ } @disk_list_q;
            $API->execute_raw_command( command => "disk_qual start -t 21 -q 8 -L 'forever' -n 512 -d @disk_list_q -o noverify" );

            sleep(200);
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
        }
        $Test->nlog("$FILER_C - Sequential write noverify started");
    }

    my $step      = 1;
    my $time_left = 0;
    for ( $step = 1 ; $step <= 8 ; $step++ ) {
        sleep $test_wait_time;
        $time_left = ( $test_time_step_hr - ( ( $step * $test_wait_time ) / 3600 ) );
        $time_left = sprintf( "%.2f", $time_left );
        my ( $s, $m, $h ) = gmtime( $time_left * 3600 );
        $Test->nlog( "Sequential write noverify time left $time_left Hr (i.e) $h HRS $m MINS $s SECS \nTotal time $hr HRS $min MINS $sec SECS : " . scalar( localtime() ) );
        $expected_comp_time = ( $expected_comp_time - ( $test_wait_time / 3600 ) );
	        my $output = $API->execute_raw_command( 'command' => "disk_qual status" );
            if ( $output =~ /idle/ ) {
                logcomment("**FAIL** : SEQUESTIAL WRITE NOVERIFY TEST IS STOPPED ");
                return logresult( "INFO", msg => 1 );
            } else {
                logcomment(" Sequential write noverify TEST is Still in Progress");
            }


        #$Test->nlog("Time left for the test to complete $expected_comp_time Hrs ". scalar( localtime() ) );

        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            my $API_obj = $FILER_API{$FILER_C};
            $API_obj->execute_raw_command( command => "\013" );
        }
    }
    $Test->nlog("Test run for $hr HRS $min MINS $sec SECS completed.");

    #------------------------------------------------------------
## Step 9 - Wait for all disks to report results and stop testing
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 9 of 16 :$FILER_C Wait for all disks to report results and stop testing.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("DEBUG: drive_array = @drive_array");
        my $disk_list   = \@drive_array;
        my $total_disks = $disk_length;
        my $timeout     = ( $total_disks * 180 ) + $TIME_OUT_5;
        my $result      = disk_qual_abort(
            Node      => $Node,
            API_Obj   => $API,
            disk_list => $disk_list,
            timeout   => $timeout,
        );

        if ( $result == 1 ) {
            $Test->nlog( "$FILER_C :  Disk qual process stopped " . scalar( localtime() ) );
        } else {
            $Test->nlog("$FILER_C - Disk qual failed to stop");
        }
        disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
    }

    #------------------------------------------------------------
## Step 10 - Compare the log sense data from each drive.
## Pass/Fail criteria:  Note any increase in errors since step 6
    #------------------------------------------------------------
    my %disk_qual_s10 = {};
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 10 of 16 :$FILER_C Record the log sense data from each drive and compare with STEP 6.",
        );
        logcomment("capturing drives for particular filers");
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my @disk_list_q = ();
        my $max_time_out = get_max_timeout( node_present => \@Nodes, Test => $Test );
        $API->set_timeout( "connectrec-timeout" => $max_time_out );

        while (@drive_array) {
            my $temp_disk;
            my $loop_no = 0;
            while ( $loop_no < $disk_max_sup ) {
                $temp_disk = pop(@drive_array);
                push( @disk_list_q, $temp_disk );
                $loop_no++;
                last if ( !(@drive_array) );
            }

            my $disk_attached = \@disk_list_q;
            grep { $disk_qual_cnt{$_}++ } @disk_list_q;
            $disk_qual_s10{$Node} = run_disk_qual_t12( API_Obj => $API, disk_list_local => $disk_attached );
            sleep(200);
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
        }
        $Test->nlog("$FILER_C - Record the log sense data completed");
        disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
    }
    foreach my $Node (@Nodes) {
        my $sas_admin_step6  = $disk_qual_s6{$Node};
        my $sas_admin_step10 = $disk_qual_s10{$Node};
        my $comp_ls          = compare_disk_qual_t12(
            disk_qual_t12_before => $sas_admin_step6,
            disk_qual_t12_after  => $sas_admin_step10,
            nm_error             => 'Y'
        );
        if ( $comp_ls == 1 ) {
            logcomment(" The output of  log sense data is identical with step 6");    #changed by abalaji
        } elsif ( $comp_ls == 0 ) {                                                   #BURT1025104 - abalaji
            logcomment(" The output of  log sense data is different with step 6");    #changed by abalaji
        }
    }

    #------------------------------------------------------------
## Step 11 - Start MIXED TEST using 67% Random Reads
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 11 of 16 :$FILER_C Start MIXED TEST using 67% Random Reads.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my $max_time_out = get_max_timeout( node_present => \@Nodes, Test => $Test );
        $API->set_timeout( "connectrec-timeout" => $max_time_out );

        while (@drive_array) {
            my $temp_disk;
            my $loop_no     = 0;
            my @disk_list_q = ();
            while ( $loop_no < $disk_max_sup ) {
                $temp_disk = pop(@drive_array);
                push( @disk_list_q, $temp_disk );
                $loop_no++;
                last if ( !(@drive_array) );
            }
            grep { $disk_qual_cnt{$_}++ } @disk_list_q;
            $API->execute_raw_command( command => "disk_qual start -t 16 -q 6 -f 50000 -L 'forever' -n 512 -d @disk_list_q" );
            sleep(200);
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
        }
        $Test->nlog("$FILER_C - MIXED TEST using 67% Random Reads started");
    }
    my $step      = 1;
    my $time_left = 0;
    for ( $step = 1 ; $step <= 8 ; $step++ ) {
        sleep $test_wait_time;
        $time_left = ( $test_time_step_hr - ( ( $step * $test_wait_time ) / 3600 ) );
        $time_left = sprintf( "%.2f", $time_left );
        my ( $s, $m, $h ) = gmtime( $time_left * 3600 );
        $Test->nlog( "Start MIXED TEST using 67% Random Reads testtime left $time_left Hr (i.e) $h HRS $m MINS $s SECS \nTtotal time $hr HRS $min MINS $sec SECS : " . scalar( localtime() ) );
        $expected_comp_time = ( $expected_comp_time - ( $test_wait_time / 3600 ) );

        #$Test->nlog("Time left for the test to complete $expected_comp_time Hrs ". scalar( localtime() ) );

        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            my $API_obj = $FILER_API{$FILER_C};
            $API_obj->execute_raw_command( command => "\013" );
        }

    } ## end for ($step = 1 ; $step ...
    $Test->nlog("Test run for $hr HRS $min MINS $sec SECS completed.");

    #------------------------------------------------------------
## Step 12 - Wait for all disks to report results and stop testing
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 12 of 16 :$FILER_C Wait for all disks to report results and stop testing.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUGi*** : drive_array=@drive_array ");
        my $disk_list   = \@drive_array;
        my $total_disks = $disk_length;
        my $timeout     = ( $total_disks * 180 ) + $TIME_OUT_5;
        my $result      = disk_qual_abort(
            Node      => $Node,
            API_Obj   => $API,
            disk_list => $disk_list,
            timeout   => $timeout,
        );

        if ( $result == 1 ) {
            $Test->nlog( "$FILER_C :  Disk qual process stopped " . scalar( localtime() ) );
        } else {
            $Test->nlog("$FILER_C - Disk qual failed to stop");
        }
        disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
    }

    #------------------------------------------------------------
## Step 13 - Start Random Read Noverify Test
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 13 of 16 :$FILER_C Start Random Read Noverify Test.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my $max_time_out = get_max_timeout( node_present => \@Nodes, Test => $Test );
        $API->set_timeout( "connectrec-timeout" => $max_time_out );

        while (@drive_array) {
            my $temp_disk;
            my $loop_no     = 0;
            my @disk_list_q = ();
            while ( $loop_no < $disk_max_sup ) {
                $temp_disk = pop(@drive_array);
                push( @disk_list_q, $temp_disk );
                $loop_no++;
                last if ( !(@drive_array) );
            }
            grep { $disk_qual_cnt{$_}++ } @disk_list_q;
            $API->execute_raw_command( command => "disk_qual start -t 3 -q 3 -L 'forever' -n 512 -f 50000 -d @disk_list_q -o noverify" );
            sleep(200);
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
        }
        $Test->nlog("$FILER_C - Random Read Noverify Test started");
    }
    my $step      = 1;
    my $time_left = 0;
    for ( $step = 1 ; $step <= 8 ; $step++ ) {
        sleep $test_wait_time;
        $time_left = ( ( $test_time_step - ( $step * $test_wait_time ) ) / 3600 );
        $time_left = sprintf( "%.2f", $time_left );
        my ( $s, $m, $h ) = gmtime( $time_left * 3600 );
        $Test->nlog( "Start Random Read Noverify Test time left $time_left Hr (i.e) $h HRS $m MINS $s SECS \nTotal time $hr HRS $min MINS $sec SECS : " . scalar( localtime() ) );
        $expected_comp_time = ( $expected_comp_time - ( $test_wait_time / 3600 ) );

        #$Test->nlog("Time left for the test to complete $expected_comp_time Hrs ". scalar( localtime() ) );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            my $API_obj = $FILER_API{$FILER_C};
            $API_obj->execute_raw_command( command => "\013" );
        }
    } ## end for ($step = 1 ; $step ...

    $Test->nlog("Test run for $hr HRS $min MINS $sec SECS completed.");

    #------------------------------------------------------------
## Step 14 - Wait for all disks to report results and stop testing
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 14 of 16 :$FILER_C Wait for all disks to report results and stop testing.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my $disk_list   = \@drive_array;
        my $total_disks = $disk_length;
        my $timeout     = ( $total_disks * 180 ) + $TIME_OUT_5;
        my $result      = disk_qual_abort(
            Node      => $Node,
            API_Obj   => $API,
            disk_list => $disk_list,
            timeout   => $timeout,
        );

        if ( $result == 1 ) {
            $Test->nlog( "$FILER_C :  Disk qual process stopped " . scalar( localtime() ) );
        } else {
            $Test->nlog("$FILER_C - Disk qual failed to stop");
        }
        disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
    }

    #------------------------------------------------------------
## Step 15 - Start Sequential Verify Test
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 15 of 16 :$FILER_C Start Sequential Verify Test.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my $max_time_out = get_max_timeout( node_present => \@Nodes, Test => $Test );
        $API->set_timeout( "connectrec-timeout" => $max_time_out );

        while (@drive_array) {
            my $temp_disk;
            my $loop_no     = 0;
            my @disk_list_q = ();
            while ( $loop_no < $disk_max_sup ) {
                $temp_disk = pop(@drive_array);
                push( @disk_list_q, $temp_disk );
                $loop_no++;
                last if ( !(@drive_array) );
            }
            grep { $disk_qual_cnt{$_}++ } @disk_list_q;
            $API->execute_raw_command( command => "disk_qual start -t 18 -q 3 -f 50000 -L 'forever' -n 1024 -d @disk_list_q" );
            sleep(200);
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
        }
        $Test->nlog("$FILER_C - Sequential Verify Test started");
    }
    my $step      = 1;
    my $time_left = 0;
    for ( $step = 1 ; $step <= 8 ; $step++ ) {
        sleep $test_wait_time;
        $time_left = ( ( $test_time_step - ( $step * $test_wait_time ) ) / 3600 );
        $time_left = sprintf( "%.2f", $time_left );
        my ( $s, $m, $h ) = gmtime( $time_left * 3600 );
        $Test->nlog( "Start Sequential Verify Test time left $time_left Hr (i.e) $h HRS $m MINS $s SEC\nTotal time $hr HRS $min MINS $sec SECS : " . scalar( localtime() ) );
        $expected_comp_time = ( $expected_comp_time - ( $test_wait_time / 3600 ) );

        #$Test->nlog("Time left for the test to complete $expected_comp_time Hrs ". scalar( localtime() ) );

        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            my $API_obj = $FILER_API{$FILER_C};
            $API_obj->execute_raw_command( command => "\013" );
        }
    } ## end for ($step = 1 ; $step ...

    $Test->nlog("Test run for $hr HRS $min MINS $sec SECS completed.");

    #------------------------------------------------------------
## Step 16 - Wait for all disks to report results and stop testing
## Pass/Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        eye_catcher(
            Test   => $Test,
            string => "STEP 16 of 16 :$FILER_C Wait for all disks to report results and stop testing.",
        );
        my @array = @disk_arr_driveA;
        my @drive_array;
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @drive_array, $_ ) }
        } @array;
        logcomment("***DEBUG*** : drive_array=@drive_array ");
        my $disk_list   = \@drive_array;
        my $total_disks = $disk_length;
        my $timeout     = ( $total_disks * 180 ) + $TIME_OUT_5;
        my $result      = disk_qual_abort(
            Node      => $Node,
            API_Obj   => $API,
            disk_list => $disk_list,
            timeout   => $timeout,
        );

        if ( $result == 1 ) {
            $Test->nlog( "$FILER_C :  Disk qual process stopped " . scalar( localtime() ) );
        } else {
            $Test->nlog("$FILER_C - Disk qual failed to stop");
			$test_status = 1;
        }
        disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
    }
    my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
    my $disk_qual_path = $actual_log_dir . "/disk_qual_cnt_$TC_name.log";
    logcomment("**DEBUG** : log path : $disk_qual_path");
    open( FH, ">>$disk_qual_path" ) || die "Failed to open the write file disk_qual.txt";
    foreach my $dsk ( keys %disk_qual_cnt ) {
        logcomment("$dsk\t\t$disk_qual_cnt{$dsk}");
        print FH "$dsk\t\t$disk_qual_cnt{$dsk}\n";
    }
    close FH;
###########################################################################################
## Test log/results
###########################################################################################
    logcomment("Test $TC_name is completed.");
    logresult( "INFO", msg => $test_status );

}
#############################################################
#Cleanup
####################################################################
sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    $Test->nlog("Nodes present - @Nodes");
    my $node_ref = \@Nodes;
    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ###########################################################################################
	nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );

    return $TCD::PASS;
}
#########################################
## Subroutines
#########################################
sub _disk_qual_run_script ($$$) {

    my ( $API, $disk_arr, $Node ) = @_;
    my ( $timeout, $after );
    my $flag    = 0;
    my $Filer_C = $Node->name();
    $Test->nlog( "FILER- $Filer_C : Disk qual test script started on all drives - @$disk_arr  " . scalar( localtime() ) );
    $timeout = get_max_timeout( node_present => [$Node] );
    $timeout = $timeout + ( scalar(@$disk_arr) * 300 ) + 300;
    $API->set_timeout( "connectrec-timeout" => $timeout );
    logcomment("DEBUG: timeout - $timeout");
    my $time_out = 0;
    my @disk_list_q;
    my $disk_max_sup = 40;

    logcomment("Executes the default disk_qual test script on drives : @$disk_arr");
    my @drive_array = @$disk_arr;
    logcomment("***DEBUG*** : drive_array=@drive_array");
    while (@drive_array) {
        my $temp_disk;
        my $loop_no = 0;
        while ( $loop_no < $disk_max_sup ) {
            $temp_disk = pop(@drive_array);
            push( @disk_list_q, $temp_disk );
            $loop_no++;
            last if ( !(@drive_array) );
        }
        my $match = $API->execute_raw_command( command => "disk_qual runscript -d @disk_list_q" );
        while ( 1 && ( $time_out < $timeout ) ) {
            my $run_output = $API->execute_raw_command( command => "" );
            foreach my $line ( split( /\r|\n/, $run_output ) ) {
                if ( $line =~ /sysread failed:|Ending script execution|PANIC|state: Failed/ ) {
                    $flag = 1;
                }
            }
            if ( $flag == 1 ) {
                $match = $run_output;
                last;
            } else {
                sleep(200);
                $time_out = $time_out + 200;
            }
        }
        sleep(300);
        $API->execute_raw_command( command => "\013" );
        $API->execute_raw_command( command => "\013" );

        my $after = $API->execute_raw_command( command => "disk_qual status" );
        if ( $after =~ /running/i ) {

            logcomment("$Filer_C - disk_qual is still running for all the drives");
            logcomment("$Filer_C - Debug statement: Match returned: $match");
            $after    = '';
            $time_out = 0;
            $flag     = 0;
            while ( 1 && ( $time_out < $timeout ) ) {
                my $run_output = $API->execute_raw_command( command => "" );
                foreach my $line ( split( /\r|\n/, $run_output ) ) {
                    if ( $line =~ /sysread failed:|Ending script execution|PANIC|state: Failed/ ) {
                        $flag = 1;
                    }
                }
                if ( $flag == 1 ) {
                    $after = $run_output;
                } else {
                    sleep(100);
                    $time_out = $time_out + 100;
                }
                last if ( $flag == 1 );
            }
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
            $match = $after if ( $match =~ /^$/ );
        }
        if ( $match =~ /disk_qual not found/ ) {
            $test_status = 1;
            $Test->nlog( "FILER- $Filer_C : disk_qual command not found, exiting from the test " . scalar( localtime() ) );
            $email_subject = "$TC_name - $Filer_C: Test FAILED: FATAL";
            $email_body    = "$Filer_C - disk_qual command not found, exiting from the test. \nLog location: $LOGDIR";
            send_message(
                mail_body    => $email_body,
                mail_subject => $email_subject,
                MAIL_FROM    => $MAIL_FROM,
                MAIL_TO      => $MAIL_TO,
                EOE          => $EOE,
                EMAIL        => $EMAIL
            );
        }

        if ( $match =~ /Ending/ ) {
            logcomment("$Filer_C - Default script ran Successfully for all drives");

        }
        if ( $match =~ /PANIC/ ) {
            sleep 300;
            $test_status = 1;
            $Test->nlog( "$Filer_C : Filer panic while waiting for disk_qual end, exiting from the test " . scalar( localtime() ) );
            $email_subject = "$TC_name - $Filer_C: Test FAILED: FATAL";
            $email_body    = "$Filer_C - Filer panic while waiting for disk_qual end, exiting from the test. \nLog location: $LOGDIR";

            send_message(
                mail_body    => $email_body,
                mail_subject => $email_subject,
                MAIL_FROM    => $MAIL_FROM,
                MAIL_TO      => $MAIL_TO,
                EOE          => $EOE,
                EMAIL        => $EMAIL
            );
        }
        $match    = '';
        $time_out = 0;
        $flag     = 0;
    }

    if ( $test_status == 1 ) {
        $Test->nlog( "$Filer_C : Following drives not able to run the default script:\n  " . scalar( localtime() ) );
    }
    $Test->nlog( "$Filer_C : Following drives successfully executed the default script:\n @$disk_arr " . scalar( localtime() ) );

} ## end sub _disk_qual_run_script ($$$)