#!/usr/software/bmn/perl
## All rights reserved.
##
## @summary Mode
##  Establish a baseline for drive IOPS (no vibration) by performing IOs per the parameters
##
## @description
##  	Verifying baseline for drive IOPS (no vibration) by performing IOs per the parameters
##
## @Test Mode
##  Maintenance mode Non Debug Kernel
##
## @Test bed setup
##	A full shelf of single model, single capacity drives OnTap non debug kernel with disk_qual support
##  FC   :Dual Path, Cluster Fabric
##  SAS  :Dual Path
##
## @usage
##  The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  There is no dependency
##
## @steps
##  The test will execute steps mentioned below:
##  PRE_CHECK : A full shelf of single model(match with user input model), single capacity drives OnTap non debug kernel with disk_qual support
##  1. Execute 4K Byte sequential Writes, q=8 for 3 minutes.
##  2. Execute 64K Byte random Writes, q=8 for 3 minutes.
##  3. Execute 4K Byte random Writes, q=8 for 3 minutes.
##  4. Execute 4K Byte random Reads, q=8 for 3 minutes.
##
##
## @param FILER        - optional for cluster setup
##                     - required for Dual path setup      Name of filer to be used
## @param FILERA       - required for cluster setup
##                     - optional for Dual path setup      Name of filer to be used
## @param FILERB       - required for cluster setup
##                     - optional for Dual path setup      Name of the partner node
## @param TEST_CONFIG  - optional Type of the setup, C for cluster, D for dual path (default)
## @param TEST_SETUP   - optional Default value set to 'FC'
## @param FILER_CONN   - optional Type of connection, default is set to 'console' for this test
## @param MODE         - optional Filer mode maint/normal, default set to 'maint' for this test
## @param FILER_PROMPT - optional prompt of the filer
## @param LOGDIR       - optional This is required to generate 'END-LOG'. Default set to main log directory.
## @param EOE          - optional 'default' is set to default value
## @param BOOT_MODE    - optional Boot mode for filer , default set to '1' for this test
## @param BOOT_TYPE    - optional Boot type, default set to 'A' (automatic) for this test
## @param SSD          - optional Setup is SSD or not, default set to 'no'
## @param EMAIL        -  optional Default value set to 'Y'
## @param EMAIL_TO     - required for setting to address sending email to recipient
## @param EMAIL_FROM   - required for setting  from address for sending email
## @param FILER_TYPE   - Required to set DataONTAP version    DataONTAP version is either IC or BR
## @param FIRMWARE     -  required to load the firmware files
## @param MINI_CREATE_SIZE - required indicating the choice of the user to create mini disks
## @param MINI_CREATE - required indicating the choice of the user for the size of the mini disks to be created
##
## @status  Automated
## @author  kritiu@netapp.com
## @burt    1089481
## @change  YY/MM/DD from auther: Description of the change
##
#############################################################################
#############################################################################
########################################
### Library functions
##########################################
# Compiler directives.
use strict;

# Module imports
use TCD;
use NACL::APISet;
use NACL::C::Node;
use NACL::APISet::Host;
use NACL::C::Disk;
use NACL::Transit;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;

sub _array_diff(\@\@);

#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $ARMADILLO
  $RESETVALUE
  $MODE
  $DRIVE_ID
  $Primary_Node
  $TEST_DRIVES
  $DRIVE_MODEL

);

my $params = NATE::ParamSet->new( global => 1 );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'D' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 'default' );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$FILER_PROMPT   = $params->get( 'FILER_PROMPT',   default => '\*>' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '5' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'C' );
$MODE           = $params->get( 'MODE',           default => 'MAINT' );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$RESETVALUE     = $params->get( 'RESETVALUE',     default => '90' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'Sets the filer Ironcity or Boliermaker' );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_DRIVES    = $params->get( "TEST_DRIVES",    default => 'single' );
$FILER_CONN     = $params->get( "FILER_CONN",     default => '#console' );
$DRIVE_ID       = $params->get( "DRIVE_ID",       default => '' );
$Primary_Node   = $params->get( 'Primary_Node',   default => 'N' );
$DRIVE_MODEL    = $params->get( 'DRIVE_MODEL',    default => '' );

################
# Testcase name
################
my $TC_name;
$TC_name = "210_NADQ02_NVMe_DVT_Baseline";

########################################
### Initialization/declaration
#########################################
my $test_status = 0;
my %nodes_filer = {};
my @FILER_ARR;
my @Nodes;
my $filer_names;
my $Mode;
my $flag_setup = 0;
my @API_ARRAY;
my $FILER_STATE;
my $API_Object;
my %FILER_API = ();

my ( $email_subject, $email_body );
$EOE = 1;

#########################################
# Testcase available for execution
#########################################
## Pre-test processes
#########################################

my @Testcases = ( DVT_Baseline => " DVT_Baseline - Maintenance mode" );

&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );    # Testcases will be executed using TCD object.
    if ( $Test->error ) {
        logcomment( $Test->errmsg );
        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        logcomment( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main

########## INIT  ###################################################
# init automatically called before executing tests
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node objects used for test execution
    grep { logcomment( "Filers passed to the ntest are " . $_->name() ) } @Nodes;
    $Mode = $Nodes[0]->mode();

    logcomment("Mode of the Filers $Mode");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );

        $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("FILER- $FILER_C - Is in $FILER_STATE state");
        if ( $FILER_STATE !~ /MAINT/ ) {
            $Transit_obj->change_state( to => "MAINT" );
        }
        $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        if ( $FILER_STATE =~ /MAINT/ ) {
            my $host = host($FILER_C);
            $API_Object = NACL::APISet->new(
                hostobj   => $host,
                category  => "Node",
                interface => "CLI",
                set       => "Maintenance"
            );
            push( @API_ARRAY, $API_Object );
            $FILER_API{$FILER_C} = $API_Object;
        }
    }
    version_test( node_present => \@Nodes, tc_name => $TC_name, );
    logcomment("Checking for execution mode");

    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}

########## SETUP  ###################################################
# Setup automatically called before executing tests
#####################################################################
sub setup() {
    $Test->description("Setup the environment for the test exectuion ");
    logcomment("Filer present : @Nodes");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );
    return $TCD::PASS;
    my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
    my $disk_qual_path = $actual_log_dir . "/disk_qual_cnt_$TC_name.log";
    logcomment("**DEBUG** : log path : $disk_qual_path");
    open( FH, ">>$disk_qual_path" ) || die "Failed to open the write file $!";
    close FH;
}

########## TEST 1 ###################################################
#NADQ02_DVT_Baseline
#####################################################################
sub DVT_Baseline {
    my @subtests;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filers are $filer_names" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&DVT_Baseline_sub, -runid, "DVT_Baseline_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $return = status_return( subtest_ref => [@subtests] );
    if ( $return == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }

}
#######################
##Subtest_create
#######################
sub DVT_Baseline_sub {

    #############
    ### Test Variables
    #############
    my @Nodes;
    my %disk_qual_cnt;
    push( @Nodes, shift(@_) );
    $Test->description(" DVT_Baseline  Maintenance mode ");
###########################################################################################
## TEST STARTS
###########################################################################################
    my $test_wait_time = 1800;
    logcomment("Default wait time is 30 minutes");
    if ( $TEST_WAIT_TIME == 1 ) {
        $test_wait_time = ( $test_wait_time / 10 );
        my ( $sec, $min, $hour ) = gmtime($test_wait_time);
        logcomment("Total wait time is reduced to 10% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 1");
    } elsif ( $TEST_WAIT_TIME == 2 ) {
        $test_wait_time = ( $test_wait_time / 2 );
        my ( $sec, $min, $hour ) = gmtime($test_wait_time);
        logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 2");
    } elsif ( $TEST_WAIT_TIME == 4 ) {
        $test_wait_time = ( $test_wait_time * .95 );
        my ( $sec, $min, $hour ) = gmtime($test_wait_time);
        logcomment("Total wait time is reduced to 95% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 4");
    }
    my ( $sec, $min ) = gmtime($test_wait_time);
    my $test_time_step     = ( 24 * $test_wait_time );
    my $test_time_step_hr  = ( $test_time_step / 3600 );
    my $expected_comp_time = ( ( 24 * ($test_wait_time) / 3600 ) + 2 );

    my $FILER_C          = $Nodes[0]->name();
    my $API              = $FILER_API{$FILER_C};
    my $disk_show_output = disk_show_drive_maint(
        API_Obj => $API,
        Filer   => $FILER_C
    );
    my $driveA          = $disk_show_output->{driveA};
    my $driveB          = $disk_show_output->{driveB};
    my @disk_arr_driveA = @$driveA;
    my @disk_arr_driveB = @$driveB;
    logcomment("***DEBUG***: disk_arr_driveA = @disk_arr_driveA \n\n disk_arr_driveB = @disk_arr_driveB");

    #------------------------------------------------------------
    ## Step 1 - Execute 4K Byte sequential Writes
    ## Pass/Fail criteria: Report the failure and continue to the next run (if 5 passes are not completed) or next step.
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        $API->set_timeout( "connectrec-timeout" => 7200 );
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 1 of 4:$FILER_C Execute 4K Byte sequential Writes",
        );
        my %SEQUENTIAL = {};
        my $output;
        my @IOPS_drive;
        my @disk_array = @disk_arr_driveA;
        my @disk_array_1;
        grep {
            if ( $_ !~ /P\d+/i ) { push( @disk_array_1, $_ ) }
        } @disk_array;
        my $i;
        my %average = {};
        my %flag;

        for ( $i = 0 ; $i < 5 ; $i++ ) {
            logcomment("#################### RUN $i ############################");
            disk_qual_abort( Node => $Node, API_Obj => $API, disk_list => \@disk_arr_driveA );
            logcomment("$FILER_C : Disk_qual is stopped");
            grep { $disk_qual_cnt{$_}++ } @disk_array_1;
            $output = $API->execute_raw_command( command => "disk_qual start -t 5 -q 8 -n 8 -d @disk_array_1 -o nopattern noverify" );
            sleep(30);
            if ( $output =~ /Starting SEQUENTIAL WRITE test/i ) {
                logcomment("SEQUENTIAL WRITE Test on all drives is started Successfully");
            } else {
                $test_status = 1;
                logcomment("**FAIL** : $FILER_C : SEQUENTIAL WRITE TEST ON ALL DRIVES IS NOT STARTED PROPERLY");
                return logresult( "INFO", msg => 1 );
            }
            sleep(180);
            $API->set_timeout( "connectrec-timeout" => 7200 );
            $Node->refresh_command_interface();
            $API->execute_raw_command( command => "disk_qual stop -d @disk_array_1" );
            sleep(60);
            $Node->refresh_command_interface();
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
            my $kill_time     = $test_wait_time;
            my $time_out_temp = 0;

            while ( $kill_time > $time_out_temp ) {
                my ( $s, $m ) = gmtime($time_out_temp);
                logcomment("DEBUG:Total Wait time:$kill_time (i.e) $min MINS $sec SECS Elapsed Time: $time_out_temp (i.e) $m MINS $s SECS");
                my $status = $API->execute_raw_command( 'command' => "disk_qual status" );
                if ( $status =~ /idle/ ) {
                    logcomment("$FILER_C : Disk_qual is completed");
                    last;
                } elsif ( $status =~ /running/ ) {
                    sleep 120;
                    $time_out_temp += 120;
                }
            }
            foreach my $disk_report (@disk_array_1) {
                my $result = $API->execute_raw_command( command => "disk_qual report -d $disk_report" );
                foreach my $line ( split( "\n", $result ) ) {
                    if ( $line =~ /\[(\S+)\]\: IOPS\:\s+(\d+)/ig ) {
                        my $dsk   = $1;
                        my $value = $2;
                        logcomment("DEBUG: IOPS value for $dsk is $value");
                        push( @IOPS_drive, $dsk );
                        $SEQUENTIAL{$i}->{$dsk} = $value;
                        $average{$dsk} = $average{$dsk} + $value;
                    }
                }
            }
            sleep(10);
            disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
            logcomment("$FILER_C : $i SEQUENTIAL WRITE Test on all drives is Compleated");
            next if ( $i == 0 );
            my $previous = $i - 1;
            foreach my $dsk (@disk_array_1) {
                my $perc = ( ( ( $SEQUENTIAL{$previous}->{$dsk} - $SEQUENTIAL{$i}->{$dsk} ) * 100 ) / $SEQUENTIAL{$previous}->{$dsk} );
                my $perc = abs($perc);
                $perc = sprintf "%0.2f", $perc;
                logcomment("$dsk : IOPS Percentage = $perc");
                if ( $perc > 3 ) {
                    $test_status = 1;
                    logcomment("**FAIL** : $FILER_C : Disk :$dsk IOPS OF EACH RUN SHOULD BE WITHIN 3% OF EACH OF THE PREVIOUS RUN IN STEP 1($i) . PERCENTAGE : $perc \tPreivious IOPS VALUE : $SEQUENTIAL{$previous}->{$dsk}\tPresent IOPS VLAUE: $SEQUENTIAL{$i}->{$dsk}");
                    $flag{$dsk} = 1;

                }
            }
        }
        logcomment("Caluculating the IOPS Average of 5 Runs");
        foreach my $dsk (@disk_array_1) {
            if ( $flag{$dsk} != 1 ) {
                my $avg = $average{$dsk} / 5;
                logcomment("$FILER_C : Disk : $dsk\t Average of IOPS values for 5 Runs in STEP 1 is $avg");
            } else {
                logcomment("$FILER_C : Disk : $dsk\t Average of IOPS values for 5 Runs is not Caluculating in STEP 1, Since Percentage is not with in 3%");
            }
        }
        logcomment("$FILER_C : SEQUENTIAL WRITE is compleated on all drives for 5 RUNS ");
    }

    #------------------------------------------------------------
    ## Step 2 - Execute 64K Byte random Writes
    ## Pass/Fail criteria: The IOPs of each run is within 3% of each of the previous run
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        $API->set_timeout( "connectrec-timeout" => 7200 );
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 2 of 4:$FILER_C Executing 64K Byte random Writes",
        );
        my %SEQUENTIAL = {};
        my $output;
        my @IOPS_drive;
        my @disk_array = @disk_arr_driveA;
        my @disk_array_1;
        grep {
            if ( $_ !~ /P\d+/i ) { push( @disk_array_1, $_ ) }
        } @disk_array;
        my $i;
        my %average = {};
        my %flag;

        for ( $i = 0 ; $i < 5 ; $i++ ) {
            logcomment("#################### RUN $i ############################");
            disk_qual_abort( Node => $Node, API_Obj => $API, disk_list => \@disk_arr_driveA );
            logcomment("$FILER_C : Disk_qual is stopped");
            grep { $disk_qual_cnt{$_}++ } @disk_array_1;
            $output = $API->execute_raw_command( command => "disk_qual start -t 7 -q 8 -n 128 -d @disk_array_1 -o nopattern noverify" );
            sleep(30);
            if ( $output =~ /Starting RANDOM WRITE test/i ) {
                logcomment("RANDOM WRITE Test on all drives is started Successfully");
            } else {
                $test_status = 1;
                logcomment("**FAIL** : $FILER_C : RANDOM  WRITE TEST ON ALL DRIVES IS NOT STARTED PROPERLY");
                return logresult( "INFO", msg => 1 );
            }
            sleep(180);    #waiting for 3 min
            $API->set_timeout( "connectrec-timeout" => 7200 );
            $Node->refresh_command_interface();
            $API->execute_raw_command( command => "disk_qual stop -d @disk_array_1" );
            sleep(120);
            $Node->refresh_command_interface();
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
            my $kill_time     = $test_wait_time;
            my $time_out_temp = 0;

            while ( $kill_time > $time_out_temp ) {
                my ( $s, $m ) = gmtime($time_out_temp);
                logcomment("DEBUG:Total Wait time:$kill_time (i.e) $min MINS $sec SECS Elapsed Time: $time_out_temp (i.e) $m MINS $s SECS");
                my $status = $API->execute_raw_command( 'command' => "disk_qual status" );
                if ( $status =~ /idle/ ) {
                    logcomment("$FILER_C : Disk_qual is completed");
                    last;
                } elsif ( $status =~ /running/ ) {
                    sleep 120;
                    $time_out_temp += 120;
                }
            }
            foreach my $disk_report (@disk_array_1) {
                my $result = $API->execute_raw_command( command => "disk_qual report -d $disk_report" );
                foreach my $line ( split( "\n", $result ) ) {
                    if ( $line =~ /\[(\S+)\]\: IOPS\:\s+(\d+)/ig ) {
                        my $dsk   = $1;
                        my $value = $2;
                        logcomment("***DEBUG***: IOPS value for $dsk is $value");
                        push( @IOPS_drive, $dsk );
                        $SEQUENTIAL{$i}->{$dsk} = $value;
                        $average{$dsk} = $average{$dsk} + $value;
                    }
                }
            }

            sleep(10);
            disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
            logcomment("$FILER_C : $i RANDOM WRITE Test on all drives is Compleated");
            next if ( $i == 0 );
            my $previous = $i - 1;
            foreach my $dsk (@disk_array_1) {
                my $perc = ( ( ( $SEQUENTIAL{$previous}->{$dsk} - $SEQUENTIAL{$i}->{$dsk} ) * 100 ) / $SEQUENTIAL{$previous}->{$dsk} );
                $perc = abs($perc);
                $perc = sprintf "%0.2f", $perc;
                logcomment("$dsk : IOPS Percentage = $perc");
                if ( $perc > 3 ) {
                    $test_status = 1;
                    logcomment("**FAIL** : $FILER_C : DISK : $dsk IOPS OF EACH RUN SHOULD BE WITHIN 3% OF EACH OF THE PREVIOUS RUN IN STEP 2($i) . PERCENTAGE : $perc\tPREVIOUS IOPS VALUE : $SEQUENTIAL{$previous}->{$dsk}\tPRESENT IOPS VALUE: $SEQUENTIAL{$i}->{$dsk}");
                    $flag{$dsk} = 1;

                }

            }
        }
        logcomment("Caluculating the IOPS Average of 5 Runs");
        foreach my $dsk (@disk_array_1) {
            if ( $flag{$dsk} != 1 ) {
                my $avg = $average{$dsk} / 5;
                logcomment("$FILER_C : Disk : $dsk\t Average of IOPS values for 5 Runs in STEP 2 is $avg");
            } else {
                logcomment("$FILER_C : Disk : $dsk\t Average of IOPS values for 5 Runs is not Caluculating in STEP 2, Since Percentage is not with in 3%");
            }
        }
        logcomment("$FILER_C : RANDOM WRITE Test is compleated on all drives for 5 RUNS");
    }

    #------------------------------------------------------------
    ## Step 3 - Execute 4K Byte random Writes
    ## Pass/Fail criteria: The IOPs of each run is within 3% of each of the previous run
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        $API->set_timeout( "connectrec-timeout" => 7200 );
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 3 of 4:$FILER_C Executing 4K Byte Random Writes",
        );
        my %SEQUENTIAL = {};
        my $output;
        my @IOPS_drive;
        my @disk_array = @disk_arr_driveA;
        my @disk_array_1;
        grep {
            if ( $_ !~ /P\d+/i ) { push( @disk_array_1, $_ ) }
        } @disk_array;
        my $i;
        my %average = {};
        my %flag;

        for ( $i = 0 ; $i < 5 ; $i++ ) {
            logcomment("#################### RUN $i ############################");
            disk_qual_abort( Node => $Node, API_Obj => $API, disk_list => \@disk_arr_driveA );
            logcomment("$FILER_C : Disk_qual is stopped");
            grep { $disk_qual_cnt{$_}++ } @disk_array_1;
            $output = $API->execute_raw_command( command => "disk_qual start -t 7 -q 8 -n 8 -d @disk_array_1 -o nopattern noverify" );
            sleep(30);
            if ( $output =~ /Starting RANDOM WRITE test/i ) {
                logcomment("RANDOM WRITE Test on all drives is started Successfully");
            } else {
                $test_status = 1;
                logcomment("**FAIL** : $FILER_C : RANDOM WRITE TEST ON ALL DRIVES IS NOT STARTED PROPERLY");
                return logresult( "INFO", msg => 1 );
            }
            sleep(180);    #waiting for 3 min
            $API->set_timeout( "connectrec-timeout" => 7200 );
            $Node->refresh_command_interface();
            $API->execute_raw_command( command => "disk_qual stop -d @disk_array_1" );
            sleep(120);
            $Node->refresh_command_interface();
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
            my $kill_time     = $test_wait_time;
            my $time_out_temp = 0;

            while ( $kill_time > $time_out_temp ) {
                my ( $s, $m ) = gmtime($time_out_temp);
                logcomment("DEBUG:Total Wait time:$kill_time (i.e) $min MINS $sec SECS Elapsed Time: $time_out_temp (i.e) $m MINS $s SECS");
                my $status = $API->execute_raw_command( 'command' => "disk_qual status" );
                if ( $status =~ /idle/ ) {
                    logcomment("$FILER_C : Disk_qual is completed");
                    last;
                } elsif ( $status =~ /running/ ) {
                    sleep 120;
                    $time_out_temp += 120;
                }
            }
            foreach my $disk_report (@disk_array_1) {
                my $result = $API->execute_raw_command( command => "disk_qual report -d $disk_report" );
                foreach my $line ( split( "\n", $result ) ) {
                    if ( $line =~ /\[(\S+)\]\: IOPS\:\s+(\d+)/ig ) {
                        my $dsk   = $1;
                        my $value = $2;
                        logcomment("***DEBUG***: IOPS value for $dsk is $value");
                        push( @IOPS_drive, $dsk );
                        $SEQUENTIAL{$i}->{$dsk} = $value;
                        $average{$dsk} = $average{$dsk} + $value;
                    }
                }
            }

            sleep(10);
            disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
            logcomment("$FILER_C : $i RANDOM WRITE Test on all drives is Compleated");
            next if ( $i == 0 );
            my $previous = $i - 1;
            foreach my $dsk (@disk_array_1) {
                my $perc = ( ( ( $SEQUENTIAL{$previous}->{$dsk} - $SEQUENTIAL{$i}->{$dsk} ) * 100 ) / $SEQUENTIAL{$previous}->{$dsk} );
                $perc = abs($perc);
                $perc = sprintf "%0.2f", $perc;
                logcomment("$dsk : IOPS Percentage = $perc");
                if ( $perc > 3 ) {
                    $test_status = 1;
                    logcomment("**FAIL** : $FILER_C : DISK : $dsk IOPS OF EACH RUN SHOULD BE WITHIN 3% OF EACH OF THE PREVIOUS RUN IN STEP 3($i) . PERCENTAGE : $perc\tPREVIOUS IOPS VALUE : $SEQUENTIAL{$previous}->{$dsk}\tPRESENT IOPS VALUE: $SEQUENTIAL{$i}->{$dsk}");
                    $flag{$dsk} = 1;

                }

            }
        }
        logcomment("Caluculating the IOPS Average of 5 Runs");
        foreach my $dsk (@disk_array_1) {
            if ( $flag{$dsk} != 1 ) {
                my $avg = $average{$dsk} / 5;
                logcomment("$FILER_C : Disk : $dsk\t Average of IOPS values for 5 Runs in STEP 3 is $avg");
            } else {
                logcomment("$FILER_C : Disk : $dsk\t Average of IOPS values for 5 Runs is not Caluculating in STEP 3, Since Percentage is not with in 3%");
            }
        }
        logcomment("$FILER_C : RANDOM WRITE is compleated on all drives for 5 RUNS");
    }

    #------------------------------------------------------------
    ## Step 4 - Execute 4K Byte RANDOM READ
    ## Pass/Fail criteria: The IOPs of each run is within 3% of each of the previous run
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API     = $FILER_API{$FILER_C};
        $API->set_timeout( "connectrec-timeout" => 7200 );
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 4 of 4:$FILER_C Executing 4K Byte RANDOM READ on all drives",
        );
        my %SEQUENTIAL = {};
        my $output;
        my @IOPS_drive;
        my @disk_array = @disk_arr_driveA;
        my @disk_array_1;
        grep {
            if ( $_ !~ /P\d+/i ) { push( @disk_array_1, $_ ) }
        } @disk_array;
        my $i;
        my %average = {};
        my %flag;

        for ( $i = 0 ; $i < 5 ; $i++ ) {
            logcomment("#################### RUN $i ############################");
            disk_qual_abort( Node => $Node, API_Obj => $API, disk_list => \@disk_arr_driveA );
            logcomment("$FILER_C : Disk_qual is stopped");
            grep { $disk_qual_cnt{$_}++ } @disk_array_1;
            $output = $API->execute_raw_command( command => "disk_qual start -t 3 -q 8 -n 8 -d @disk_array_1 -o nopattern noverify" );
            sleep(30);
            if ( $output =~ /Starting RANDOM READ test/i ) {
                logcomment("RANDOM READ Test on all drives is started Successfully");
            } else {
                $test_status = 1;
                logcomment("**FAIL** : $FILER_C : RANDOM  WRITE TEST ON ALL DRIVES IS NOT STARTED PROPERLY");
                return logresult( "INFO", msg => 1 );
            }
            sleep(180);    #waiting for 3 min
            $API->set_timeout( "connectrec-timeout" => 7200 );
            $Node->refresh_command_interface();
            $API->execute_raw_command( command => "disk_qual stop -d @disk_array_1" );
            sleep(120);
            $Node->refresh_command_interface();
            $API->execute_raw_command( command => "\013" );
            $API->execute_raw_command( command => "\013" );
            my $kill_time     = $test_wait_time;
            my $time_out_temp = 0;

            while ( $kill_time > $time_out_temp ) {
                my ( $s, $m ) = gmtime($time_out_temp);
                logcomment("DEBUG:Total Wait time:$kill_time (i.e) $min MINS $sec SECS Elapsed Time: $time_out_temp (i.e) $m MINS $s SECS");
                my $status = $API->execute_raw_command( 'command' => "disk_qual status" );
                if ( $status =~ /idle/ ) {
                    logcomment("$FILER_C : Disk_qual is completed");
                    last;
                } elsif ( $status =~ /running/ ) {
                    sleep 120;
                    $time_out_temp += 120;
                }
            }
            foreach my $disk_report (@disk_array_1) {
                my $result = $API->execute_raw_command( command => "disk_qual report -d $disk_report" );
                foreach my $line ( split( "\n", $result ) ) {
                    if ( $line =~ /\[(\S+)\]\: IOPS\:\s+(\d+)/ig ) {
                        my $dsk   = $1;
                        my $value = $2;
                        logcomment("***DEBUG***: IOPS value for $dsk is $value");
                        push( @IOPS_drive, $dsk );
                        $SEQUENTIAL{$i}->{$dsk} = $value;
                        $average{$dsk} = $average{$dsk} + $value;
                    }
                }
            }

            sleep(10);
            disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@disk_arr_driveA );
            logcomment("$FILER_C : $i RANDOM READ Test on all drives is Compleated");
            next if ( $i == 0 );
            my $previous = $i - 1;
            foreach my $dsk (@disk_array_1) {
                my $perc = ( ( ( $SEQUENTIAL{$previous}->{$dsk} - $SEQUENTIAL{$i}->{$dsk} ) * 100 ) / $SEQUENTIAL{$previous}->{$dsk} );
                $perc = abs($perc);
                $perc = sprintf "%0.2f", $perc;
                logcomment("$dsk : IOPS Percentage = $perc");
                if ( $perc > 3 ) {
                    $test_status = 1;
                    logcomment("**FAIL** : $FILER_C : DISK : $dsk IOPS OF EACH RUN SHOULD BE WITHIN 3% OF EACH OF THE PREVIOUS RUN IN STEP 4($i) . PERCENTAGE : $perc\tPREVIOUS IOPS VALUE : $SEQUENTIAL{$previous}->{$dsk}\tPRESENT IOPS VALUE: $SEQUENTIAL{$i}->{$dsk}");
                    $flag{$dsk} = 1;

                }

            }
        }
        logcomment("Caluculating the IOPS Average of 5 Runs");
        foreach my $dsk (@disk_array_1) {
            if ( $flag{$dsk} != 1 ) {
                my $avg = $average{$dsk} / 5;
                logcomment("$FILER_C : Disk : $dsk\t Average of IOPS values for 5 Runs in STEP 4 is $avg");
            } else {
                logcomment("$FILER_C : Disk : $dsk\t Average of IOPS values for 5 Runs is not Caluculating in STEP 4, Since Percentage is not with in 3%");
            }
        }
        logcomment("$FILER_C : RANDOM READ is compleated on all drives for 5 RUNS");
    }
    my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
    my $disk_qual_path = $actual_log_dir . "/disk_qual_cnt_$TC_name.log";
    logcomment("**DEBUG** : log path : $disk_qual_path");
    open( FH, ">>$disk_qual_path" ) || die "Failed to open the write file disk_qual.txt";
    foreach my $dsk ( keys %disk_qual_cnt ) {
        logcomment("$dsk\t\t$disk_qual_cnt{$dsk}");
        print FH "$dsk\t\t$disk_qual_cnt{$dsk}\n";
    }
    close FH;
    logresult( "INFO", msg => $test_status );
}

#####################################################################
## Cleanup - Post Process
#####################################################################
sub cleanup() {
    logcomment(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ##########################################################################################
    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode,

    ) if ( $flag_setup != 1 );
    return $TCD::PASS;
}

sub _array_diff(\@\@) {
    my %e = map { $_ => undef } @{ $_[1] };
    return @{ [ ( grep { ( exists $e{$_} ) ? ( delete $e{$_} ) : (1) } @{ $_[0] } ), keys %e ] };
}