#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/TEST_SCRIPTS/CMODE/NADQ02_FC_Disk_Perfm_Norm.thpl#16 $
# Copyright (c) 2005 Network Appliance, Inc.
# All rights reserved
#
## @summary Mode
##  Performance Normalization Test - disktest
##
## @description
##  Basic test to detect any gross drive-to-drive interference performance degradation using the disktest tool.
##
##
## hardware configuration
## Each test must be run on a full shelf of drives, (14 or 24 drives).
##
## @Test Mode
##  Maintenance mode
##
## @Test bed setup
##  FC   :Dual Path, Cluster Fabric
##  SATA :Dual Path
##  SAS  :Dual Path, Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  None
##
## @steps
## The test will execute steps mentioned below:
##   1.  Begin new, scripted console session.
##   2.  Check the drive under test total LBAs. <Max LBA+1> = scsi capacity "blocks" value.
##   3.  Run disk_qual: read 256 sectors (3% of disk @ ID area) all disks. Allow test to complete. <ID start LBA> = (scsi capacity "blocks" value) - (0.03 * scsi capacity "blocks" value). <ID last LBA> = scsi capacity "blocks" value- 1.
##   4.  Compute for all drives with the same Product ID string --> (max_iops - min_iops)/max iops.
##   5.  Run disk_qual: read 512 sectors (10% of disk @ OD area)all disks. Allow test to complete. <OD last LBA> = 0.10 * (scsi capacity "blocks" value).
##   6.  Compute for all drives with the same Product ID string --> (max_iops - min_iops)/max iops.
##   7.  Run disk_qual: write 256 sectors (3% of disk @ ID area) all disks. <ID start LBA> = (scsi capacity "blocks" value) - (0.03 * scsi capacity "blocks" value). <ID last LBA> = scsi capacity "blocks" value- 1
##   8.  Compute for all drives with the same Product ID string --> (max_iops - min_iops)/max iops.
##   9.  Run disk_qual: write 512 sectors (10% of disk @ OD area) all disks. Allow test to complete. <OD last LBA> = 0.10 * (scsi capacity "blocks" value).
##   10. Compute for all drives with the same Product ID string --> (max_iops - min_iops)/max iops.
##   11. Sequential read on ALL drives at the same time (10% of disk @ OD area). This is to check SAS fairness.
##   12. Sequential read on ALL drives at the same time (10% of disk @ OD area). This is to check SAS fairness.
##
## @param FILER       - optional for cluster setup
##                    - required for Dual path setup      Name of filer to be used
## @param FILERA      - required for cluster setup
##                    - optional for Dual path setup      Name of filer to be used
## @param FILERB      - required for cluster setup
##                    - optional for Dual path setup      Name of the partner node
## @param TEST_CONFIG - optional Type of the setup, C for cluster, D for dual path (default)
## @param TEST_SETUP  - optional Default value set to 'FC'
## @param FILER_CONN  - optional Type of connection, default is set to 'console' for this test
## @param Mode        - optional Filer mode maint/normal, default set to 'maint' for this test
## @param FILER_PROMPT - optional prompt of the filer
## @param LOGDIR      - optional This is required to generate 'END-LOG'. Default set to main log directory.
## @param EOE         - optional 'default' is set to default value
## @param BOOT_MODE   - optional Boot mode for filer , default set to '5' for this test
## @param BOOT_TYPE   - optional Boot type, default set to 'A' (automatic) for this test
## @param SSD  	 - optional Setup is SSD or not, default set to 'no'
## @param MAIL_TO     - Required. Destination email id.
## @param MAIL_FROM   - Required. Sender email id.
## @param EMAIL       - optional 'default' is set to 'Y'.
## @param FILER_TYPE  - Required to set DataONTAP version    DataONTAP version is either IC or BR
##
## @status  Automated
## @author  himangin@netapp.com
## @burt    844011
## @change  shifalis : Modified for Burt 1033256
##
#############################################################################

#########################################
### Library functions
##########################################

# Compiler directives
use strict;
use TCD;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;

##user-defined functions
sub _array_diff(\@\@);
sub _process($$$$);
use Params::Validate qw(validate OBJECT SCALAR HASHREF ARRAYREF);
use Storage::Common_Lib;
use Storage::SASUtils qw(disable_dbg_msgs);

# Global parameters
use vars qw(
  $FILER
  $TEST_CONFIG
  $TEST_SETUP
  $MODE
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
);

my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'D' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'FC' );
$MODE           = $params->get( 'MODE',           default => 'maint' );
$FILER_PROMPT   = $params->get( 'FILER_PROMPT',   default => '\*>' );
$LOGDIR         = $params->get( 'LOGDIR',         default => '' );
$EOE            = $params->get( 'EOE',            default => 'default' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '5' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$SSD            = $params->get( 'SSD',            default => 'no' );
$EMAIL          = $params->get( 'EMAIL',          default => 'Y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'DataONTAP version' );

# CMode Global Variables
$BOOT_TYPE = "A";
my @Nodes;
my @FILER_ARR;
my $filer_names;
my $Mode;
my @API_ARRAY;
my $test_status = 0;
my %nodes_filer = ();
my %FILER_API   = ();
my %FILER_DISK  = ();
my $FILER_STATE;
my %disk_iops;

logcomment("TEST CONFIG - $TEST_CONFIG");
my $TC_name = "303_NADQ02_NVMe_Disk_Perf_Norm";

# Testcase available for execution
my @Testcases = ( Disk_Perf_Norm => "Performance Normalization Test (disktest) - Maintenance Mode" );
#########################################
## Pre-test processes
#########################################
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main

########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }

    my @Nodes_key = keys(%nodes_filer);
    foreach my $key (@Nodes_key) {
        if ( $key =~ /HASH/ ) {
            delete $nodes_filer{$key};
        }
    }
    @Nodes = values(%nodes_filer);

    @FILER_ARR = split( /,/, $filer_names );
    $Mode = $Nodes[0]->mode();
    $Test->nlog("Mode of the Filers $Mode");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );

        #$FILER_STATE = $Transit_obj->get_state();
        $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        $Test->nlog("FILER- $FILER_C - Is in $FILER_STATE state");
        if ( $FILER_STATE !~ /MAINT/ ) {
            $Transit_obj->change_state( to => "MAINT" );
        }
        $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );

        if ( $FILER_STATE =~ /MAINT/ ) {
            my $host       = host($FILER_C);
            my $API_Object = NACL::APISet->new(
                hostobj   => $host,
                category  => "Node",
                interface => "CLI",
                set       => "Maintenance"
            );
            push( @API_ARRAY, $API_Object );
            $FILER_API{$FILER_C} = $API_Object;
            my @temp_disk_info = nvme_get_disk_owned( Node => $Node );
            @temp_disk_info = sort(@temp_disk_info);
            $FILER_DISK{$FILER_C} = [@temp_disk_info];
        }
    }
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

##Test will run only Single Filer##
    version_test( node_present => \@Nodes, tc_name => $TC_name );

    return $TCD::PASS;
}

########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test exectuion ");
    logcomment("Filer present : $filer_names");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
	nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );

    my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
    my $disk_qual_path = $actual_log_dir . "/disk_qual_cnt_$TC_name.log";
    logcomment("**DEBUG** : log path : $disk_qual_path");
    open( FH, ">>$disk_qual_path" ) || die "Failed to open the write file $!";
    close FH;
    return $TCD::PASS;
}

##########################################
####   Test starts here
##########################################

sub Disk_Perf_Norm {
    my @subtests;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filers are $filer_names" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&Disk_Perf_Norm_sub, -runid, "Disk_Perf_Norm_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $status = status_return( subtest_ref => [@subtests] );
    logcomment("Total test status is : $status");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub Disk_Perf_Norm_sub {
    my @Nodes;
    $Test->description("Disk Performance Normalisation started");
    logcomment("Starting execution of test steps. Total steps to be executed as part of the script is: 12");
    push( @Nodes, shift(@_) );
    logcomment( "Filer passed to the subtest :: " . $Nodes[0]->name() );

#######################
## Test variables
#######################
    #my %disk_iops;
    my %disk_qual_cnt;
    my $test_iops = {};
    my %block_size;
    my %capacity_disk;
    my @prod_id_array;
    my @disk_array;
    my $maxlba;
    my @test_arr;
    my ( $OD_Start_LBA, $MD_Start_LBA, $ID_Start_LBA );

    ##These steps are performed in Maintenance mode##
    #------------------------------------------------------------
    ## Step 1 - Check the drive under test total LBAs. <Max LBA+1> = scsi capacity "blocks" value.
    ## Pass/Fail criteria: Verify physical disk count
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API_Obj = $FILER_API{$FILER_C};
        my @array   = @{ $FILER_DISK{$FILER_C} };
        grep {
            if ( $_ !~ /P\d+$/ ) { push( @disk_array, $_ ) }
        } @array;
        my $capacity;
        eye_catcher( Test => $Test, string => "STEP 1 of 4 : $FILER_C :Check the drive under test total LBAs. <Max LBA+1> = scsi capacity blocks value" );
        foreach my $disk (@disk_array) {
            logcomment("Filer : $FILER_C : checking the scsi capacity of the drive $disk");
            my $output = $API_Obj->execute_raw_command( command => "scsi capacity $disk" );    #7501476528 blocks x 520 bytes/blk = 3900767794560 bytes
            if ( $output =~ /(\d+)\s+blocks/ ) {
                $capacity = $1;
            }
            $block_size{$FILER_C}{$disk} = $capacity;
            logcomment("Capacity of the drive $disk is $capacity");
            push( @{ $capacity_disk{$FILER_C}{$capacity} }, $disk );                           ##pushing the drives to array with capacity as a key
        }
        my @check = keys( %{ $capacity_disk{$FILER_C} } );
        if ( scalar @check > 1 ) {
            logcomment( "Total number of different capacity are " . scalar @check . " and the capacity are : @check" );
            grep { logcomment( @{ $capacity_disk{$FILER_C}{$_} } . " disks have same capacity i.e $_ " ) } @check;
            $test_status = 1;
            logcomment("**FATAL** : $FILER_C TEST REQUIRES ALL THE DISKS SHOULD HAVE SAME CAPACITY,BUT SOME OF THE DISK HAVE DIFERENT CAPACITY.Exiting from the test - STEP 1");
            return logresult( "INFO", msg => $test_status );
        } else {
            logcomment("All the drives have same capacity <Read Capacity+1> i.e @check");
            $maxlba = $check[0] - 1;
        }
    }

    #---------------------------------------------------------------
    ## Step 2 - calculate starting LBAs for OD,MD and ID Test Areas
    ## 	    also make sure start LBAs are 4KB aligned
    ## Pass/Fail criteria:N/A
    #--------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 2 of 4 : Calculate starting LBAs for OD,MD and ID Test Areas, also make sure start LBAs are 4KB aligned" );
        $OD_Start_LBA = 4160000;
        $MD_Start_LBA = int( ( ( $maxlba / 2 ) - 10000000 ) / 4160 ) * 4160;
        $ID_Start_LBA = int( ( $maxlba - 20000000 ) / 4160 ) * 4160;
        logcomment("OD Start LBA - $OD_Start_LBA\nMD Start LBA - $MD_Start_LBA\nID Start LBA - $ID_Start_LBA");
        push( @test_arr, $OD_Start_LBA, $MD_Start_LBA, $ID_Start_LBA );
    }

    #---------------------------------------------------------------
    ## Step 3 - Perform sequential reads/writes at OD/MD/ID for each drive
    ##          This step will perform the following:
    ##   		STEP: 3.1) OD READS
    ##   		STEP: 3.2) OD WRITES
    ##		STEP: 3.3) MD READS
    ##   		STEP: 3.4) MD WRITES
    ##   		STEP: 3.5) ID READS
    ##   		STEP: 3.6) ID WRITES
    ## Pass/Fail criteria:N/A
    #---------------------------------------------------------------
    # type 1=sequential read, 5=sequential write
    my @test_type = qw (1 5);
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API_Obj = $FILER_API{$FILER_C};
        eye_catcher( Test => $Test, string => "STEP 3 of 4 : Perform sequential reads/writes at OD/MD/ID for each drive" );
        my $string;
        foreach my $test_area (@test_arr) {
            my $val;
            if ( $test_area =~ /$OD_Start_LBA$/ ) {
                $val = "OD";
            } elsif ( $test_area =~ /$MD_Start_LBA$/ ) {
                $val = "MD";
            } elsif ( $test_area =~ /$ID_Start_LBA$/ ) {
                $val = "ID";
            }
            logcomment("=== Starting tests in $val area ===");
            foreach my $test (@test_type) {
                if ( $test == 1 ) {
                    $string = "sequential read";
                } elsif ( $test == 5 ) {
                    $string = "sequential write";
                }
                logcomment("=== Starting $string tests for each drive ===");
                $API_Obj->set_timeout( "connectrec-timeout" => 1000 );

                foreach my $drive (@disk_array) {
                    grep { $disk_qual_cnt{$_}++ } $drive;
                    my $output = $API_Obj->execute_raw_command( 'command' => "disk_qual start -t $test -n 512 -q 8 -f $test_area -l $maxlba -L 37560 -d $drive -o noverify" );
                    sleep(5);
                    my $check         = 0;
                    my $time_out_temp = 0;
                    my $max_time      = 18000;
                    while ( $time_out_temp < $max_time ) {
                        my $after1 = $API_Obj->execute_raw_command( 'command' => "" );
                        $output = $output . $after1;
                        if ( $output =~ /Data Rate:\s+(\S+)\s+/ ) {
                            my $data_rate = $1;
                            logcomment("Data rate of drive $drive : $data_rate");
                            push( @{ $disk_iops{$test_area}{$string} }, $data_rate );
                            $test_iops->{$test_area}->{$test} = $data_rate;
                            $check = 1;
                            last;
                        } elsif ( $output =~ /(\w+.\w+),passed,\d+,\d+,\w+\s*\w+,\d+,\d+,\d+,\d+,\d+,\d+,\d+,\d+,-\d+,\d+,\d+,\d+,\d+,((\d+)|(\d+.\d+)),.*/ ) {
                            my $data_rate = $2;
                            logcomment("Data rate of drive $drive : $data_rate");
                            push( @{ $disk_iops{$test_area}{$string} }, $data_rate );
                            $test_iops->{$test_area}->{$test} = $data_rate;
                            $check = 1;
                            last;
                        } else {
                            sleep 120;
                            $time_out_temp = $time_out_temp + 120;
                        }
                    }
                    if ( $check == 1 ) {
                        logcomment("Disk_qual completed for drive : $drive");
                    } else {
                        $test_status = 1;
                        logcomment("**FAIL** : $FILER_C : DISK_QUAL NOT COMPLETED FOR DRIVE $drive WITHIN THE GIVEN TIME - STEP : 3");
                    }
                }

            }

        }
    }

    # ----------------------------------------------------
    # STEP 4: Determine PASS/FAIL for OD/MD/ID Test
    # Check for greater than 10%
    # ----------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API_Obj = $FILER_API{$FILER_C};
        eye_catcher( Test => $Test, string => "STEP 4 of 4 : Determine PASS/FAIL for OD/MD/ID Test" );
        my ( @OD_read,       @MD_read,       @ID_read );
        my ( $OD_read_flag,  $MD_read_flag,  $ID_read_flag );
        my ( @OD_write,      @MD_write,      @ID_write );
        my ( $OD_write_flag, $MD_write_flag, $ID_write_flag );
        if (%disk_iops) {

            foreach my $keys ( keys %disk_iops ) {
                if ( $keys =~ /$OD_Start_LBA$/ ) {
                    foreach ( keys %{ $disk_iops{$keys} } ) {
                        if ( $_ =~ /write/ ) {
                            @OD_write = @{ $disk_iops{$keys}{$_} };
                        } elsif ( $_ =~ /read/ ) {
                            @OD_read = @{ $disk_iops{$keys}{$_} };
                        }
                    }
                } elsif ( $keys =~ /$MD_Start_LBA$/ ) {
                    foreach ( keys %{ $disk_iops{$keys} } ) {
                        if ( $_ =~ /write/ ) {
                            @MD_write = @{ $disk_iops{$keys}{$_} };
                        } elsif ( $_ =~ /read/ ) {
                            @MD_read = @{ $disk_iops{$keys}{$_} };
                        }
                    }
                } elsif ( $keys =~ /$ID_Start_LBA$/ ) {
                    foreach ( keys %{ $disk_iops{$keys} } ) {
                        if ( $_ =~ /write/ ) {
                            @ID_write = @{ $disk_iops{$keys}{$_} };
                        } elsif ( $_ =~ /read/ ) {
                            @ID_read = @{ $disk_iops{$keys}{$_} };
                        }
                    }
                }
            }
            @OD_read = sort ( unique(@OD_read) );
            logcomment("**DEBUG** : OD_Read - @OD_read");
            @OD_write = sort ( unique(@OD_write) );
            logcomment("**DEBUG** : OD_Write - @OD_write");
            @MD_read = sort ( unique(@MD_read) );
            logcomment("**DEBUG** : MD_Read - @MD_read");
            @MD_write = sort ( unique(@MD_write) );
            logcomment("**DEBUG** : MD_Write - @MD_write");
            @ID_read = sort ( unique(@ID_read) );
            logcomment("**DEBUG** : ID_Read - @ID_read");
            @ID_write = sort ( unique(@ID_write) );
            logcomment("**DEBUG** : ID_Write - @ID_write");
            my $OD_Delta_read = ( ( $OD_read[-1] - $OD_read[0] ) / $OD_read[-1] ) * 100;

            if ( $OD_Delta_read > 10 ) {
                $test_status = 1;
                logcomment("**FAIL** :  OD READ IS FOUND GREATER THAN 10% - STEP : 4");
                $OD_read_flag = "FAIL";
            } else {
                $OD_read_flag = "PASS";
            }
            my $OD_Delta_write = ( ( $OD_write[-1] - $OD_write[0] ) / $OD_write[-1] ) * 100;

            if ( $OD_Delta_write > 10 ) {
                $test_status = 1;
                logcomment("**FAIL** :  OD WRITE IS FOUND GREATER THAN 10% - STEP : 4");
                $OD_write_flag = "FAIL";
            } else {
                $OD_write_flag = "PASS";
            }
            my $MD_Delta_read = ( ( $MD_read[-1] - $MD_read[0] ) / $MD_read[-1] ) * 100;
            if ( $MD_Delta_read > 10 ) {
                $test_status = 1;
                logcomment("**FAIL** : MD READ IS FOUND GREATER THAN 10% - STEP : 4");
                $MD_read_flag = "FAIL";
            } else {
                $MD_read_flag = "PASS";
            }
            my $MD_Delta_write = ( ( $MD_write[-1] - $MD_write[0] ) / $MD_write[-1] ) * 100;
            if ( $MD_Delta_write > 10 ) {
                $test_status = 1;
                logcomment("**FAIL** : MD WRITE IS FOUND GREATER THAN 10% - STEP : 4");
                $MD_write_flag = "FAIL";
            } else {
                $MD_write_flag = "PASS";
            }
            my $ID_Delta_read = ( ( $ID_read[-1] - $ID_read[0] ) / $ID_read[-1] ) * 100;
            if ( $ID_Delta_read > 10 ) {
                $test_status = 1;
                logcomment("**FAIL** : ID READ IS FOUND GREATER THAN 10% - STEP : 4");
                $ID_read_flag = "FAIL";
            } else {
                $ID_read_flag = "PASS";
            }
            my $ID_Delta_write = ( ( $ID_write[-1] - $ID_write[0] ) / $ID_write[-1] ) * 100;
            if ( $ID_Delta_write > 10 ) {
                $test_status = 1;
                logcomment("**FAIL** : ID WRITE IS FOUND GREATER THAN 10% - STEP : 4");
                $ID_write_flag = "FAIL";
            } else {
                $ID_write_flag = "PASS";
            }

#########################################
## Summary Result
#########################################
            logcomment("\n#########################################\n## Summary Result\n#########################################");
            logcomment("\nREAD OD: Max = $OD_read[-1] Min = $OD_read[0] Del = $OD_Delta_read = $OD_read_flag\nREAD MD: Max = $MD_read[-1] Min = $MD_read[0] Del = $MD_Delta_read = $MD_read_flag\nREAD ID: Max = $ID_read[-1] Min = $ID_read[0] Del = $ID_Delta_read = $ID_read_flag");
            logcomment("\nWRITE OD: Max = $OD_write[-1] Min = $OD_write[0] Del = $OD_Delta_write = $OD_write_flag\nWRITE MD: Max = $MD_write[-1] Min = $MD_write[0] Del = $MD_Delta_write = $MD_write_flag\nWRITE ID: Max = $ID_write[-1] Min = $ID_write[0] Del = $ID_Delta_write = $ID_write_flag");
        } else {
            $test_status = 1;
            logcomment("**FAIL** : DATA RATE IS NOT CAPTURED PROPERLY - STEP : 4");
        }
    }
    my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
    my $disk_qual_path = $actual_log_dir . "/disk_qual_cnt_$TC_name.log";
    logcomment("**DEBUG** : log path : $disk_qual_path");
    open( FH, ">>$disk_qual_path" ) || die "Failed to open the write file disk_qual.txt";
    foreach my $dsk ( keys %disk_qual_cnt ) {
        logcomment("$dsk\t\t$disk_qual_cnt{$dsk}");
        print FH "$dsk\t\t$disk_qual_cnt{$dsk}\n";
    }
    close FH;
#################################################
    logresult( "INFO", msg => $test_status );
}

#########################################
##cleanup
#########################################
sub cleanup() {
    logcomment(" $filer_names - Clean up and post test process");
    logcomment("Nodes present - $filer_names");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ###########################################################################################
	nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );

    return $TCD::PASS;
}