#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Compliance.thpl#1 $
#
## @summary Mode
##  Disk Firmware Download in background/foreground mode
#
##
## @description
## Verify that the drive can download disk firmware in foreground and background mode properly
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground
##
## @steps
## The test will execute steps mentioned below:
##
##
## @status  Automated
########################################
### Library functions
##########################################
use strict;
use Subtest;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use Storage::Tahiti_Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
use Data::Dumper;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
  $USR_PRE_COND
  $RUN_TIME
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$USR_PRE_COND   = $params->get( 'USR_PRE_COND',   default => 'no pre_cond' );
$RUN_TIME       = $params->get( 'RUN_TIME',       default => '3600' );
################
# Testcase name
################
my $TC_name;
$TC_name = "2202_NADQ02_SCM_Stress_Mixed_90R_10W_RAND";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home           = $ENV{HOME};
my $FILER_DRV_DETS = {};

my $test_wait_time = 12 * 3600;    #12 Hrs test
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( SCM_IO => "SCM IO  script." );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    scm_pre_test(
        node_present => $node_ref,
        Test         => $Test,
        filer_mode   => $Mode
    );

    return $TCD::PASS;
}

sub SCM_IO {
    my @subtests;
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&ex_cmd_exe, -runid, "scm_io_cmd_exe_$FILER_C", -bg, "--", $Node ) );
        Subtest::wait_finish( subtest => [@subtests] );
    }

    # Subtest::wait_finish( subtest => [@subtests] );
    my $status = status_return( subtest_ref => [@subtests] );
    logcomment("Total test status is : $status");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub ex_cmd_exe {
    $Test->description("Executing IO commands on SCM drive");
    my ( @Nodes, %filer_nvol );
    my $Home = $ENV{HOME};
    my @NON_RT_AGG;
    my @NON_RT_VOLS;
    my $Api_Set_Obj_SS;
    my $drv_result      = {};
    my $flash_cache_drv = {};
    my $fl_drvs         = {};
    my $summary_drv     = {};
    my @nv_drv;
    my $Node             = shift(@_);
    my $FILER_C          = $Node->name();
    my $Host             = host($FILER_C);
    my $create_agg_n_vol = 0;
    logcomment("Total Nodes  - $Node");
    logcomment( "Filer passed to the subtest :: " . $Node->name() );

    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );

    #my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();

    #STEP 1
    nvme_eye_catcher( Test => $Test, string => "STEP 1 of 7 : Set Privilage level to test" );
    logcomment(" Privilage Level ");
    $Api_Set_Obj->execute_raw_command( 'command' => "set" );
    logcomment("Set privilage level to diag");
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    logcomment("Privilage level diag set");
    $Api_Set_Obj->execute_raw_command( 'command' => "set" );
    logcomment("Set console log level");
    $Api_Set_Obj->execute_raw_command( 'command' => "event config modify -suppression on -console off -console-log-level NODE_ERROR" );

    #STEP 2
    nvme_eye_catcher( Test => $Test, string => "STEP 2 of 7 : Get filer State" );
    logcomment("Get filer state ");
    my $Transit_obj = NACL::Transit->new( name => $FILER_C );
    my $FILER_O_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
    logcomment("FILER- $FILER_C - Is in $FILER_O_STATE state");
    if ( $FILER_O_STATE =~ /CLI|UP/ ) {
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        $FILER_DRV_DETS->{$FILER_C}->{'FILER_NAME'} = $FILER_C;
        $FILER_DRV_DETS->{$FILER_C}->{'FILER_API'}  = $Api_Set_Obj;
    } else {
        logcomment("Filer has to be in CLI/FileSystem mode, Boot filer in file system mode and re-start test");
        $test_status = 1;
        logresult( "INFO", msg => $test_status );
        logresult( "FAIL", "FAILED TO INSTANTIATE TEST OBJECT." );
        return $TCD::FAIL;
    }

    #STEP 3
    nvme_eye_catcher( Test => $Test, string => "STEP 3 of 7 : Check for flash cache drives" );
    logcomment("$FILER_C : check for flash cache drives");
    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "flash-cache show" );
    foreach my $line ( split( /\n/, $out ) ) {
        if ( $line =~ /(\d+\-\d+|\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s+(\S+)/ ) {
            my $serial = $4;
            $flash_cache_drv->{$serial}->{'FILER'}  = $FILER_C;
            $flash_cache_drv->{$serial}->{'SLOT'}   = $1;
            $flash_cache_drv->{$serial}->{'MODEL'}  = $2;
            $flash_cache_drv->{$serial}->{'PART'}   = $3;
            $flash_cache_drv->{$serial}->{'SERIAL'} = $serial;
            $flash_cache_drv->{$serial}->{'FW'}     = $5;
            $flash_cache_drv->{$serial}->{'CAP'}    = $6;
            $flash_cache_drv->{$serial}->{'STATE'}  = $7;
        }
    }
    my @fl_drv = keys(%$flash_cache_drv);
    if (@fl_drv) {
        logcomment("Flash-Caache Drives Found : @fl_drv ");
        foreach my $ser ( keys %$flash_cache_drv ) {
            logcomment("FILER : $flash_cache_drv->{$ser}->{'FILER'}\n SERIAL : $ser \n MODEL : $flash_cache_drv->{$ser}->{'MODEL'}\n PART NUMBER : $flash_cache_drv->{$ser}->{'PART'} \n SLOT : $flash_cache_drv->{$ser}->{'SLOT'}\n CAPACITY : $flash_cache_drv->{$ser}->{'CAP'} \n FIRMWARE : $flash_cache_drv->{$ser}->{'FW'}\n STATE : $flash_cache_drv->{$ser}->{'STATE'}");
        }
    } else {
        logcomment("No Flash-Cache drive found, Stopping test ");
        $test_status = 1;
        logresult( "INFO", msg => $test_status );
        logresult( "FAIL", "No Flash-Cache drive found" );
        return $TCD::FAIL;
    }

    #STEP 4
    nvme_eye_catcher( Test => $Test, string => "STEP 4 of 7  : Determine controllers, namespaces and format" );
    logcomment("Set State to offline ");
    logcomment("$FILER_C : Set State to offline in order to run stress commands");
    $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme state get" );
    sleep 1;
    $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme state set offline" );
    logcomment("NVMe State set to offline");
    $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme state get" );
    sleep 2;

    logcomment("$FILER_C : Determine controllers, namespaces and format of existing flash cache drives");
    my $fout = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;system node nvme show" );
    my $i = 1;
    foreach my $line ( split( /\n/, $fout ) ) {
        if ( $line =~ /(\d+)\s+(\d+)\s+(\S+)\s+(\S+\s+\S+)\s+(\S+\s+\/\s+\S+)\s+(\S+)/ ) {
            my $cap = $4;
            if ( $cap !~ /0\s+B/ ) {
                my $drv_cnt = "drv_" . $i;
                $fl_drvs->{$drv_cnt}->{'ctrl'} = $1;
                $fl_drvs->{$drv_cnt}->{'nsp'}  = $2;
                $fl_drvs->{$drv_cnt}->{'slot'} = $3;
                $fl_drvs->{$drv_cnt}->{'cap'}  = $cap;
                my $fmt = $5;
                my ( $f1, $f2 ) = $fmt =~ /(\S+)\s+\/\s+(\S+)/g;
                my $lbasperop = $f1 + $f2;
                my $fmt2      = $lbasperop;
                $lbasperop = 256 if ( $lbasperop == 520 );
                $lbasperop = 32  if ( $lbasperop == 4160 );
                $fl_drvs->{$drv_cnt}->{'fmt'}   = $fmt;
                $fl_drvs->{$drv_cnt}->{'LBAOP'} = $lbasperop;
                $fl_drvs->{$drv_cnt}->{'state'} = $6;
                $fl_drvs->{$drv_cnt}->{'frmt2'} = $fmt2;
                $i++;
            }
        }
    }
    logcomment("Contollers and namespace details ");
    my $forma;
    logcomment("Check for format of drive");
    my $cap;
    foreach my $d ( keys %$fl_drvs ) {
        $forma = $fl_drvs->{$d}->{'frmt2'};
        $cap   = $fl_drvs->{$d}->{'cap'};
    }
    logcomment("Format is : $forma");
    my @exe_handle = qw ( RAND );

    #STEP 5
    foreach my $exe (@exe_handle) {
        logcomment("Execute $exe ");
        nvme_eye_catcher( Test => $Test, string => "STEP 5 of 7  : Execute Stress command - Mixed 90% Read 10% Write Random" ) if ( $exe =~ /RAND/ );
        logcomment("Executing - Mixed Random 90%-read 10%-write") if ( $exe =~ /RAND/ );
        my @tras_sz;
        if ( $forma == 4160 ) {
            @tras_sz = qw (1 2 4 8 16);
        } elsif ( $forma == 520 ) {
            @tras_sz = qw (8 16 32 64 128);
        } else {
            logcomment("**WARNING** : Cannot get drive fromat, Default 520 format Transfer size will be executed");
            @tras_sz = qw (1 2 4 8 16);
        }

        my @que_dept = qw (1 4 8 16 32 64 128 256);
        my $runtime  = 600;
        my $exe_cmd;
        logcomment("RUNID : $RUNID");
        logcomment("LOGDIR : $LOGDIR");
        my ($loc)   = $LOGDIR =~ /(.*)\/HDD\/(.*)/;
        my ($fname) = $RUNID  =~ /\S+\/\S+\/\S+\/(\S+)\/.*/;
        my $filename;

        if ( $exe =~ /RAND/ ) {
            $filename = $fname . "_IOPS_LAT_90PRD_10PWR_RAND_Report.html" if ( $exe =~ /RAND/ );
            logcomment("DEBUG : Filename : $filename");
        }
        my $parfile = $loc . "/" . $filename;

        #open( my $fh, '>>', '/home/arun_51/report_results.txt' );
        #my $parfile = "/home/arun_51/report_scm_new.html";
        logcomment("Results FILE location : $parfile");
        open( FOUT, ">>", $parfile ) or die "Could not open file $parfile: $!";
        system("chmod 777 $parfile");
        print FOUT "\n<table border=2 style width=75%>";
        print FOUT "<tr style=background-color:navy;color:white><th colspan=2><font size=6 face=arial> RND_Mixed : NADQ02_SCM_Stress_Mixed_90R_10W_RAND - $FILER_C</th><tr>" if ( $exe =~ /RAND/ );
        my $my_bgcolor = "#ffdead";
        print FOUT "\n<tr style=background-color:$my_bgcolor;color:black><th><font face=arial> Test Name     </th><th><font face=arial> NADQ02_SCM_Stress_Mixed_90R_10W_RAND </th></tr>";
        print FOUT "\n<tr style=background-color:$my_bgcolor;color:black><th><font face=arial> IO            </th><th><font face=arial> 90%-READ | 10%-WRITE </th></tr>" if ( $exe =~ /RAND/ );
        print FOUT "\n<tr style=background-color:$my_bgcolor;color:black><th><font face=arial> Filer 	 </th><th><font face=arial> $FILER_C </th></tr>";
        print FOUT "\n<tr style=background-color:$my_bgcolor;color:black><th><font face=arial> Drive fromat  </th><th><font face=arial> $forma </th></tr>";
        print FOUT "\n<tr style=background-color:$my_bgcolor;color:black><th><font face=arial> Drive Capacity </th><th><font face=arial> $cap </th></tr>";
        print FOUT "\n<tr style=background-color:$my_bgcolor;color:black><th><font face=arial> Runtime       </th><th><font face=arial> $runtime </th></tr>";
        print FOUT "\n<tr style=background-color:$my_bgcolor;color:black><th><font face=arial> Queues        </th><th><font face=arial> @que_dept </th></tr>";
        print FOUT "\n<tr style=background-color:$my_bgcolor;color:black><th><font face=arial> Transfer size </th><th><font face=arial> @tras_sz  </th></tr>";
        print FOUT "\n</table>";
        print FOUT "\n<table border=2 style width=75%>";
        print FOUT "\n<tr align=center bgcolor=cyan>";
        print FOUT "\n<th bgcolor=cyan><font face=arial>Slot:Ctrl:Nspace</th>";
        print FOUT "\n<th bgcolor=magenta><font face=arial>QueueDept</th>";
        print FOUT "\n<th bgcolor=sandybrown><font face=arial>TransferSize</th>";
        print FOUT "\n<th bgcolor=#ADFF2F><font face=arial>ReadIops</th>";
        print FOUT "\n<th bgcolor=#ADFF2F><font face=arial>ReadLat(us)</th>";
        print FOUT "\n<th bgcolor=#ADFF2F><font face=arial>ReadTput(MiB/s)</th>";
        print FOUT "\n<th bgcolor=#32CD32><font face=arial>WriteIops</th>";
        print FOUT "\n<th bgcolor=#32CD32><font face=arial>WriteLat(us)</th>";
        print FOUT "\n<th bgcolor=#32CD32><font face=arial>WriteTput(MiB/s)</th>";
        print FOUT "\n </tr>";
        my @y_axis_rdlat;
        my @x_axis_rdiops;

        foreach my $que (@que_dept) {
            logcomment("Executing for Que - $que");
            logcomment("Transfer Size : @tras_sz");
            foreach my $val (@tras_sz) {
                logcomment("Data Transfer size - $val");
                foreach my $d ( keys %$fl_drvs ) {
                    my $cmd1;
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    sleep 3;
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    my $ctrl  = $fl_drvs->{$d}->{'ctrl'};
                    my $naspc = $fl_drvs->{$d}->{'nsp'};
                    if ( $exe =~ /RAND/ ) {
                        $cmd1 = "system node nvme stress start -controller $ctrl -namespace $naspc -threads 1 -ops-per-thread $que -rd-percent 90 -rd-verify false -lbas-per-op $val -runtime $runtime -rd-op rand -wr-iter 0 -wr-op rand -wr-skip-freq 0 -wr-skip-amt 0";
                    } else {
                        logcomment("**FATAL** : Cannot get stress command ");
                        logresult( "FATAL", "Cannot get / execute stress command" );
                    }
                    $exe_cmd = $cmd1;
                    logcomment("COMMAND : $cmd1");
                    my $fout = $Api_Set_Obj->execute_raw_command( 'command' => "$cmd1" );
                    if ( $fout =~ /An error occurred in the sysctl/ ) {
                        logcomment("An error occurred in the sysctl, Test will not proceed further");
                        logcomment("Command : $cmd1");
                        logcomment("**FATAL** : An error occurred in the sysctl");
                        logresult( "FATAL", "An error occurred in the sysctl" );
                    }

                }
                my $mina = int( $runtime / 60 );
                logcomment("Command will be executed for $mina min");
                my $start_time = time;
                my $end_time   = $start_time + $runtime;
                logcomment("DEBUG : End time : $end_time");
                while ( $end_time > time ) {
                    sleep 120;
                    my $ct = ( $runtime - 120 ) / 60;
                    logcomment("DEBUG : Remaining Time : $ct min");
                }
                my $etime = time;
                logcomment("DEBUG : Completed nvme stress command executed for 10 min");
                logcomment("DEBUG : Start Time : $start_time and End Time : $etime ");
                sleep 5;
                my $fout = $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                sleep 5;
                logcomment("Execute Stress status and get IOPS and Latency values ");

                foreach my $d ( keys %$fl_drvs ) {
                    my $ctrl  = $fl_drvs->{$d}->{'ctrl'};
                    my $naspc = $fl_drvs->{$d}->{'nsp'};
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    sleep 2;
                    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme stress stats -controller $ctrl -namespace $naspc" );
                    my ( $iops_rd, $iops_wr, $avg_rd_lat, $avg_rd_thput, $avg_wr_lat, $avg_wr_thput );
                    my $rd_flag = 0;
                    $out = $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme stress stats -controller $ctrl -namespace $naspc" ) if ( $out eq '' );

                    foreach my $line ( split( /\n/, $out ) ) {
                        $line =~ s/MiB\/s|us//g;
                        $line =~ s/\s+//g;
                        if ( $line =~ /^ReadsIOPS:|Reads/ ) {
                            ($iops_rd) = $line =~ /ReadsIOPS:(\S+)/;
                            $rd_flag = 1;
                        }
                        if ( $line =~ /^WritesIOPS:|Writes/ ) {
                            ($iops_wr) = $line =~ /WritesIOPS:(\S+)/;
                            $rd_flag = 0;
                        }
                        if ( ( $rd_flag == 1 ) && ( $line =~ /IOPS/ ) ) {
                            ($iops_rd) = $line =~ /IOPS:(\S+)/;
                        }
                        if ( ( $rd_flag == 1 ) && ( $line =~ /AverageLatency/ ) ) {
                            ($avg_rd_lat) = $line =~ /AverageLatency:(\S+)/;
                        }
                        if ( ( $rd_flag == 1 ) && ( $line =~ /AverageThroughput/ ) ) {
                            ($avg_rd_thput) = $line =~ /AverageThroughput:(\S+)/;
                        }
                        if ( ( $rd_flag == 0 ) && ( $line =~ /IOPS/ ) ) {
                            ($iops_wr) = $line =~ /IOPS:(\S+)/;
                        }
                        if ( ( $rd_flag == 0 ) && ( $line =~ /AverageLatency/ ) ) {
                            ($avg_wr_lat) = $line =~ /AverageLatency:(\S+)/;
                        }
                        if ( ( $rd_flag == 0 ) && ( $line =~ /AverageThroughput/ ) ) {
                            ($avg_wr_thput) = $line =~ /AverageThroughput:(\S+)/;
                        }

                    }
                    nvme_eye_catcher( Test => $Test, string => "SUMMARY for controller $ctrl -namespace $naspc | Drive - $d | Que - $que | Transfer Size - $val" );
                    logcomment("Controller : $ctrl NameSpace :$naspc");
                    logcomment("READ_IOPS : $iops_rd\nREAD_AVG_LATENCY : $avg_rd_lat\nREAD_AVG_THROUGHPUT : $avg_rd_thput");
                    logcomment("WRITE_IOPS : $iops_wr\nWRITE_AVG_LATENCY : $avg_wr_lat\nWRITE_AVG_THROUGHPUT : $avg_wr_thput");
                    my $slot = $fl_drvs->{$d}->{'slot'};
                    my $ID   = $slot . ":" . $ctrl . ":" . $naspc;
                    logcomment("ID is $ID");
                    $summary_drv->{$d}->{'Drive'}   = $d;
                    $summary_drv->{$d}->{'Format'}  = $forma;
                    $summary_drv->{$d}->{'CTRL'}    = $ctrl;
                    $summary_drv->{$d}->{'NSPACE'}  = $naspc;
                    $summary_drv->{$d}->{'Que'}     = $que;
                    $summary_drv->{$d}->{'Transf'}  = $val;
                    $summary_drv->{$d}->{'RDIOPS'}  = $iops_rd;
                    $summary_drv->{$d}->{'RDTHRP'}  = $avg_rd_thput;
                    $summary_drv->{$d}->{'RDLAT'}   = $avg_rd_lat;
                    $summary_drv->{$d}->{'WRIOPS'}  = $iops_wr;
                    $summary_drv->{$d}->{'WRTHRP'}  = $avg_wr_thput;
                    $summary_drv->{$d}->{'WRLAT'}   = $avg_wr_lat;
                    $summary_drv->{$d}->{'CMD_EXE'} = $exe_cmd;
                    $summary_drv->{$d}->{'FILE'}    = $parfile;
                    my $my_bg3color = "#FFEAEA";
                    print FOUT "\n<tr align=center>";
                    print FOUT "\n<td bgcolor=$my_bg3color><font face=arial>$ID</td>";
                    print FOUT "\n<td bgcolor=$my_bg3color><font face=arial>$que</td>";
                    print FOUT "\n<td bgcolor=$my_bg3color><font face=arial>$val</td>";
                    print FOUT "\n<td bgcolor=$my_bg3color><font face=arial>$iops_rd</td>";
                    print FOUT "\n<td bgcolor=$my_bg3color><font face=arial>$avg_rd_lat</td>";
                    print FOUT "\n<td bgcolor=$my_bg3color><font face=arial>$avg_rd_thput</td>";
                    print FOUT "\n<td bgcolor=$my_bg3color><font face=arial>$iops_wr</td>";
                    print FOUT "\n<td bgcolor=$my_bg3color><font face=arial>$avg_wr_lat</td>";
                    print FOUT "\n<td bgcolor=$my_bg3color><font face=arial>$avg_wr_thput</td>";
                    print FOUT "\n</tr>";
                    logcomment("DATA CAPTURED : $d");
                    push( @x_axis_rdiops, $iops_rd );
                    push( @y_axis_rdlat,  $avg_rd_lat );
                }
            }
        }
        print FOUT "\n</table>";
        print FOUT "<p>.</p>";

        logcomment("Close file ");
        close(FOUT);

    }    #end of foreach SED|RAND
         #STEP 6
    nvme_eye_catcher( Test => $Test, string => "STEP 6 of 7 : Get SUMMARY" );
    logcomment("SUMMARY RESULTS");
    logcomment( "$FILER_C : Dumper " . Dumper($summary_drv) );
    logcomment("$FILER_C : Completed Sequential read command on all available controllers ");
    foreach my $d ( keys %$summary_drv ) {
        logcomment("Drive : $d");
        my $filea = $summary_drv->{$d}->{'FILE'};
        logcomment("Summary report available : $filea ");
    }
    return logresult( "INFO", msg => $test_status );
}

# End of sub_exe
#####################################################################
# Cleanup - Post Process
#####################################################################
sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    scm_pre_test(
        node_present => $node_ref,
        Test         => $Test,
        filer_mode   => $Mode
    );

    return $TCD::PASS;
}