#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Compliance.thpl#1 $
#
## @summary Mode
##  Disk Firmware Download in background/foreground mode
#
##
## @description
## Verify that the drive can download disk firmware in foreground and background mode properly
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground
##
## @steps
## The test will execute steps mentioned below:
##
##
## @status  Automated
########################################
### Library functions
##########################################
use strict;
use Subtest;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use Storage::Tahiti_Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
use Data::Dumper;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
  $USR_PRE_COND
  $RUN_TIME
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$USR_PRE_COND   = $params->get( 'USR_PRE_COND',   default => 'no pre_cond' );
$RUN_TIME       = $params->get( 'RUN_TIME',       default => '3600' );
################
# Testcase name
################
my $TC_name;
$TC_name = "1102_NADQ02_SCM_100_RND_W";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home           = $ENV{HOME};
my $FILER_DRV_DETS = {};

my $test_wait_time = 12 * 3600;    #12 Hrs test
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( SCM_IO => "SCM IO  script." );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    scm_pre_test(
        node_present => $node_ref,
        Test         => $Test,
        filer_mode   => $Mode
    );

    return $TCD::PASS;
}

sub SCM_IO {
    my @subtests;
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&ex_cmd_exe, -runid, "scm_io_cmd_exe_$FILER_C", -bg, "--", $Node ) );
        Subtest::wait_finish( subtest => [@subtests] );
    }

    # Subtest::wait_finish( subtest => [@subtests] );
    my $status = status_return( subtest_ref => [@subtests] );
    logcomment("Total test status is : $status");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub ex_cmd_exe {
    $Test->description("Executing IO commands on SCM drive");
    my ( @Nodes, %filer_nvol );
    my $Home = $ENV{HOME};
    my @NON_RT_AGG;
    my @NON_RT_VOLS;
    my $Api_Set_Obj_SS;
    my $drv_result      = {};
    my $flash_cache_drv = {};
    my $fl_drvs         = {};
    my @nv_drv;
    my $Node             = shift(@_);
    my $FILER_C          = $Node->name();
    my $Host             = host($FILER_C);
    my $create_agg_n_vol = 0;
    logcomment("Total Nodes  - $Node");
    logcomment( "Filer passed to the subtest :: " . $Node->name() );

    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );

    #my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();

    #STEP 1
    nvme_eye_catcher( Test => $Test, string => "STEP 1 of 7 : Set Privilage level to test" );
    logcomment(" Privilage Level ");
    $Api_Set_Obj->execute_raw_command( 'command' => "set" );
    logcomment("Set privilage level to diag");
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    logcomment("Privilage level diag set");
    $Api_Set_Obj->execute_raw_command( 'command' => "set" );
    logcomment("Set console log level");
    $Api_Set_Obj->execute_raw_command( 'command' => "event config modify -suppression on -console off -console-log-level NODE_ERROR" );

    #STEP 2
    nvme_eye_catcher( Test => $Test, string => "STEP 2 of 7 : Get filer State" );
    logcomment("Get filer state ");
    my $Transit_obj = NACL::Transit->new( name => $FILER_C );
    my $FILER_O_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
    logcomment("FILER- $FILER_C - Is in $FILER_O_STATE state");
    if ( $FILER_O_STATE =~ /CLI|UP/ ) {
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        $FILER_DRV_DETS->{$FILER_C}->{'FILER_NAME'} = $FILER_C;
        $FILER_DRV_DETS->{$FILER_C}->{'FILER_API'}  = $Api_Set_Obj;
    } else {
        logcomment("Filer has to be in CLI/FileSystem mode, Boot filer in file system mode and re-start test");
        $test_status = 1;
        logresult( "INFO", msg => $test_status );
        logresult( "FAIL", "FAILED TO INSTANTIATE TEST OBJECT." );
        return $TCD::FAIL;
    }

    #STEP 3
    nvme_eye_catcher( Test => $Test, string => "STEP 3 of 7 : Check for flash cache drives" );
    logcomment("$FILER_C : check for flash cache drives");
    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "flash-cache show" );
    foreach my $line ( split( /\n/, $out ) ) {
        if ( $line =~ /(\d+\-\d+|\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s+(\S+)/ ) {
            my $serial = $4;
            $flash_cache_drv->{$serial}->{'FILER'}  = $FILER_C;
            $flash_cache_drv->{$serial}->{'SLOT'}   = $1;
            $flash_cache_drv->{$serial}->{'MODEL'}  = $2;
            $flash_cache_drv->{$serial}->{'PART'}   = $3;
            $flash_cache_drv->{$serial}->{'SERIAL'} = $serial;
            $flash_cache_drv->{$serial}->{'FW'}     = $5;
            $flash_cache_drv->{$serial}->{'CAP'}    = $6;
            $flash_cache_drv->{$serial}->{'STATE'}  = $7;
        }
    }
    my @fl_drv = keys(%$flash_cache_drv);
    if (@fl_drv) {
        logcomment("Flash-Caache Drives Found : @fl_drv ");
        foreach my $ser ( keys %$flash_cache_drv ) {
            logcomment("FILER : $flash_cache_drv->{$ser}->{'FILER'}\n SERIAL : $ser \n MODEL : $flash_cache_drv->{$ser}->{'MODEL'}\n PART NUMBER : $flash_cache_drv->{$ser}->{'PART'} \n SLOT : $flash_cache_drv->{$ser}->{'SLOT'}\n CAPACITY : $flash_cache_drv->{$ser}->{'CAP'} \n FIRMWARE : $flash_cache_drv->{$ser}->{'FW'}\n STATE : $flash_cache_drv->{$ser}->{'STATE'}");
        }
    } else {
        logcomment("No Flash-Cache drive found, Stopping test ");
        $test_status = 1;
        logresult( "INFO", msg => $test_status );
        logresult( "FAIL", "No Flash-Cache drive found" );
        return $TCD::FAIL;
    }

    #STEP 4
    nvme_eye_catcher( Test => $Test, string => "STEP 4 of 7  : Determine controllers, namespaces and format" );
    logcomment("$FILER_C : Determine controllers, namespaces and format of existing flash cache drives");
    my $fout = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;system node nvme show" );
    my $i = 1;
    foreach my $line ( split( /\n/, $fout ) ) {
        if ( $line =~ /(\d+)\s+(\d+)\s+(\S+)\s+(\S+\s+\S+)\s+(\S+\s+\/\s+\S+)\s+(\S+)/ ) {
            my $cap = $4;
            if ( $cap !~ /0\s+B/ ) {
                my $drv_cnt = "drv_" . $i;
                $fl_drvs->{$drv_cnt}->{'ctrl'} = $1;
                $fl_drvs->{$drv_cnt}->{'nsp'}  = $2;
                $fl_drvs->{$drv_cnt}->{'slot'} = $3;
                $fl_drvs->{$drv_cnt}->{'cap'}  = $cap;
                my $fmt = $5;
                my ( $f1, $f2 ) = $fmt =~ /(\S+)\s+\/\s+(\S+)/g;
                my $lbasperop = $f1 + $f2;
                $lbasperop = 256 if ( $lbasperop == 520 );
                $lbasperop = 32  if ( $lbasperop == 4160 );
                $fl_drvs->{$drv_cnt}->{'fmt'}   = $fmt;
                $fl_drvs->{$drv_cnt}->{'LBAOP'} = $lbasperop;
                $fl_drvs->{$drv_cnt}->{'state'} = $6;
                $i++;
            }
        }
    }
    my @flsh_drv = keys(%$fl_drvs);
    logcomment("Contollers and namespace details ");
    foreach my $d ( keys %$fl_drvs ) {
        logcomment("Temporary Drive ID : $d \n Controller : $fl_drvs->{$d}->{'ctrl'} \n Namespace : $fl_drvs->{$d}->{'nsp'}\n Slot : $fl_drvs->{$d}->{'slot'} \n Capacity : $fl_drvs->{$d}->{'cap'} \n Format : $fl_drvs->{$d}->{'fmt'}\n LBAs/op : $fl_drvs->{$d}->{'LBAOP'} \n State: $fl_drvs->{$d}->{'state'}");
    }

    #STEP 5
    nvme_eye_catcher( Test => $Test, string => "STEP 5 of 7 : Set nvme state to offline" );
    logcomment("Get nvme state ");
    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme state get" );
    if ( $out =~ /Online/i ) {
        logcomment("NVMe state is Online , Offline nvme State");
        $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme state set offline" );
    } else {
        logcomment("NVMe state is offline ");
    }
    logcomment("Re-check nvme state to ");
    my $re_out = $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme state get" );
    if ( $re_out =~ /offline/i ) {
        logcomment("Gemini/NVMe state set to offline");
    } else {
        $test_status = 1;
        logresult( "INFO", msg => $test_status );
        logresult( "FAIL", "NVMe State cannot set to offline." );
        return $TCD::FAIL;
    }

    #STEP 4
    nvme_eye_catcher( Test => $Test, string => "STEP 6 of 7 : Start Pre-Condition on SCM drive and in parallel get stats" );
    my @pre_drv = keys(%$fl_drvs);
    logcomment("$FILER_C : Pre-condition @pre_drv drives");
    logcomment("$FILER_C : user selected pre condition SCM drive : $USR_PRE_COND");
    logcomment("$FILER_C : User selected runtime : $RUN_TIME");
    foreach my $sr_dr (@pre_drv) {
        logcomment("$FILER_C : FLASH DRIVE serial number : $sr_dr");
        my $ctrlid    = $fl_drvs->{$sr_dr}->{'ctrl'};
        my $namespace = $fl_drvs->{$sr_dr}->{'nsp'};
        my $lbas      = $fl_drvs->{$sr_dr}->{'LBAOP'};
        logcomment("$FILER_C : Contoller ID : $ctrlid and Namespace : $namespace");
        logcomment("$FILER_C : LBASs/op : $lbas");
        my $stat = scm_pre_condit_drv( Node => $Node, pre_condition_opt => $USR_PRE_COND, ctrl => $ctrlid, namesp => $namespace, lbas => $lbas );
        sleep 2;
        logcomment("Precondition Started on -controller $ctrlid -namespace $namespace with -lbas-per-op $lbas");
        my $interval = 60;
        exe_stat_15min( $Node, $ctrlid, $namespace, $interval, $sr_dr );
        logcomment("Precondition Completed on -controller $ctrlid -namespace $namespace with -lbas-per-op $lbas");
        logcomment("Proceed with next controller");
    }

=head
    logcomment("DEBUG : Precondition Started on all available controller and namespace");
    my $interval = 60;
    sleep 3;
    logcomment("Execute nvme stress stats every $interval seconds ");
    my @subtests_pre;
    foreach my $sr_dr (@pre_drv) {
        my $ctrlid    = $fl_drvs->{$sr_dr}->{'ctrl'};
        my $namespace = $fl_drvs->{$sr_dr}->{'nsp'};
        push( @subtests_pre, subtest( \&exe_stat_15min, -runid, "scm_pre_cond_stats_$sr_dr" . "_$ctrlid" . "_$namespace" . "_" . "$FILER_C", -bg, "--", $Node, $ctrlid, $namespace, $interval, $sr_dr ) );
        logcomment("Starting stats on -controller $ctrlid -namespace $namespace");
        sleep 1;
    }
    Subtest::wait_finish( subtest => [@subtests_pre] );
=cut 

    logcomment("$FILER_C : Pre-condition completed on all flash cache drives");

    #STEP 7
    nvme_eye_catcher( Test => $Test, string => "STEP 7 of 7 : Start Random Write to each controller in parallel and get stats" );
    my @pre_drv = keys(%$fl_drvs);
    my @subtests_s7;
    foreach my $sr_dr (@pre_drv) {
        my $interval  = 60;
        my $ctrlid    = $fl_drvs->{$sr_dr}->{'ctrl'};
        my $namespace = $fl_drvs->{$sr_dr}->{'nsp'};
        my $lbas      = $fl_drvs->{$sr_dr}->{'LBAOP'};
        logcomment("$FILER_C : Contoller ID : $ctrlid and Namespace : $namespace");
        logcomment("$FILER_C : LBASs/op : $lbas");

		my $cmd = "system node nvme stress start -controller $ctrlid -namespace $namespace -threads 16 -ops-per-thread 32 -rd-percent 0 -rd-verify false -lbas-per-op $lbas -runtime 21600 -rd-op seq -wr-op rand -wr-skip-freq 0 -wr-skip-amt 0";
		
        logcomment("$FILER_C : Execute stress command : \n $cmd");
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        $Api_Set_Obj->execute_raw_command( 'command' => "$cmd" );
        logcomment("$FILER_C : Started Random Write on -controller $ctrlid -namespace $namespace with -lbas-per-op $lbas");
        sleep 2;
        logcomment("$FILER_C : Execute stats command every $interval seconds ");
        exe_stat_15min( $Node, $ctrlid, $namespace, $interval, $sr_dr );
        logcomment("$FILER_C : Random Write on -controller $ctrlid -namespace $namespace with -lbas-per-op $lbas COMPLETED");
    }

=head
    logcomment("$FILER_C : Execute stats command every $interval seconds ");
    foreach my $sr_dr (@pre_drv) {
        my $ctrlid    = $fl_drvs->{$sr_dr}->{'ctrl'};
        my $namespace = $fl_drvs->{$sr_dr}->{'nsp'};
        push( @subtests_s7, subtest( \&exe_stat_15min, -runid, "scm_seq_rd_stats_$sr_dr" . "_$ctrlid" . "_$namespace" . "_" . "$FILER_C", -bg, "--", $Node, $ctrlid, $namespace, $interval, $sr_dr ) );
        logcomment("Starting stats on -controller $ctrlid -namespace $namespace");
        sleep 1;
		Subtest::wait_finish( subtest => [@subtests_s7] );
    }
=cut

    #STEP 8
    nvme_eye_catcher( Test => $Test, string => "STEP 8 of 8 : Set nvme state to online" );
    logcomment("Get nvme state ");
    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme state get" );
    if ( $out =~ /Offline/i ) {
        logcomment("NVMe state is Offline , Online nvme State");
        $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme state set online" );
    } else {
        logcomment("NVMe state is online ");
    }
    logcomment("Re-check nvme state to ");
    my $re_out = $Api_Set_Obj->execute_raw_command( 'command' => "system node nvme state get" );
    if ( $re_out =~ /online/i ) {
        logcomment("Gemini/NVMe state set to online");
    } else {
        $test_status = 1;
        logresult( "INFO", msg => $test_status );
        logresult( "FAIL", "NVMe State is  offline." );
        return $TCD::FAIL;
    }

    logcomment("$FILER_C : Completed Random Write command on all available controllers ");
    return logresult( "INFO", msg => $test_status );
}

# End of sub_exe
#####################################################################
# Cleanup - Post Process
#####################################################################
sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;
    scm_pre_test(
        node_present => $node_ref,
        Test         => $Test,
        filer_mode   => $Mode
    );

    return $TCD::PASS;
}

sub exe_stat_15min {
    my ( $Node, $ctrlid, $namespace, $interval, $sr_dr ) = @_;
    my $FILER_C  = $Node->name();
    my $time_out = 9000000099999;
    logcomment("$FILER_C : Execute Stats Command every $interval seconds on controller $ctrlid, namespace $namespace");
    logcomment("Execute stats command every $interval seconds");
    logcomment("$FILER_C : Drive : $sr_dr");
    my $stop = 0;

    my $SSH_FILER_A_1 = $Node->get_7m_or_nodescope_apiset( connid => 'console' );

    #my $SSH_FILER_A_1 = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
    $SSH_FILER_A_1->set_timeout( "connectrec-timeout" => $time_out );

    #$Node->refresh_command_interface();
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $SSH_FILER_A_1->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    sleep 5;
    $SSH_FILER_A_1->execute_raw_command( 'command' => "\013" );
    my $runtme;
    my $s_runtime;
    my @runtime;

    while (1) {
        logcomment("Executing system node nvme stress stats :");

        my $output = $SSH_FILER_A_1->execute_raw_command( command => "system node nvme stress stats -controller $ctrlid -namespace $namespace" );
        foreach my $line ( split( /\n/, $output ) ) {
            if ( $line =~ /Runtime:\s+(\S+)/ ) {
                $runtme = $1;
                logcomment("$FILER_C : -controller $ctrlid -namespace $namespace : Runtime : $runtme");
                push( @runtime, $runtme );
            }
            if ( $line =~ /No test was run on this namespace/ ) {
                logcomment("No test was run on this namespace");
                $stop = 1;
            }
        }
        $SSH_FILER_A_1->execute_raw_command( command => "\013" );
        $SSH_FILER_A_1->execute_raw_command( command => "\013" );
        logcomment("Wait for $interval seconds ");
        sleep($interval);
        logcomment("Check for nvme stress stats again on -controller $ctrlid -namespace $namespace");

        #$Node->refresh_command_interface();
        #my $SSH_FILER_A_1 = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $SSH_FILER_A_1->execute_raw_command( command   => "\013" );
        $SSH_FILER_A_1->execute_raw_command( 'command' => " " );
        my $s_output = $SSH_FILER_A_1->execute_raw_command( command => "system node nvme stress stats -controller $ctrlid -namespace $namespace" );
        foreach my $line ( split( /\n/, $s_output ) ) {
            if ( $line =~ /Runtime:\s+(\S+)/ ) {
                $s_runtime = $1;
                push( @runtime, $s_runtime );
                logcomment("$FILER_C : -controller $ctrlid -namespace $namespace : Runtime : $s_runtime");
            }
            if ( $line =~ /No test was run on this namespace/ ) {
                logcomment("No test was run on this namespace");
                $stop = 1;
            }
        }
        logcomment("DEBUG :: Controller-$ctrlid :: NameSpace-$namespace :: Runtime Start - $runtme and Current - $s_runtime");
        logcomment("DEBUG : TIME : @runtime");
        if ( $runtme == $s_runtime ) {
            logcomment("Runtime Appears to be constant, which indicates stress command has been completed");
            logcomment("Issuing stress stop command on -controller $ctrlid -namespace $namespace ");
            my $API_OBJ = $SSH_FILER_A_1;
            $API_OBJ->execute_raw_command( 'command' => " " );
            $API_OBJ->execute_raw_command( 'command' => "system node nvme stress stop -controller $ctrlid -namespace $namespace" );
            my ( $error, $data_ver_err, $apx_ver_err );
            foreach my $line ( split( /\n/, $s_output ) ) {
                $line =~ s/\s+//g;
                if ( $line =~ /^Errors:/ ) {
                    ($error) = $line =~ /Errors:(\S+)/;
                    logcomment("Error is : $error\n");
                } elsif ( $line =~ /DataVerifyErrors/ ) {
                    ($data_ver_err) = $line =~ /DataVerifyErrors:(\S+)/;
                    logcomment("Data verify errors : $data_ver_err\n");
                } elsif ( $line =~ /ApxVerifyErrors/ ) {
                    ($apx_ver_err) = $line =~ /ApxVerifyErrors:(\S+)/;
                    logcomment("Apx verify errors : $apx_ver_err\n");
                }
            }
            if ( $error == 1 ) {
                logcomment("Error Found !!!! ");
                logcomment("**FAIL** : Errors found in nvme stress stats output");
            }
            $stop = 2;
        } else {
            logcomment("$FILER_C : NVMe Stress command is still in progress on -controller $ctrlid -namespace $namespace ");
        }
        if ( $stop == 1 ) {
            logcomment("Errors / No test was run on this namespace, Exiting stats command");
            last;
        } elsif ( $stop == 2 ) {
            logcomment("$FILER_C : Stats Command completed on -controller $ctrlid -namespace $namespace");
            last;
        } else {
            logcomment("$FILER_C :Stress Command is in progess on -controller $ctrlid -namespace $namespace");
        }
        sleep 2;
    }

}

sub exe_command {
    my ( $Node, $command, $conn_obj ) = @_;
    my $FILER_C  = $Node->name();
    my $time_out = 90000000;
    logcomment("$FILER_C : Execute command : $command");
    $Node->refresh_command_interface();
    $conn_obj->set_timeout( "connectrec-timeout" => $time_out );
    my $output = $conn_obj->execute_raw_command( command => "$command" );
}