#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/TEST_SCRIPTS/SSD/NADQ02_Stress_Test_50_50.thpl#0 $
## All rights reserved.
##
## @summary Mode
##  Stress Test 50 50
##
## @description
##  Evaluates the first level performance at system level with homogeneous aggrs
##
## @Test Mode
##  File System Creation Mode
##
## @Test bed setup
##  FC   :Dual Path, Cluster Fabric
##  SAS  :Dual Path
##
## @usage
##  The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  There is no dependency
##
## @steps
##  The test will execute steps mentioned below:
##   1. Perform Sysconfig -v to locate the shelf.diskid that matches ID_test_drive
##   2. Execute disk show -owner local to see the topology of the disks and ownership of local owner on which test will be run
##   3. Create an aggregate with the disks identified using step #1 and #2
##   4. Create a vserver
##   5. Create a volume of user defined size.
##   6. Run user defined number of filersio threads
##   7. Start statisics command for the disks that have been selected in step #2
##   8. Statistics show -object disk -instance on selected disks
##   9. Stop the statiscs statistics
##   10.Collect statit information
##   11.End the statit collection after 90 seconds
##   12.Stop the filersio command
##   13.Delete the aggregate, vserver, volume created for the test
##
## @param FILER        - optional for cluster setup
##                     - required for Dual path setup      Name of filer to be used
## @param FILERA       - required for cluster setup
##                     - optional for Dual path setup      Name of filer to be used
## @param FILERB       - required for cluster setup
##                     - optional for Dual path setup      Name of the partner node
## @param TEST_CONFIG  - optional Type of the setup, C for cluster, D for dual path (default)
## @param TEST_SETUP   - optional Default value set to 'FC'
## @param FILER_CONN   - optional Type of connection, default is set to 'console' for this test
## @param MODE         - optional Filer mode maint/normal, default set to 'maint' for this test
## @param FILER_PROMPT - optional prompt of the filer
## @param LOGDIR       - optional This is required to generate 'END-LOG'. Default set to main log directory.
## @param EOE          - optional 'default' is set to default value
## @param BOOT_MODE    - optional Boot mode for filer , default set to '1' for this test
## @param BOOT_TYPE    - optional Boot type, default set to 'A' (automatic) for this test
## @param SSD          - optional Setup is SSD or not, default set to 'no'
## @param EMAIL        -  optional Default value set to 'Y'
## @param EMAIL_TO     - required for setting to address sending email to recipient
## @param EMAIL_FROM   - required for setting  from address for sending email
## @param FILER_TYPE   - Required to set DataONTAP version    DataONTAP version is either IC or BR
## @param FILERSIO_READ_THREADS - optional to start filersio on the volume default 5
## @param DRIVE_ID     - required to select the disks for the test
## @param DRIVES_NUMBR - required to create the aggregate
## @param VOL_SIZE     - required to create the volume
##
## @status  Automated
## @author  aditis@netapp.com
## @burt    1091013 and 1106863
## @change  khosur : For Burt-1155455
##
#############################################################################
#############################################################################
# Compiler directives.
use strict;
use Storage::Common_Lib;
use Storage::NVMe_Common_Lib;
use NACL::C::Statistics;
use TCD;
use POSIX;
use boot;
use NATE::BaseException qw(:try);
use Date::Calc qw(Add_Delta_Days);
use Date::Parse;
use List::Util qw[min max];

#use Date::Format;

#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $ARMADILLO
  $FILERSIO_READ_THREADS
  $DRIVE_ID
  $VOL_SIZE
  $DRIVES_NUMBR
  $DRIVE_TYPE
  $DAYS
  $AUTO_PUSH
  $MODEL_DRIVE_ID
  $ENV_FIL_LOC
);
my $params = NATE::ParamSet->new( global => 1 );
$TEST_CONFIG           = $params->get( 'TEST_CONFIG',           default => 'D' );
$TEST_WAIT_TIME        = $params->get( 'TEST_WAIT_TIME',        default => '3' );
$LOGDIR                = $params->get( 'LOGDIR',                default => undef );
$EOE                   = $params->get( 'EOE',                   default => 'default' );
$EMAIL                 = $params->get( 'EMAIL',                 default => 'y' );
$MAIL_TO               = $params->get( 'MAIL_TO',               default => 'Email to' );
$MAIL_FROM             = $params->get( 'MAIL_FROM',             default => 'Email from' );
$FILER_PROMPT          = $params->get( 'FILER_PROMPT',          default => '\*>' );
$TEST_SETUP            = $params->get( 'TEST_SETUP',            default => 'SAS' );
$BOOT_MODE             = $params->get( 'BOOT_MODE',             default => '1' );
$BOOT_TYPE             = $params->get( 'BOOT_TYPE',             default => 'A' );
$SSD                   = $params->get( 'SSD',                   default => 'no' );
$ARMADILLO             = $params->get( 'ARMADILLO',             default => '2' );
$FILER_TYPE            = $params->get( 'FILER_TYPE',            default => 'BR' );
$FILERSIO_READ_THREADS = $params->get( 'FILERSIO_READ_THREADS', default => '5' );
$DRIVE_ID              = $params->get( 'DRIVE_ID',              default => undef );
$MODEL_DRIVE_ID        = $params->get( 'MODEL_DRIVE_ID',        default => undef );
$VOL_SIZE              = $params->get( 'VOL_SIZE',              default => '250' );
$DRIVES_NUMBR          = $params->get( 'DRIVES_NUMBR',          default => '5' );
$DRIVE_TYPE            = $params->get( 'DRIVE_TYPE',            default => undef );
$DAYS                  = $params->get( 'NO_OF_DAYS',            default => undef );
$AUTO_PUSH             = $params->get( 'AUTO_PUSH',             default => '0' );
$ENV_FIL_LOC           = $params->get( 'ENV_FIL_LOC',           default => '/home/nvme/NDATE/FILER_INFO/NVMe_ENV_VAR_166_167' );
my $num_of_days;

$DRIVE_ID = $MODEL_DRIVE_ID;

if ( $DAYS =~ /Infinite/ ) {
    $num_of_days = 10;
    $DAYS        = 10;
} else {
    $num_of_days = $DAYS;
}
################
# Testcase name
################
my $TC_name;
$TC_name = "NADQ02_NVMe_Stress_Test_50_50";

########################################
### Initialization/declaration
#########################################
my $test_status = 0;
my %nodes_filer = {};
my @Nodes;
my @nodes_use = ();
my $filer_names;
my $Mode;
my ( $email_subject, $email_body );
$EOE = 1;
my $timeout;
my @Node_use;
my %fil_owner     = {};
my %Fil_aggr_disk = {};
my %Fil_dis_use   = {};
my %Filr_dsk      = {};
my %autoassign    = {};
my $partition;
my $Node_use;
my $norun_flag = 0;
my $day        = 1;
 my @DRVS_2_USE = ();
#########################################
# Testcase available for execution
#########################################
## Pre-test processes
#########################################

my @Testcases = ( NVMe_Stress_Test_50_50 => "Stress_Test_50_50 - CLI" );

&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status   = 1;
        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }
    exit(0);
} ## end sub main

########## INIT  ###################################################
# init automatically called before executing tests
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    logcomment("DRIVE TYPE is $DRIVE_TYPE");
    logcomment("MODEL ID which user has selected : $DRIVE_ID");
    logcomment("Number of DRIVES which user has selected for aggregate creation : $DRIVES_NUMBR");
    logcomment( "Volume size which user has selected : $VOL_SIZE" . 'g' );
    logcomment("Number of filersio read threads which user has selected : $FILERSIO_READ_THREADS");
    logcomment("Number of days which user has selected : $DAYS");

    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes =
      map  { @{$_} }
      grep { (/\S/) } values(%nodes_filer);      # Contains Node objects used for test execution.
    grep { logcomment( "Filers passed to the script are " . $_->name() ) } @Nodes;
    logcomment("Checking for execution mode");
    $Mode = $Nodes[0]->mode();

    version_test( node_present => \@Nodes, tc_name => $TC_name );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );
    return $TCD::PASS;
}

########## SETUP  ###################################################
# Setup automatically called before executing tests
#####################################################################
sub setup() {
    $Test->description("Setup the environment for the test exectuion ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref          = \@Nodes;
    my $filer_env_details = {};

    # Checking whether the setup is partioned or not
    logcomment("Checking whether the system is PARTITIONED or UNPARTITIONED");
    my $FILER_C = $Nodes[0]->name();
    my $disk_hash = disk_show_v( Node => $Nodes[0] );
    foreach ( keys %{$disk_hash} ) {
        if ( $_ =~ /P\d+$/ ) {
            $partition = 1;
            last;
        }
    }
    if ( $DRIVE_TYPE =~ /SSD$/i ) {
        if ( $partition == 1 ) {
            logcomment("SYSTEM is PARTITIONED");
        } else {
            $norun_flag = 1;
            logcomment("**NORUN**: SYSTEM IS UNPARTITIONED.As drive type is $DRIVE_TYPE, system should be partioned.");
            return $TCD::FAIL;
        }
    }


    #####################################################################
    # PRE -CONDITION ALL DRIVES
    #####################################################################
    nvme_eye_catcher( Test => $Test, string => "PRE-CONDITION : START");
    nvme_pre_condition_drvs ( Nodes  => \@Nodes );
    nvme_eye_catcher( Test => $Test, string => "PRE-CONDITION : COMPLETED");
    #####################################################################
    # Unpartiton and re-assign all drive
    #####################################################################
    remove_partition(Noderef => \@Nodes);
    logcomment("Partition removed, Re-assign drive");
    nvme_drv_assign(Nodes  => \@Nodes, force  => 1, assign =>'yes');
    logcomment("Drive assigne to both nodes");


    # REINITILIZING FILERS BEFORE STARTING TEST , TO WIPEOUT METADATA
    logcomment("REINITILIZING FILERS BEFORE STARTING TEST , TO WIPEOUT METADATA");

    #READ ENV FILE
    logcomment("Read ENV variables");
    my $Home = $ENV{HOME};
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        $ENV_FIL_LOC = "$Home/NDATE/FILER_INFO/$FILER_C";
        open( ENV_FL, "$ENV_FIL_LOC" ) || die "Can not open file $ENV_FIL_LOC";
        my @env_data = <ENV_FL>;
        logcomment("Get path for filer ");
        my ( $env, $nb );
        foreach my $line (@env_data) {
            chomp($line);
            if ( $line =~ /FILER=(\S+)/ ) {
                print "Filer : $1\n";
                $FILER_C = $1;
                $filer_env_details->{$FILER_C} = { 'filer' => $1 };
            }
            if ( $line =~ /PATH=(\S+)/ ) {
                $env = $1;
                print "ENV : $env\n";
                $filer_env_details->{$FILER_C} = { 'env_p' => $env };
            }
            if ( $line =~ /NETBOOT_IMG=(\S+)/ ) {
                $nb = $1;
                print "NB : $nb\n";
            }
        }
    }
    logcomment( "Dumper " . Dumper($filer_env_details) );

    #Clean_config
    my @sub_clean;

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $env_data;
        foreach my $fil ( keys %$filer_env_details ) {
            logcomment("Get Env path to be set for filer $fil");
            if ( $fil =~ /$FILER_C/ ) {
                $env_data = $filer_env_details->{$fil}->{'env_p'};
                logcomment("Env variable to be used :  $env_data");
            }
        }
        push( @sub_clean, subtest( \&nvme_clean_config, -runid, "clean_config_node_$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) );
    }
    Subtest::wait_finish( subtest => [@sub_clean] );
    logcomment("Re-initilization of filers compelts, Proceed with cluster Create and Join");
    create_join_cluster( node_present => [@Nodes], nvme_setup => "yes" );
    logcomment("NVMe Setup compelted");
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );

    logcomment("Drive Type : $DRIVE_TYPE");
    logcomment("User Selected DRIVE ID : $DRIVE_ID\n Number of Drives Used for Aggreagate creation : $DRIVES_NUMBR\n Volume Size : $VOL_SIZE GB\n Number of Filersio Threads : $FILERSIO_READ_THREADS");

    #-------------------------------------------------------------------------
    #checking for non-root aggr , if present deleting and waiting till zeroing.
    #-------------------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        $Node->refresh_command_interface();
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 );
        $Api_Set_Obj->execute_raw_command(
            'command'                => "set test",
            'connectrec-match_table' => $prompts_answers
        );
        logcomment("DEBUG: $FILER_C : Checking for the Non Root aggregate , if present deleting the aggregate ");
        aggregate_volume_delete( Node => $Node );
        vserver_delete( Node => $Node );
        logcomment("Set disk autoassign to OFF");
        my $output = $Api_Set_Obj->execute_raw_command( 'command' => "storage disk option show -fields autoassign -node $FILER_C" );
        ( $autoassign{$FILER_C} ) = $output =~ /$FILER_C\s*(\S+)/;

        if ( $autoassign{$FILER_C} =~ /on/ ) {
            logcomment("$FILER_C : Set disk autoassign to OFF");
            $Api_Set_Obj->execute_raw_command( 'command' => "storage disk option modify -node $FILER_C  -autoassign off" );
        }
    }
    return $TCD::PASS;
}
########## TEST 1 ###################################################
#NADQ02_Homo_Perf_Test
#####################################################################
sub NVMe_Stress_Test_50_50 {
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filers are $filer_names" );
    $Test->description("System Level Performance for Homogenous Array");

    #####TEST VARIABLES#####
    my %step1_arr     = ();
    my %step1_arr_fin = ();
    my @disk_used     = ();
    my $FILERB;
    my $loop      = 0;
    my $loop1     = 0;
    my $flag_fail = 0;
    my $check     = 2;
    #######################

## TEST STARTS
###########################################################
## STEP 1 - Perform sysconfig -v to locate the shelf.diskid that matches ID_test_driv : $DRIVE_ID
## Pass/Fail criteria: N/A
###########################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 1 of 12: $FILER_C : Perform sysconfig -v to locate the shelf.diskid that matches ID_test_driv : $DRIVE_ID"
        );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 );
        $Api_Set_Obj->execute_raw_command(
            'command'                => "set test",
            'connectrec-match_table' => $prompts_answers
        );
        my $output = $Api_Set_Obj->execute_command( 'command' => "sysconfig -v" );
        my @arr1 = split( /\n/, $output );
        my @arr_disk = ();

         nvme_eye_catcher( Test => $Test, string => "STEP 2 of 12: $FILER_C : Check for Spare Drives available and sufficient to create aggregate" );
        @DRVS_2_USE = nvme_get_drv_by_user_sel_mod_id( API_con => $Api_Set_Obj, Node => $Node, model => $DRIVE_ID, local_name => 'no', Filer => $FILER_C );
        $step1_arr{$FILER_C} = [@DRVS_2_USE];
        my $user_sele_drv_count = @DRVS_2_USE;
        logcomment("$FILER_C : Drives from selection : @DRVS_2_USE");
        logcomment( "Disks which matches the DRIVE_ID: $DRIVE_ID given by user are " . scalar @DRVS_2_USE . " and the drives are @DRVS_2_USE" );
        logcomment("Total Number of drives selected by user : $DRIVES_NUMBR");

        if ( $user_sele_drv_count < $DRIVES_NUMBR ) {
            $flag_fail == 1;
        }
    }
    if ( $flag_fail == 1 ) {
        logcomment("Not enough drives present. $DRIVES_NUMBR drives needed for aggreagte creation which belongs to MODEL $DRIVE_ID");
        return $TCD::FAIL;
    }
    my @nodes = ();
    my @subtests;
    my $diskloop        = 0;
    my @zeroed_drive    = ();
    my @drives_use      = ();
    my @drives_use_finl = ();
 
     my @zeroed_drive1 = shuffle_array( \@DRVS_2_USE );
     @zeroed_drive1 = unique(@zeroed_drive1);
     @drives_use_finl =  splice(@zeroed_drive1,0,$DRIVES_NUMBR);
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("$FILER_C: Using random drives : @drives_use_finl to create aggregate");
        push( @subtests, subtest( \&Stress_Test_50_50_sub, -runid, "Stress_Test_50_50_$FILER_C", -bg, "--", $Node, \@drives_use_finl, $DRIVE_ID, $DRIVES_NUMBR, $VOL_SIZE, $FILERSIO_READ_THREADS, $DAYS ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $return = status_return( subtest_ref => [@subtests] );
###########################################################################################
## Test log/results
###########################################################################################
    logcomment("Test $TC_name is completed.");
    if ( $return == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub Stress_Test_50_50_sub {
    $Test->description("Homogeneous Aggregate Performance Test ");
    my $Node                  = shift(@_);
    my $arr_ref               = shift(@_);
    my @drives_use            = @$arr_ref;
    my $DRIVE_ID              = shift(@_);
    my $DRIVES_NUMBR          = shift(@_);
    my $VOL_SIZE              = shift(@_);
    my $FILERSIO_READ_THREADS = shift(@_);
    my $DAYS                  = shift(@_);
    my $wr_volume_obj;
    my $FILER_C = $Node->name();
    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
    $Api_Set_Obj->set_timeout( "connectrec-timeout" => 720000000000000 );
    #################################################################################
    ## STEP 3 - Create an aggregate with the disks identified using step #1 and #2 ##
    ## Pass/Fail criteria: N/A						       ##
    #################################################################################
    logcomment("DEBUG : $FILER_C : Executing pre test");
    nvme_pre_test(
        node_present    => [$Node],
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );
    nvme_eye_catcher(
        Test   => $Test,
        string => "STEP 3 of 12: $FILER_C : Create an aggregate with the disks identified using step #1 and #2"
    );
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command(
        'command'                => "set test",
        'connectrec-match_table' => $prompts_answers
    );
    logcomment("As the number of drives which user has selected is $DRIVES_NUMBR, $DRIVES_NUMBR drives will be used for aggregate creation");
    my $aggregate_name = "stress_test_" . "$FILER_C";
    $aggregate_name =~ s/\-/_/g if ( $aggregate_name =~ /\-/ );
    logcomment("$FILER_C : Creating aggregate $aggregate_name... ");
    logcomment("$FILER_C : Drive used to create aggregate : @drives_use");
    my $job;
    my $aggr_create = nvme_create_aggregate( Node => $Node, aggr_name => $aggregate_name, drive_to_use => \@drives_use );
    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    my $output = $Api_Set_Obj->execute_raw_command(
        'command'                => "aggr show $aggregate_name -fields availsize",
        'connectrec-match_table' => $prompts_answers1
    );

    if ( $output =~ /$aggregate_name/i ) {
        logcomment("$FILER_C : Aggregate $aggregate_name created on Node $FILER_C");
    } else {
        logcomment("**FAIL** : $FILER_C : Aggregate $aggregate_name is NOT created, so exiting from test");
        return logresult( "INFO", msg => 1 );
    }

    ###########################################################
    ## STEP 4 - Create Vserver
    ## Pass/Fail criteria: N/A
    ###########################################################
    nvme_eye_catcher(
        Test   => $Test,
        string => "STEP 4 of 12: $FILER_C : Create vserver"
    );
    my $vserver_name = "vser_test_" . "$FILER_C";
    $vserver_name =~ s/\-/_/g if ( $vserver_name =~ /\-/ );
    logcomment("$FILER_C : Creating vserver $vserver_name...");
    my $root = "vol0";
    my $ns   = "file";
    my $sec  = "unix";
    logcomment("Create Vserver $vserver_name with root volume $root on filer $FILER_C");
    my $prompts_answers = [ ".*Do you want to continue??.*" => 'y' ];
    my $vser_o = $Api_Set_Obj->execute_raw_command( 'command' => "vserver create -vserver $vserver_name -rootvolume $root", 'connectrec-match_table' => $prompts_answers );

    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    my $output = $Api_Set_Obj->execute_raw_command(
        'command'                => "vserver show",
        'connectrec-match_table' => $prompts_answers1
    );

    if ( $output =~ /$vserver_name/i ) {
        logcomment("$FILER_C : vserver $vserver_name created on Node $FILER_C");
    } else {
        logcomment("**FAIL** : $FILER_C : VSERVER $vserver_name HAS NOT BEEN CREATED PROPERLY, so exiting the test");
        return logresult( "INFO", msg => 1 );
    }
    $Api_Set_Obj->execute_raw_command( 'command' => "vserver modify -vserver $vserver_name -volume-delete-retention-hours 0" );

    logcomment("Capturing Start time");
    my $event_start_time;
    my $event_end_time;
    my $strt_time = $Api_Set_Obj->execute_raw_command( 'command' => "cluster date show -fields utc-date -node $FILER_C" );
    my $n         = 0;
    my @lines     = split( /\n/, $strt_time );
    while ( $n <= $#lines ) {
        last if ( $lines[$n] =~ /^-/ );
        $n++;
    }
    $n++;
    foreach my $var ( @lines[ $n .. $#lines ] ) {
        $var =~ s/^\s+|\s+$//gi;
        if ( $var =~ /$FILER_C\s*(\S+\s*\S+)/ ) {
            $event_start_time = $1;
        }
    }
    logcomment("$FILER_C: Start time is :: $event_start_time");
    ###########################################################
    ## STEP 5 - Create a volume per the size defined by the user (vol_size)
    ## Pass/Fail criteria: N/A
    ###########################################################
    nvme_eye_catcher(
        Test   => $Test,
        string => "STEP 5 of 12: $FILER_C : Create 2 volume as per the size defined by the user vol_size"
    );
    my $aggr_size;
    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    my $output = $Api_Set_Obj->execute_raw_command(
        'command'                => "aggr show $aggregate_name -fields availsize",
        'connectrec-match_table' => $prompts_answers1
    );

    foreach my $line ( split( /\n/, $output ) ) {
        if ( $line =~ /^$aggregate_name\s+(\S+)[GT]B/i ) {
            $aggr_size = $1;
            $aggr_size = $aggr_size * 1024
              if ( $line =~ /^$aggregate_name\s+(\S+)TB/i );
            my ($val) = readable_disk_size( GB => $aggr_size );
            logcomment("$FILER_C : Size of created aggregate  --> $val");
            logcomment("Two volumes has to be created and Size of each volume which user has given is $VOL_SIZE i.e., $VOL_SIZE + $VOL_SIZE");
            if ( $aggr_size <= $VOL_SIZE * 2 ) {
                logcomment("Size of Aggregate $aggregate_name is less than the size of the volumes which user has given");
                my $vol_size = floor( $aggr_size * 0.90 );
                my ($res) = readable_disk_size( GB => $vol_size );
                $VOL_SIZE = $vol_size / 2;
                logcomment("Hence taking 90% of the aggregate size $val that will be used for creating the volume : $res");
                logcomment("Each volumewill be created of size : $VOL_SIZE");
            }
        }
    }

    ###########################################################
    # Disable volume snapshot ploicy
    ###########################################################
    nvme_eye_catcher(
        Test   => $Test,
        string => "Disable volume snapshot policy"
    );
    logcomment("disabling volume policy");
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule daily -newcount 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule weekly -newcount 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule hourly -newcount 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify -policy default -enabled false" );
    logcomment("Disabled snapshot policy");

    my $volsize_g = $VOL_SIZE . "g";
    my $vol_rd    = "vol_read_$FILER_C";
    my $vol_wr    = "vol_write_$FILER_C";
    $vol_rd =~ s/\-/_/g if ( $vol_rd =~ /\-/ );
    $vol_wr =~ s/\-/_/g if ( $vol_wr =~ /\-/ );
    my @vol_names       = ( "$vol_rd", "$vol_wr" );
    my $cmd             = "vol create -vserver $vserver_name -volume $vol_rd -aggregate $aggregate_name -size $volsize_g -state online -policy default -unix-permissions ---rwxr-xr-x";
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command(
        'command'                => "$cmd",
        'connectrec-match_table' => $prompts_answers
    );
    sleep(10);
    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    my $output = $Api_Set_Obj->execute_raw_command(
        'command'                => "vol show",
        'connectrec-match_table' => $prompts_answers1
    );

    if ( $output =~ /$vol_rd/i ) {
        my ($res) = readable_disk_size( GB => $VOL_SIZE );
        logcomment("$FILER_C : Volume $vol_rd of size $res created on Node $FILER_C");
    } else {
        logcomment("**FAIL** : $FILER_C -  $vol_wr COULD NOT BE CREATED ON NODE $FILER_C");
        return logresult( "INFO", msg => 1 );
    }

    logcomment("$FILER_C : Executing df");
    my $df_output = $Api_Set_Obj->execute_raw_command(
        'command'                => "run local df",
        'connectrec-match_table' => $prompts_answers1
    );
    my ($vol_size_df) = $df_output =~ /\/vol\/$vol_rd\/\s+\d+\s+\d+\s+(\d+)/i;    #/vol/vol_read_aa3_235/  249036800
    my ($val) = readable_disk_size( KB => $vol_size_df );
    logcomment("DEBUG : Size of volume created : $val");
    my $vol_size_df = floor( $vol_size_df / ($FILERSIO_READ_THREADS) );
    my ($val) = readable_disk_size( KB => $vol_size_df );
    $val =~ s/\s+//;
    logcomment("DEBUG : Size of each file to be created : $val");

    for ( my $i = 1 ; $i <= $FILERSIO_READ_THREADS ; $i++ ) {
        $Api_Set_Obj->execute_command( 'command' => "mkfile $val /vol/$vol_rd/f$i" );
    }
    for ( my $i = 1 ; $i <= $FILERSIO_READ_THREADS ; $i++ ) {

        my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio asyncio_active 100 -r 100 64k 0 $val -1 16 /vol/$vol_rd/f$i" );
        if ( $output =~ /failed to start/ig ) {
            logcomment("**FAIL** : $FILER_C : FILERSIO FAILED TO START");
            return logresult( "INFO", msg => 1 );
        } else {
            logcomment(" Filersio threads are started on $i instance");
        }
    }
    my $m = 0;
    my $j = 0;
    while ( $m < $DAYS ) {
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        logcomment( "$FILER_C : Loop count " . ( $j + 1 ) . "" );
        logcomment("$FILER_C : Checking available aggregate size before creating write volume");
        my $prmpts_answers1 = [ ".*to page down, .*" => ' ' ];
        my $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "aggr show $aggregate_name -fields availsize",
            'connectrec-match_table' => $prmpts_answers1
        );
        foreach my $line ( split( /\n/, $output ) ) {
            if ( $line =~ /^$aggregate_name\s+(\S+)[GT]B/i ) {
                my $aggr_size = $1;
                $aggr_size = $aggr_size * 1024
                  if ( $line =~ /^$aggregate_name\s+(\S+)TB/i );
                my ($val) = readable_disk_size( GB => $aggr_size );
                logcomment("$FILER_C : Available Size of created aggregate  --> $val");
                logcomment("One write volume has to be created and Size of write volume which user has given is $VOL_SIZE");
                if ( $aggr_size <= $VOL_SIZE ) {
                    logcomment("Available Size of Aggregate $aggregate_name is less than the size of the volume which user has given");
                    my $vol_size = floor( $aggr_size * 0.90 );
                    my ($res) = readable_disk_size( GB => $vol_size );
                    $VOL_SIZE = $vol_size;
                    logcomment("Hence taking 90% of the aggregate size $val that will be used for creating the volume : $res");
                    logcomment("Write volume will be created of size : $VOL_SIZE");
                }
            }
        }
        my $volsize_g = $VOL_SIZE . "g";
        logcomment("$FILER_C : Creating write volume $vol_wr");
        try {
            my $valo_out = $Api_Set_Obj->execute_raw_command( 'command' => "volume create -vserver $vserver_name -volume $vol_wr -aggregate $aggregate_name -size $volsize_g" );
        }
        catch NACL::APISet::Exceptions::CommandFailedException with {
            logcomment("**FATAL** : $FILER_C : CANNOT CREATE VOLUME $vol_wr on aggregate $aggregate_name");
            return logresult( "INFO", msg => 1 );
        };
        sleep(10);
        my $output = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows=0;vol show" );
        if ( $output =~ /$vol_wr/i ) {
            my ($res) = readable_disk_size( GB => $VOL_SIZE );
            logcomment("$FILER_C : Volume $vol_wr of size $res created on Node $FILER_C");
        } else {
            logcomment("**FAIL** : $FILER_C -  $vol_wr COULD NOT BE CREATED ON NODE $FILER_C");
            return logresult( "INFO", msg => 1 );
        }
        logcomment("$FILER_C : Adding SnapRestore license");
        $Api_Set_Obj->execute_raw_command( 'command' => "license add -license-code INOURWDUSAFDAAAAAAAAAAAAAAAA" );
        logcomment("$FILER_C : Creating CLEAN snapshot");
        $Api_Set_Obj->execute_raw_command( 'command' => "snapshot create -vserver $vserver_name -volume $vol_wr -snapshot CLEAN -foreground true" );
        my $output_snap = $Api_Set_Obj->execute_raw_command( 'command' => "snapshot show -vserver $vserver_name -volume $vol_wr" );
        if ( $output_snap =~ /CLEAN/i ) {
            logcomment("$FILER_C : snapshot CLEAN has been created");
        } else {
            logcomment("**FAIL** : $FILER_C - snapshot CLEAN COULD NOT BE CREATED ON NODE $FILER_C");
            return logresult( "INFO", msg => 1 );
        }
        sleep(5);
        $Api_Set_Obj->execute_raw_command( 'command' => "vol show -vserver $vserver_name -volume $vol_wr -fields percent-used" );
        logcomment("$FILER_C : Executing sysconfig -r to check drive assignment and layout of drives");
        $Api_Set_Obj->execute_raw_command( 'command' => "run local sysconfig -r" );
        ###########################################################
        ## STEP 6 - Start #user_defined number of filersio threads on the volume created.
        ## Pass/Fail criteria: N/A
        ###########################################################
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 6 of 12 : $FILER_C : Start $FILERSIO_READ_THREADS number of filersio threads on the volume ",
        );
        logcomment("$FILER_C : Executing df");
        my $df_output = $Api_Set_Obj->execute_raw_command(
            'command'                => "run local df",
            'connectrec-match_table' => $prompts_answers1
        );
        my ($vol_size_df) = $df_output =~ /\/vol\/$vol_wr\/\s+\d+\s+\d+\s+(\d+)/i;    #/vol/vol_read_aa3_235/  249036800
        my ($val) = readable_disk_size( KB => $vol_size_df );
        logcomment("DEBUG : Size of write volume created : $val");
        my $vol_size_df = floor( $vol_size_df / ($FILERSIO_READ_THREADS) );
        my ($val) = readable_disk_size( KB => $vol_size_df );
        $val =~ s/\s+//;
        logcomment("DEBUG : Size of each file to be created : $val");

        #starts $user_defined no of filersio threads on the volume created $vol_name of size $vol_size
        for ( my $i = 1 ; $i <= $FILERSIO_READ_THREADS ; $i++ ) {
            my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio asyncio_active 0 -r 0 64k 0 $val -1 16 /vol/$vol_wr/f$i -create" );
            if ( $output =~ /failed to start/ig ) {
                logcomment("**FAIL** : $FILER_C : FILERSIO FAILED TO START");
                return logresult( "INFO", msg => 1 );
            } else {
                logcomment(" Filersio threads are started on $i instance");
                next;
            }
        }
        sleep(30);
        my $filersio_stat = $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio status" );
        for ( my $i = 0 ; $i < $FILERSIO_READ_THREADS ; $i++ ) {
            if ( $filersio_stat =~ /filersio: instance $i: asyncio_active running/i ) {
                logcomment( "$FILER_C : Thread " . ( $i + 1 ) . " : filersio is running" );
            }
        }
        ###########################################################
        ## STEP 7 - Start the statistics for the disks selected in step#3
        ## Pass/Fail criteria: N/A
        ###########################################################
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 7 of 12 : $FILER_C : Start statisics command for the disks that have been selected",
        );
        my $sample_id;
        my $filr = $FILER_C;
        $filr =~ s/-/_/g;
        my $disks;
        foreach my $dsk (@drives_use) {
            $disks = $disks . $dsk . "|";
        }
        $disks =~ s/\|$//;
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Node->refresh_command_interface();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 );
        $Api_Set_Obj->execute_raw_command(
            'command'                => "set test",
            'connectrec-match_table' => $prompts_answers
        );
        my $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "set -rows=0;statistics start -object disk -sample-id sample_$filr -instance $disks",
            'connectrec-match_table' => $prompts_answers
        );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );

        if ( $output =~ /Statistics collection is being started for sample-id:\s+(\S+)/ ) {
            logcomment("$FILER_C : Statistics command for the selected disks as been started");
            $sample_id = $1;
            logcomment("DEBUG: Captured Sample -id $sample_id to Stop the Statistics");
        } else {
            logcomment("**FAIL** : $FILER_C - STATISTICS COMMAND FOR THE SELECTED DISKS COULD NOT BE STARTED");
            return logresult( "INFO", msg => 1 );
        }
        logcomment("$FILER_C : Waiting for Statistics object to start on disk");
        sleep(60);

        ###########################################################
        ## STEP 8 - Display the statistics for the disks selected in step#3
        ## Pass/Fail criteria: N/A
        ###########################################################

        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 8 of 12 : $FILER_C : Displaying the statisics for the disks that have been selected",
        );
        my $prompts_answer = [ ".*to page down, .*" => ' ' ];
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 10000 );
        my $disks;
        foreach my $dsk (@drives_use) {
            $disks = $disks . $dsk . "|";
        }
        $disks =~ s/\|$//;
        my $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "set -rows=0;statistics show -object disk -instance $disks -sample-id $sample_id",
            'connectrec-match_table' => $prompts_answer
        );
        if ( $output =~ /show failed/ ) {
            logcomment("**WARNING** : FAILED TO SHOW THE STATISTICS FOR DISKS $disks");
        }

        ###########################################################
        ## STEP 9 - Stop the statistics for the disks selected in step#3
        ## Pass/Fail criteria: N/A
        ###########################################################

        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 9 of 12 : $FILER_C : Stop the statisics for the disks that have been selected",
        );

        my $output = $Api_Set_Obj->execute_raw_command( 'command' => "statistics stop -sample-id $sample_id" );
        if ( $output =~ /(stopped\s*for\s*sample-id.*$sample_id)|("$sample_id"\s*already\s*stopped)/ig ) {
            logcomment("Statistics collection is being stopped for sample-id: $sample_id");
        } else {
            logcomment("**FAIL** : FAILED TO STOP THE STATSTICS  FOR SAMPLE-ID $sample_id");
            return logresult( "INFO", msg => 1 );
        }
        ###########################################################
        ## STEP 10 - Collect the statit info
        ## Pass/Fail criteria: N/A
        ###########################################################

        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 10 of 12 : $FILER_C : Collect the statit info",
        );
        $Api_Set_Obj->execute_command( 'command' => " statit -b " );
        sleep(360);

        ###########################################################
        ## STEP 11 - Stop the statit
        ## Pass/Fail criteria: N/A
        ###########################################################

        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 11 of 12 : $FILER_C : Stop the statit",
        );
        my $output = $Api_Set_Obj->execute_command( 'command' => "filersio status" );
        if ( $output =~ /(asyncio_active running|running)/ig ) {
            logcomment("filersio threads are running");
        }
        $output = $Api_Set_Obj->execute_command( 'command' => "statit -e" );
        sleep 5;
        logcomment("Deleting the sample-id: $sample_id before starting it again in next loop");
        $Api_Set_Obj->execute_raw_command( 'command' => "set -rows=0;statistics samples delete -sample-id $sample_id -force true" );
        ###########################################################
        ## STEP 12 - Stop the filersio instances running if volume is full more than or equal to 90%.
        ## Pass/Fail criteria: N/A
        ###########################################################
        my $flag_aggr = 0;
        my $loop = 0;
         while (1) {
           if($loop == 0){ 
           nvme_eye_catcher(
                Test   => $Test,
                string => "STEP 12 of 12 : $FILER_C : Check for write volume and stop the filersio instances if it is full.",
            );
          }else{
             nvme_eye_catcher(
                Test   => $Test,
                string => "STEP 12($loop) of 12 : $FILER_C : Check for write volume and stop the filersio instances if it is full.",
            );
          }
           $loop++; 
            logcomment("Checking for write volume $vol_wr size");
            my $vol_used;
            my $vol_flag_full = 0;
            my $output        = $Api_Set_Obj->execute_raw_command( 'command' => "vol show -vserver $vserver_name -volume $vol_wr -fields percent-used" );
            my $n             = 0;
            my @lines         = split( /\n/, $output );
            while ( $n <= $#lines ) {
                last if ( $lines[$n] =~ /^-----/ );
                $n++;
            }
            $n++;
            foreach my $var ( @lines[ $n .. $#lines ] ) {
                $var =~ s/^\s+|\s+$//gi;
                if ( $var =~ /$vserver_name\s*$vol_wr\s*(\d*%)\s*/ ) {
                    $vol_used = $1;
                }
            }
             logcomment("$FILER_C : Volume $vol_wr is $vol_used full");
            if ( $vol_used >= 90 ) {
                logcomment("$FILER_C : Volume is full $vol_used ,Stopping filersio");
                $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 );
                my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio stop\"" ); 
               logcomment("Waiting for 5 minutes");
                sleep(300);
                 $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio status\"" );
                if ( $output =~ /(filersio:\s*aborting\s*instance\s*)|/ ) {
                    logcomment("Filersio threads are stopped Successfully");
                } else {
                    logcomment("**WARNING** : Filersio threads are not  stopped ");
                }
                nvme_eye_catcher(
                    Test   => $Test,
                   string => "STEP 12a of 12: $FILER_C : Performing Sequential read",
                );

                for ( my $i = 1 ; $i <= $FILERSIO_READ_THREADS ; $i++ ) {

                    my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio asyncio_active 100 -r 0 64k 0 $val -1 16 /vol/$vol_wr/f$i" );
                    if ( $output =~ /failed to start/ig ) {
                        logcomment("**FAIL** : $FILER_C : FILERSIO FAILED TO START");
                        return logresult( "INFO", msg => 1 );
                    } else {
                        logcomment(" Filersio threads are started on $i instance");
                    }
                }
                nvme_eye_catcher(
                    Test   => $Test,
                    string => "STEP 12b of 12: $FILER_C : Check the sysstat ouput and note the minimum Read in KB/s",
                );

                my $response   = $Api_Set_Obj->sysstat( 'count' => 10, 'print-utilization' => 1 );
                my $parsed_op  = $response->get_parsed_output();
                my $nfs_stat   = $parsed_op->[0]->{'Avg'}{'NFS'};
                my @disk_reads = ();
                my $wait_time;

                for my $i ( 0 .. 9 ) {
                    my $disk_read = $parsed_op->[0]->{'stat'}->[$i]->{'Disk-read'};
                    push( @disk_reads, $disk_read );
                }
                my $min_disk_read = min @disk_reads;
                logcomment("The minimum Disk-read rate in KB/s : $min_disk_read ");
                if ( $min_disk_read == 0 ) {
                    logcomment("$FILER_C : Wait time for read filersio cannot be calculated as Minimum disk read rate is 0");
                } else {
                    nvme_eye_catcher(
                        Test   => $Test,
                        string => "STEP 12c of 12: $FILER_C : Calculating wait time for read filersio",
                    );
                    my $df_output = $Api_Set_Obj->execute_raw_command(
                        'command'                => "run local df",
                        'connectrec-match_table' => $prompts_answers1
                    );
                    my ($vol_size_df) = $df_output =~ /\/vol\/$vol_wr\/\s+(\d+)\s+\d+\s+\d+/i;    #/vol/vol_read_aa3_235/  249036800
                    logcomment("Size of write vol $vol_wr in kbytes is : $vol_size_df");
                    $wait_time = floor( $vol_size_df / $min_disk_read );
                    $wait_time = $wait_time * 2;
                    logcomment("$FILER_C : Wait time(2(total_size_write_vol/minimum_disk_read_rate)) in seconds : $wait_time");
                }
                logcomment("$FILER_C : Checking filersio status , filersio will run for $wait_time seconds");
                my $wait_time_sec = 0;
                while ( $wait_time_sec < $wait_time ) {
                    my $filersio_flag = 0;
                    my $filersio_stat = $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio status" );
                    for ( my $i = 0 ; $i < $FILERSIO_READ_THREADS ; $i++ ) {
                        if ( $filersio_stat =~ /filersio: instance $i: asyncio_active running/i ) {
                            logcomment( "$FILER_C : Thread " . ( $i + 1 ) . " : filersio is running" );
                        } else {
                            $filersio_flag = 1;
                            logcomment("$FILER_C : no filersio instances currently running");
                            last;
                        }
                    }
                    last if ($filersio_flag);
                    logcomment("Sleeping for 2 minutes before checking filersio status");
                    sleep(120);
                    $wait_time_sec = $wait_time_sec + 120;
               }
                nvme_eye_catcher(
                    Test   => $Test,
                    string => "STEP 12d of 12: $FILER_C : Stopping filersio",
                );
                logcomment("$FILER_C :Stopping filersio");
                $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio status" );
                my $output_1 = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio stop\"" );
                 logcomment("Waiting for 2 minutes");
                sleep(120);
                $output_1 = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio status\"" );
                if ( $output_1 =~ /(filersio:\s*aborting\s*instance\s*)|/ ) {
                    logcomment("Filersio threads are stopped Successfully");
                } else {
                    logcomment("**WARNING** : Filersio threads are not stopped");
                }

                logcomment("$FILER_C : Restoring the snapshot CLEAN ");
                my $prompts_answers1_snap = [ ".*Do you want to continue.*" => 'y' ];
                $Api_Set_Obj->execute_raw_command(
                    'command'                => "snapshot promote -vserver $vserver_name -volume $vol_wr -snapshot CLEAN",
                    'connectrec-match_table' => $prompts_answers1_snap
                );
                $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                sleep 2;
                $Api_Set_Obj->execute_raw_command( 'command' => "vol show -vserver $vserver_name -volume $vol_wr -fields percent-used" );
                logcomment("Deleting write volume $vol_wr");
                $Api_Set_Obj->execute_raw_command( 'command' => "volume delete -vserver $vserver_name -volume $vol_wr -foreground true -force true -disable-offline-check true" );
                $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                logcomment("Sleeping for 5 minutes before proceeding for write vol creation");
                $vol_flag_full = 1;
                sleep(300);
                goto L1;

                #last;
            } else {
                logcomment("Waiting for write volume to be filled 90%, sleeping 5 mins");
                sleep(300);
            }
          L1:
            my $end_time = $Api_Set_Obj->execute_raw_command( 'command' => "cluster date show -fields utc-date -node $FILER_C" );
            my $n        = 0;
            my @lines    = split( /\n/, $end_time );
            while ( $n <= $#lines ) {
                last if ( $lines[$n] =~ /^-/ );
                $n++;
            }
            $n++;
            my $event_end_time;
            foreach my $var ( @lines[ $n .. $#lines ] ) {
                $var =~ s/^\s+|\s+$//gi;
                if ( $var =~ /$FILER_C\s*(\S+\s*\S+)/ ) {
                    $event_end_time = $1;
                }
            }
            logcomment("$FILER_C: Current end time is :: $event_end_time");
            my $diff = str2time($event_end_time) - str2time($event_start_time);
            if ( ( $diff >= 86400 ) && ( $day <= $num_of_days ) ) {
                logcomment("$FILER_C: End time is :: $event_end_time");               
                logcomment("DEBUG: $FILER_C : Executing post test");
                nvme_post_test(
                    node_present    => [$Node],
                    Test            => $Test,
                    change_state_to => "CLI",
                    filer_mode      => $Mode
                );

                logcomment("As user has not selected the option -a=1, logs will not pushed to data_lake") if ( $AUTO_PUSH != 1 );
                my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) = localtime();
                $year = ( $year + 1900 - 2000 );
                $mon  = $mon + 1;
                if ( length($mon) == 1 )  { $mon  = "0$mon" }
                if ( length($mday) == 1 ) { $mday = "0$mday" }
                if ( length($sec) == 1 )  { $sec  = "0$sec" }
                if ( length($sec) == 0 )  { $sec  = "00" }
                if ( length($min) == 1 )  { $min  = "0$min" }
                if ( length($hour) == 1 ) { $hour = "0$hour" }
                my $logfile = "$LOGDIR" . "/Stress_Test_50_50_$FILER_C" . ".log";
                my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
                system("mkdir -p $actual_log_dir/day_$day");
                system("chmod 777 $actual_log_dir/day_$day");
                my $logfl_tmstmp = "$actual_log_dir/day_$day/" . "$year$mon$mday" . "_$hour$min$sec" . "_Stress_Test_50_50_$FILER_C" . "_D$day" . ".log";
                $day++;
                sleep(5);
                my @lines_write = ();
                open( INPUT, ">", $logfl_tmstmp )
                  or die "Could not open file $logfl_tmstmp: $!";
                open( FH, "<", "$logfile" )
                  or die "Could not open file $logfile";

                while ( my $line = <FH> ) {
                    if ( $line =~ /\[HDD\S+\]/i ) {
                        $line =~ s/\[HDD\S+\]//g;

                        # print INPUT $line;
                    } elsif ( $line =~ /\[\.]/i ) {
                        $line =~ s/\[\.]//;
                    }
                    print INPUT $line;

                }
                close(INPUT);
                close(FH);
                if ( $AUTO_PUSH == 1 ) {
                    push_data_lake( log_file => $logfl_tmstmp );
                } else {
                    logcomment("As user has not selected the option -a=1, logs will not pushed to data_lake");
                }
                system("cat /dev/null > $logfile");
                my $strt_time = $Api_Set_Obj->execute_raw_command( 'command' => "cluster date show -fields utc-date -node $FILER_C" );
                my $n         = 0;
                my @lines     = split( /\n/, $strt_time );

                while ( $n <= $#lines ) {
                    last if ( $lines[$n] =~ /^-/ );
                    $n++;
                }
                $n++;

                #my $event_start_time;
                foreach my $var ( @lines[ $n .. $#lines ] ) {
                    $var =~ s/^\s+|\s+$//gi;
                    if ( $var =~ /$FILER_C\s*(\S+\s*\S+)/ ) {
                        $event_start_time = $1;
                    }
                }
                logcomment("$FILER_C: Start time is :: $event_start_time");
                $m++;
                if ( $m != $DAYS ) {
                    logcomment("DEBUG : $FILER_C : Executing pre test");
                    nvme_pre_test(
                       node_present    => [$Node],
                       Test            => $Test,
                       change_state_to => "CLI",
                       filer_mode      => $Mode
                    );
 
                }

                #   $m++;
            }
            if ( ( $vol_flag_full == 1 ) || ( $m == $DAYS ) ) {
                last;
            }
        }
        $j++;
        last if ( $m == $DAYS );
    }

}

#####################################################################
## Cleanup - Post Process
#####################################################################
sub cleanup() {
    logcomment("$filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;
    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ##########################################################################################
    logcomment("Set disk autoassign to default");
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 );
        $Api_Set_Obj->execute_raw_command( 'command' => "storage disk option modify -node $FILER_C  -autoassign $autoassign{$FILER_C}" );
        logcomment("Deleting the non root aggregate created for this test");
        aggregate_volume_delete( Node => $Node );
        vserver_delete( Node => $Node );
    }
    if ( !$norun_flag ) {
        nvme_post_test(
            node_present    => $node_ref,
            Test            => $Test,
            change_state_to => "CLI",
            filer_mode      => $Mode
        );
    }
    return $TCD::PASS;
}