#!/usr/software/bmn/perl
## All rights reserved.
##
## @summary Mode
##  FA Byte Test
##
## @description
##  Evaluates the first level performance at system level with homogeneous aggrs
##
## @Test Mode
##  File System Creation Mode
##
## @Test bed setup
##  FC   :Dual Path, Cluster Fabric
##  SAS  :Dual Path
##
## @usage
##  The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  There is no dependency
##
## @steps
##  The test will execute steps mentioned below:
##   1. Perform Sysconfig -v to locate the shelf.diskid that matches ID_test_drive
##   2. Execute disk show -owner local to see the topology of the disks and ownership of local owner on which test will be run
##   3. Create an aggregate with the disks identified using step #1 and #2
##   4. Create a vserver
##   5. Create a volume of user defined size.
##   6. Run user defined number of filersio threads
##   7. Start statisics command for the disks that have been selected in step #2
##   8. Statistics show -object disk -instance on selected disks
##   9. Stop the statiscs statistics
##   10.Collect statit information
##   11.End the statit collection after 90 seconds
##   12.Stop the filersio command
##   13.Delete the aggregate, vserver, volume created for the test
##
##
## @status  Automated
########################################
### Library functions
##########################################
use strict;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
use TCD;
use POSIX;
use boot;
use NATE::BaseException qw(:try);
use List::Util qw(shuffle);

# Compiler directives.
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $ARMADILLO
  $FILERSIO_READ_THREADS
  $MODEL_DRIVE_ID
  $DRIVE_ID
  $VOL_SIZE
  $DRIVES_NUMBR
  $DRIVE_TYPE
  $ENV_FIL_LOC
);
my $params = NATE::ParamSet->new( global => 1 );
$TEST_CONFIG           = $params->get( 'TEST_CONFIG',                                       default => 'D' );
$TEST_WAIT_TIME        = $params->get( 'TEST_WAIT_TIME',                                    default => '3' );
$LOGDIR                = $params->get( 'LOGDIR',                                            default => undef );
$EOE                   = $params->get( 'EOE',                                               default => 'default' );
$EMAIL                 = $params->get( 'EMAIL',                                             default => 'y' );
$MAIL_TO               = $params->get( 'MAIL_TO',                                           default => 'Email to' );
$MAIL_FROM             = $params->get( 'MAIL_FROM',                                         default => 'Email from' );
$FILER_PROMPT          = $params->get( 'FILER_PROMPT',                                      default => '\*>' );
$TEST_SETUP            = $params->get( 'TEST_SETUP',                                        default => 'SAS' );
$BOOT_MODE             = $params->get( 'BOOT_MODE',                                         default => '1' );
$BOOT_TYPE             = $params->get( 'BOOT_TYPE',                                         default => 'A' );
$SSD                   = $params->get( 'SSD',                                               default => 'no' );
$ARMADILLO             = $params->get( 'ARMADILLO',                                         default => '2' );
$FILER_TYPE            = $params->get( 'FILER_TYPE',                                        default => 'BR' );
$FIRMWARE              = $params->get( 'FIRMWARE',                                          default => 'Firmware file not entered' );
$FILERSIO_READ_THREADS = $params->get( 'FILERSIO_READ_THREADS_NVMe_Perf_Rnd_Rd_FSIO', default => '10' );
$MODEL_DRIVE_ID        = $params->get( 'MODEL_DRIVE_ID_NVMe_Perf_Rnd_Rd_FSIO',        default => undef);

$DRIVE_ID              = $params->get( 'DRIVE_ID_NVMe_Perf_Rnd_Rd_FSIO',              default => undef );
$VOL_SIZE              = $params->get( 'VOL_SIZE_NVMe_Perf_Rnd_Rd_FSIO',              default => '250' );
$DRIVES_NUMBR          = $params->get( 'DRIVES_NUMBR_NVMe_Perf_Rnd_Rd_FSIO',          default => '5' );
$DRIVE_TYPE            = $params->get( 'DRIVE_TYPE',                                        default => undef );
$ENV_FIL_LOC           = $params->get( 'ENV_FIL_LOC',                                       default => '/home/nvme/NDATE/FILER_INFO/NVMe_ENV_VAR_166_167' );
################
# Testcase name
################
my $TC_name;

#old Nams - 804_NADQ02_NVMe_Homogns_Perf_Rand_Read
$TC_name = "804_NADQ02_NVMe_Perf_Rnd_Rd_FSIO";

########################################
### Initialization/declaration
#########################################
my $test_status = 0;
my %nodes_filer = {};
my @Nodes;
my $filer_names;
my $Mode;
my ( $email_subject, $email_body );
$EOE = 1;
my $timeout;
my @Node_use;
my %fil_owner     = {};
my %Fil_aggr_disk = {};
my %Fil_dis_use   = {};
my %autoassign    = {};
my $partition;
my $Node_use;
my $norun_flag   = 0;
my %usr_sel_drvs = ();
my @USER_SEL_DRV;
my $post_test = 0;
my @drive_arr;
my $FILER_C;
my $prompts_answers;

#########################################

my $test_wait_time = 12 * 3600;    #12 Hrs test
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( NVMe_Homo_Perf_Rand_Read_Test => "Homo_Perf_Test - Random Read CLI" );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    logcomment("DRIVE TYPE is $DRIVE_TYPE");
     if(defined($DRIVE_ID)){
     @drive_arr=split(',',$DRIVE_ID);
     logcomment("Drive ID's selected by the user are @drive_arr");}
    else{
    logcomment("MODEL ID which user has selected : $MODEL_DRIVE_ID");}
    logcomment("Number of DRIVES which user ahs selected for aggregate creation : $DRIVES_NUMBR");
    logcomment( "Volume size which user has selected : $VOL_SIZE" . 'g' );
    logcomment("Number of filersio read threads which user has selected : $FILERSIO_READ_THREADS");

    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node objects used for test execution.
    grep { logcomment( "Filers passed to the script are " . $_->name() ) } @Nodes;
    logcomment("Checking for execution mode");
    $Mode = $Nodes[0]->mode();

    version_test( node_present => \@Nodes, tc_name => $TC_name );
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 12 hours : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );
    return $TCD::PASS;

}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref          = \@Nodes;
    my $filer_env_details = {};
    logcomment("Drive Type : $DRIVE_TYPE");
    if(defined($DRIVE_ID)){
    logcomment("User Selected DRIVE ID's : @drive_arr\n Number of Drives Used for Aggreagate creation : $DRIVES_NUMBR\n Volume Size : $VOL_SIZE GB\n Number of Filersio Threads : $FILERSIO_READ_THREADS");
}else{
    logcomment("User Selected MODEL_DRIVE ID : $MODEL_DRIVE_ID\n Number of Drives Used for Aggreagate creation : $DRIVES_NUMBR\n Volume Size : $VOL_SIZE GB\n Number of Filersio Threads : $FILERSIO_READ_THREADS");
}
    #####################################################################
    # Unpartiton and re-assign all drive
    #####################################################################
    remove_partition( Noderef => \@Nodes );
    logcomment("Partition removed, Re-assign drive");
    nvme_drv_assign( Nodes => \@Nodes, force => 1, assign => 'yes' );
    logcomment("Drive assigne to both nodes");

    ####################################################################
    # Get Drives selected by user
    ####################################################################
    my @SEL_DRV;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj;
        my $Host = host($FILER_C);
        my @drives_use;
        if(!defined($DRIVE_ID)){
        logcomment("$FILER_C : Select drives with model ID provided by user  -  $MODEL_DRIVE_ID");}
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        if ( $filer_state =~ /UP|CLI/i ) {
            $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 );
        } elsif ( $filer_state =~ /MAINT/i ) {
            $Api_Set_Obj = NACL::APISet->new(
                hostobj   => $Host,
                category  => "Node",
                interface => "CLI",
                set       => "Maintenance"
            );
        }
        if(!defined($DRIVE_ID)){

        my @DRVS_2_USE = nvme_get_drv_by_user_sel_mod_id( API_con => $Api_Set_Obj, Node => $Node, model => $MODEL_DRIVE_ID, local_name => 'no', Filer => $FILER_C );
        logcomment("$FILER_C : Drives from selection : @DRVS_2_USE");
        push @SEL_DRV, @DRVS_2_USE;
    }}
     if(defined($DRIVE_ID)){
       if(scalar(@drive_arr)>=5 && scalar(@drive_arr)<=10){
        logcomment("Drive ID's selected by the user are @drive_arr");}
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        $usr_sel_drvs{$FILER_C}=[@drive_arr];
 }}else{

    logcomment("USER SELECTED DRIVES FOR PRE-CONDITION ARE : @SEL_DRV");
    logcomment("User Selected number of drive : $DRIVES_NUMBR drives of model $MODEL_DRIVE_ID");
    logcomment("Drives with model $MODEL_DRIVE_ID present on both filers : @SEL_DRV");
    my $sel_ary_drv = @SEL_DRV;
    if ( $sel_ary_drv < $DRIVES_NUMBR ) {
        logcomment("Sufficient drives are not present on filer");
        logcomment("**NORUN** : Sufficient drives with model $MODEL_DRIVE_ID is not present, Filer has $sel_ary_drv drives ");
        $post_test = 1;
        return $TCD::FAIL;
    } else {
        logcomment("Drive captured :@SEL_DRV");
        my @temp_drives_use = splice( @SEL_DRV, 0, $DRIVES_NUMBR );
        logcomment("Considering $DRIVES_NUMBR drives : @temp_drives_use");
        foreach my $Node (@Nodes) {
            my @temp_drv;
            my $FILER_C = $Node->name();
            my $Api_Set_Obj;
            my $Host        = host($FILER_C);
            my $Transit_obj = NACL::Transit->new( name => $FILER_C );
            my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
            if ( $filer_state =~ /UP|CLI/i ) {
                $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
                $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 );
            } elsif ( $filer_state =~ /MAINT/i ) {
                $Api_Set_Obj = NACL::APISet->new(
                    hostobj   => $Host,
                    category  => "Node",
                    interface => "CLI",
                    set       => "Maintenance"
                );
            }
            logcomment("Check drives belong to $FILER_C");
            my $owne_ot = $Api_Set_Obj->execute_raw_command( 'command' => "disk show -o $FILER_C" );
            my @output_data = split( /\n/, $owne_ot );
            foreach my $line (@output_data) {
                if ( $line =~ /(\S+)\s+(\S+)\s+\(\.*/ ) {
                    my $drv = $1;
                    my $own = $2;
                    foreach my $drvo (@temp_drives_use) {
                        if ( $drvo eq $drv ) {
                            push( @temp_drv, $drvo );
                        }
                    }
                }
            }
            $usr_sel_drvs{$FILER_C} = [@temp_drv];
            push @USER_SEL_DRV, @temp_drv;
        }
        logcomment("Drives with model $MODEL_DRIVE_ID is present, Total $DRIVES_NUMBR drives are available drives and drive are @USER_SEL_DRV");
    }
}
    #####################################################################
    # PRE -CONDITION ALL DRIVES
    #####################################################################
    nvme_eye_catcher( Test => $Test, string => "PRE-CONDITION : START" );
	nvme_pre_condition_drvs ( Nodes  => \@Nodes , Filer_drv => \%usr_sel_drvs );
    nvme_eye_catcher( Test => $Test, string => "PRE-CONDITION : COMPLETED" );

    #####################################################################
    # Unpartiton and re-assign all drive
    #####################################################################
    #remove_partition(Noderef => \@Nodes);
    logcomment("Partition removed, Re-assign drive");
    my $FILER_A = $Nodes[0]->name();
    my $FILER_B = $Nodes[1]->name();
    my @drv_a   = @{ $usr_sel_drvs{$FILER_A} };
    my @drv_b   = @{ $usr_sel_drvs{$FILER_B} };
    logcomment("Get drives from both nodes");
    nvme_drv_assign( Nodes => \@Nodes, force => 1, assign => 'yes' );
    logcomment("Drive assigne to both nodes");

    #####################################################################
    # Re-initilize drives to wipe out metadata
    #####################################################################
    logcomment("Re-initilize drives to wipe out metadata on filers $filer_names");

    #READ ENV FILE
    logcomment("Read ENV variables");
    my $Home = $ENV{HOME};
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        $ENV_FIL_LOC = "$Home/NDATE/FILER_INFO/$FILER_C";
        open( ENV_FL, "$ENV_FIL_LOC" ) || die "Can not open file $ENV_FIL_LOC";
        my @env_data = <ENV_FL>;
        logcomment("Get path for filer ");
        my ( $env, $nb );
        foreach my $line (@env_data) {
            chomp($line);
            if ( $line =~ /FILER=(\S+)/ ) {
                print "Filer : $1\n";
                $FILER_C = $1;
                $filer_env_details->{$FILER_C} = { 'filer' => $1 };
            }
            if ( $line =~ /PATH=(\S+)/ ) {
                $env = $1;
                print "ENV : $env\n";
                $filer_env_details->{$FILER_C} = { 'env_p' => $env };
            }
            if ( $line =~ /NETBOOT_IMG=(\S+)/ ) {
                $nb = $1;
                print "NB : $nb\n";
            }
        }
    }
    logcomment( "Dumper " . Dumper($filer_env_details) );

    #Clean_config
    my @sub_clean;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $env_data;
        foreach my $fil ( keys %$filer_env_details ) {
            logcomment("Get Env path to be set for filer $fil");
            if ( $fil =~ /$FILER_C/ ) {
                $env_data = $filer_env_details->{$fil}->{'env_p'};
                logcomment("Env variable to be used :  $env_data");
            }
        }
        push( @sub_clean, subtest( \&nvme_clean_config, -runid, "clean_config_node_$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) );
    }
    Subtest::wait_finish( subtest => [@sub_clean] );
    logcomment("Re-initilization of filers compelts, Proceed with cluster Create and Join");
    create_join_cluster( node_present => [@Nodes], nvme_setup => "yes" );
    logcomment("NVMe Setup compelted");

    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );

    return $TCD::PASS;

}

sub NVMe_Homo_Perf_Rand_Read_Test {
    my @subtests;
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment( "Starting test Steps on " . $Node->name() );
        push( @subtests, subtest( \&exe_homogen_steps, -runid, "Exe_Hmr_Srb_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $status = status_return( subtest_ref => [@subtests] );
    logcomment("Total test status is : $status");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub exe_homogen_steps {
    $Test->description("Execute Homogeneous scritp");
    my $Home     = $ENV{HOME};
    my $Exe_Node = shift;
    logcomment( "Filer passed to the subtest :: " . $Exe_Node->name() );
    #####TEST VARIABLES#####
    my %step1_arr = ();
    my @disk_used = ();
    my $FILERB;
    my $loop      = 0;
    my $flag_fail = 0;
    my $check     = 2;
    #######################

    ################################################
    #STEP 1
    ################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 1 of 15 : List all drives " );
    my $disk_list = nvme_list_only_nvme( node_present => [$Exe_Node] );

    ################################################
    #STEP 2
    ################################################
    my $FILER_C = $Exe_Node->name();
    nvme_eye_catcher( Test => $Test, string => "STEP 2 of 15: $FILER_C : Perform sysconfig -v to locate the shelf.diskid that matches ID_test_driv : $MODEL_DRIVE_ID" );
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    if(defined($DRIVE_ID)){
    logcomment("Drive ID's selected by the user are @drive_arr");}
    else{

    my $Api_Set_Obj = $Exe_Node->get_7m_or_nodescope_apiset( connid => "console" );
    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    my $output = $Api_Set_Obj->execute_command( 'command' => "sysconfig -v" );
    my @arr1 = split( /\n/, $output );
    my @arr_disk = ();

    foreach my $line (@arr1) {
        my $disk;
        my $model;
        if ( $line =~ /(\d+)\s*\:\s*\S+\s*(\S+)\s*\S+\s*\S+\s*/ ) {
            $disk  ="0n."."$1";
            $model = $2;
            if ( $model =~ /$MODEL_DRIVE_ID/i ) {
                push( @arr_disk, $disk);
                @arr_disk = unique(@arr_disk);
                $step1_arr{$FILER_C} = [@arr_disk];
            }
        }
    }
    logcomment( "Disks which matches the MODEL_DRIVE_ID: $MODEL_DRIVE_ID given by user are " . scalar @arr_disk . " and the drives are @arr_disk" );
}
    ################################################
    #STEP 3
    ################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 3 of 15: $FILER_C : Check for Spare Drives available and sufficient to create aggregate" );
    logcomment("User selected $DRIVES_NUMBR drives to create aggregate");
    my $spare_list  = nvme_sto_agg_show_spare_dsk( Node => $Exe_Node );
    my @spare_drvs  = keys %$spare_list;
    my $spare_count = @spare_drvs;
    my @drives_use;
    my @aggr_c_drv;
    if(defined($DRIVE_ID)){
       @drives_use = @drive_arr;}
    else{
       @drives_use  = @USER_SEL_DRV;}
    logcomment("Spare drive available are : @spare_drvs");
    logcomment("Total spare count is $spare_count");
    logcomment("Drive Belongs to Filer : $FILER_C : @spare_drvs");

    foreach my $sdrv (@spare_drvs) {
        my ($chk) = $sdrv =~ /\w+\.\w+\.(\w+)/;
        foreach my $usr_sel (@drives_use) {
            my ($matc) = $usr_sel =~ /\w+\.(\S+)/;
            if ( $matc == $chk ) {
                push( @aggr_c_drv, $sdrv );
            }
        }
    }
    @aggr_c_drv = unique(@aggr_c_drv);

    #my @DRVS_2_USE = nvme_get_drv_by_user_sel_mod_id( API_con => $Api_Set_Obj, Node => $Exe_Node, model => $MODEL_DRIVE_ID, local_name => 'no', Filer => $FILER_C );
    #my @drives_use = splice( @DRVS_2_USE, 0, $DRIVES_NUMBR );
    #my $spare_count_mod = @drives_use;a
    logcomment("Consider drives @aggr_c_drv which was pre-conditioned to create aggregate");
    my $spare_count_mod = @aggr_c_drv;
    if ( $spare_count_mod >= $DRIVES_NUMBR ) {
        logcomment("User selected drive count is $DRIVES_NUMBR, Sufficient spares are available to create aggregate");
    } elsif ( $spare_count_mod < $DRIVES_NUMBR ) {
        $post_test = 1;
        logcomment("User selected $DRIVES_NUMBR, but spare count is $spare_count_mod");
        logcomment("**FAIL** : Not Enough spare drives to create aggregate");
        logresult( "FATAL", "Not Enough spare drives to create aggregate" );
    }

    ################################################
    #STEP 4
    ################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 4 of 15: $FILER_C : Create an aggregate with the disks count supplied by user" );
    my $Api_Set_Obj = $Exe_Node->get_7m_or_nodescope_apiset( connid => "console" );
    $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );
    logcomment("As the number of drives which user has selected is $DRIVES_NUMBR, drives that will be used for aggregate creation ");
    my $aggregate_name = "homo_aggr_perf_test_" . "$FILER_C";
    $aggregate_name =~ s/\-/_/g if ( $aggregate_name =~ /\-/ );
    logcomment("$FILER_C : Creating aggregate $aggregate_name... ");
    my $job;
    my $stask_aggr_obj = NACL::STask::Aggregate->create(
        command_interface  => $Exe_Node,
        aggregate          => $aggregate_name,
        node               => $FILER_C,
        nacltask_if_exists => 'reuse',
        job_component      => \$job,

        #diskcount          => $DRIVES_NUMBR,
        disklist => \@aggr_c_drv,
    );
    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    my $output = $Api_Set_Obj->execute_raw_command( 'command' => "aggr show $aggregate_name -fields availsize", 'connectrec-match_table' => $prompts_answers1 );

    if ( $output =~ /$aggregate_name/i ) {
        logcomment("$FILER_C : Aggregate $aggregate_name created on Node $FILER_C");
    } else {
        $post_test = 1;
        logcomment("**FAIL** : $FILER_C : AGGREGATE $aggregate_name WAS NOT CREATED, so exiting from test");
        return logresult( "INFO", msg => 1 );
    }
    ################################################
    #STEP 5
    ################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 5 of 15: $FILER_C : Create vserver" );
    my $vserver_name = "vser_test_" . "$FILER_C";
    $vserver_name =~ s/\-/_/g if ( $vserver_name =~ /\-/ );
    logcomment("$FILER_C : Creating vserver $vserver_name...");
    my $root        = "vol0";
    my $ns          = "file";
    my $sec         = "unix";
    my $vserver_obj = NACL::STask::Vserver->create(
        command_interface           => $Exe_Node,
        vserver                     => $vserver_name,
        subtype                     => 'default',
        rootvolume                  => $root,
        aggregate                   => $aggregate_name,
        "rootvolume-security-style" => $sec,
        nacltask_if_exists          => 'reuse',
    );
    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    $output = $Api_Set_Obj->execute_raw_command( 'command' => "vserver show", 'connectrec-match_table' => $prompts_answers1 );

    if ( $output =~ /$vserver_name/i ) {
        logcomment("$FILER_C : vserver $vserver_name created on Node $FILER_C");
    } else {
        logcomment("**FAIL** : $FILER_C : VSERVER $vserver_name HAS NOT BEEN CREATED PROPERLY, so exiting the test");
        return logresult( "INFO", msg => 1 );
    }
    ###########################################################
    ## STEP 6 - Create a volume per the size defined by the user (vol_size)
    ## Pass/Fail criteria: N/A
    ###########################################################

    my $volsize_g = $VOL_SIZE . "g";
    nvme_eye_catcher( Test => $Test, string => "STEP 6 of 15: $FILER_C : Create a volume per the size defined by the user vol_size: $volsize_g" );
    ###########################################################
    # Disable volume snapshot ploicy
    ###########################################################
    eye_catcher( Test => $Test, string => "Disable volume snapshot policy" );
    logcomment("disabling volume policy");
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule daily -newcount 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule weekly -newcount 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule hourly -newcount 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify -policy default -enabled false" );
    logcomment("Disabled snapshot policy");

    my $vol_name = "vol_test_" . "$FILER_C";
    $vol_name =~ s/\-/_/g if ( $vol_name =~ /\-/ );
    my $cmd = "vol create -vserver $vserver_name -volume $vol_name -aggregate $aggregate_name -size $volsize_g -state online -policy default -unix-permissions ---rwxr-xr-x";
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command(
        'command'                => "$cmd",
        'connectrec-match_table' => $prompts_answers
    );
    sleep(10);
    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    $output = $Api_Set_Obj->execute_raw_command( 'command' => "vol show", 'connectrec-match_table' => $prompts_answers1 );

    if ( $output =~ /$vol_name/i ) {
        my ($res) = readable_disk_size( GB => $VOL_SIZE );
        logcomment("$FILER_C : Volume $vol_name of size $res created on Node $FILER_C");
    } else {
        logcomment("**FAIL** : $FILER_C -  $vol_name COULD NOT BE CREATED ON NODE $FILER_C");
        return logresult( "INFO", msg => 1 );
    }
    logcomment("$FILER_C : Executing sysconfig -r to check drive assignment and layout of drives");
    $Api_Set_Obj->execute_raw_command( 'command' => "run local sysconfig -r" );

    ################################################
    #STEP 7
    ################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 7 of 15 : $FILER_C : create file (number of threads user selected) only in read volumes using mkfile", );
    my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local df" );
    my ($vol_size_df) = ( $output =~ /\/vol\/$vol_name\/\s*\d+\s*\d+\s*(\d+)/i );
    $vol_size_df = $volsize_g if ( !$vol_size_df );
    logcomment("DEBUG : Volume $vol_name size is $vol_size_df");
    my $vol_size_df = $vol_size_df / ($FILERSIO_READ_THREADS);
    my ( $res, $res1 ) = readable_disk_size( KB => $vol_size_df );
    $res =~ s/( |B|\.\d+)//ig;
    logcomment("DEBUG : Volume $vol_name size is $res - $res1");

    for ( my $i = 1 ; $i <= $FILERSIO_READ_THREADS ; $i++ ) {
        $Api_Set_Obj->execute_command( 'command' => "mkfile $res /vol/$vol_name/f$i" );
    }
    logcomment("$FILER_C : Executing sysconfig -r to check drive assignment and layout of drives");

    #$Api_Set_Obj->execute_command( 'command' => "sysconfig -r", 'connectrec-match_table' => $prompts_answers1 );

    ###########################################################
    ## STEP 6 - Start #user_defined number of filersio threads on the volume created.
    ## Pass/Fail criteria: N/A
    ###########################################################
    eye_catcher(
        Test   => $Test,
        string => "STEP 8 of 15 : $FILER_C : Start $FILERSIO_READ_THREADS number of filersio threads on the volume ",
    );

    #starts $user_defined no of filersio threads on the volume created $vol_name of size $vol_size
    #$Api_Set_Obj->execute_raw_command( 'command' => "run local;priv set -q test" );
    #$Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );
    $Api_Set_Obj->priv_set( "level" => "test" );
    for ( my $i = 1 ; $i <= $FILERSIO_READ_THREADS ; $i++ ) {
        $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio asyncio_active 100 -r 100 64k 0 $res -1 16 /vol/$vol_name/f$i" );
        if ( $output =~ /failed to start/ig ) {
            logcomment("**FAIL** : $FILER_C : FILERSIO FAILED TO START");
            return logresult( "INFO", msg => 1 );
        } else {
            logcomment(" Filersio threads are started on $i instance");
            next;
        }
    }
    sleep(30);
    for ( my $i = 0 ; $i < $FILERSIO_READ_THREADS ; $i++ ) {
        my $filersio_stat = $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio status" );
        if ( $filersio_stat =~ /filersio: instance $i: asyncio_active running/i ) {
            logcomment( "$FILER_C : Thread " . ( $i + 1 ) . " : filersio is running" );
        }
    }

    #$Api_Set_Obj->execute_raw_command( 'command' => "exit" );

    ###########################################################
    ## STEP 7 - Start the statistics for the disks selected in step#3
    ## Pass/Fail criteria: N/A
    ###########################################################

    eye_catcher(
        Test   => $Test,
        string => "STEP 9 of 15 : $FILER_C : Start statisics command for the disks that have been selected",
    );
    my $sample_id;
    $output = $Api_Set_Obj->execute_raw_command(
        'command'                => "statistics start -object disk",
        'connectrec-match_table' => $prompts_answers
    );
    if ( $output =~ /Statistics collection is being started for sample-id:\s+(\S+)/ ) {
        logcomment("$FILER_C : Statistics command for the selected disks as been started");
        $sample_id = $1;
        logcomment("DEBUG: Captured Sample -id $sample_id to Stop the Statistics");
    } else {
        logcomment("**FAIL** : $FILER_C - STATISTICS COMMAND FOR THE SELECTED DISKS COULD NOT BE STARTED");
        return logresult( "INFO", msg => 1 );
    }
    logcomment("$FILER_C : Waiting for Statistics object to start on disk");
    eye_catcher(
        Test   => $Test,
        string => "STEP 10 of 15: $FILER_C : Collect the statit info",
    );
    $Api_Set_Obj->execute_command( 'command' => " statit -b " );

    sleep(60);
    logcomment("Wait for 30 min and then execute sattis -e");
    sleep(1800);
    logcomment("Execute statit -e");
    ###########################################################
    ## STEP 8 - Display the statistics for the disks selected in step#3
    ## Pass/Fail criteria: N/A
    ###########################################################
    eye_catcher(
        Test   => $Test,
        string => "STEP 11 of 15 : $FILER_C : Displaying the statisics for the disks that have been selected",
    );
    $prompts_answers = [ ".*to page down, .*" => ' ' ];
    $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );
    my $disks;
    my $disk_l = $Api_Set_Obj->execute_raw_command( 'command' => "aggr show $aggregate_name -fields disklist" );
    my @drives_use;
    foreach my $line ( split( /\n/, $disk_l ) ) {

        if ( ( $line =~ /\,/ ) && ( $line =~ /(\S+)\s+(\S+)/ ) ) {
            my $drs = $2;
            $drs =~ s/$FILER_C\://g;
            @drives_use = split( /\,/, $drs );
        }
    }
    logcomment("DEBUG : Aggregate drives are : @drives_use");
    foreach my $dsk (@drives_use) {
        $disks = $disks . $dsk . "|";
    }
    $disks =~ s/\|$//;
    logcomment("Get Statistics for drives : $disks ");
    $output = $Api_Set_Obj->execute_raw_command(
        'command'                => "statistics show -object disk -instance $disks",
        'connectrec-match_table' => $prompts_answers
    );
    if ( $output =~ /show failed/ ) {
        logcomment("**WARNING** : FAILED TO SHOW THE STATISTICS FOR DISKS $disks");
    }

    ###########################################################
    ## STEP 9 - Stop the statistics for the disks selected in step#3
    ## Pass/Fail criteria: N/A
    ###########################################################

    eye_catcher(
        Test   => $Test,
        string => "STEP 12 of 15 : $FILER_C : Stop the statisics for the disks that have been selected",
    );
    $output = $Api_Set_Obj->execute_raw_command( 'command' => "statistics stop -sample-id $sample_id" );
    if ( $output =~ /(stopped\s*for\s*sample-id.*$sample_id)|("$sample_id"\s*already\s*stopped)/ig ) {
        logcomment("Statistics collection is being stopped for sample-id: $sample_id");
    } else {
        logcomment("**FAIL** : FAILED TO STOP THE STATSTICS  FOR SAMPLE-ID $sample_id");
        return logresult( "INFO", msg => 1 );
    }
    sleep(60);

    ###########################################################
    ## STEP 11 - Stop the statit
    ## Pass/Fail criteria: N/A
    ###########################################################

    eye_catcher(
        Test   => $Test,
        string => "STEP 13 of 15 : $FILER_C : Stop the statit",
    );
    $output = $Api_Set_Obj->execute_command( 'command' => "filersio status" );
    if ( $output =~ /(asyncio_active running|running)/ig ) {
        logcomment("filersio threads are running");
    }
    $output = $Api_Set_Obj->execute_command( 'command' => "statit -e" );

    ###########################################################
    ## STEP 12 - Stop the filersio instances running
    ## Pass/Fail criteria: N/A
    ###########################################################

    eye_catcher(
        Test   => $Test,
        string => "STEP 14 of 15 : $FILER_C : Stop the filersio instances",
    );
    $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );
	logcomment("Dump sk trace log ");
    nvme_get_sk_trace_logs( Node =>  $Exe_Node, Destination => $LOGDIR, runtime => 300 );
    logcomment("End of sk trace log capture");
    $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio stop\"" );
    logcomment("Waiting for 5 minutes");
    sleep(300);
    $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio status\"" );

    if ( $output =~ /(filersio:\s*aborting\s*instance\s*)|/ ) {
        logcomment("Filersio threads are stopped Successfully");
    } else {
        logcomment("**WARNING** : Filersio threads are not  stopped ");
    }
    ###########################################################
    ## STEP 13 - Delete the aggregate, volume and vserver created for this test
    ## Pass/Fail criteria: N/A
    ###########################################################

    eye_catcher(
        Test   => $Test,
        string => "STEP 15 of 15 : $FILER_C : Delete the volumes, vserver and aggregate created",
    );
    logcomment("Deleting the non root aggregate created for this test");
    aggregate_volume_delete( Node => $Exe_Node );
    vserver_delete( Node => $Exe_Node );
    return logresult( "INFO", msg => 0 );
}

#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;
    if ( $post_test == 0 ) {
        nvme_post_test(
            node_present    => $node_ref,
            Test            => $Test,
            change_state_to => "MAINT",
            filer_mode      => $Mode
        );
    }
    return $TCD::PASS;
}