#!/usr/software/bmn/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/TEST_SCRIPTS/SSD/NADQ02_NVMe_WAF_Dual_port.thpl#0 $
## All rights reserved.
##
## @summary Mode
##  File System test for WAF evaluation
##
## @description
##  Evaluates the WAF on dual port
##
## @Test Mode
##  File System Creation Mode
##
## @Test bed setup
##  FC   :Dual Path, Cluster Fabric
##  SAS  :Dual Path
##
## @usage
##  The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  There is no dependency
##
## @steps
##  The test will execute steps mentioned below:
##   1.  Perform Sysconfig -v to locate the shelf.diskid that matches ID_test_drive
##   2a. Assigning P1 partitions to one Node2  and p2 to Node1
##   2b. Execute disk show -owner local to see the topology of the disks and ownership of local owner on which test will be run
##   3. Create an aggregate with the disks identified using step #1 and #2
##   4. Create a vserver
##   5. Create a volume of of size 98% of aggregate.
##   6. Run filersio thread
##   7. Disable volume snapshot ploicy
##   8. Pre Data collection for WAF calculation
##      - a. Start statit
##      - b. Start statistics
##      - c. Show disk statistics
##      - d. Read log sense page 0xC0 and get host_writes and physical_writes
##   9. Check filersio status
##   10. Post Data collection for WAF calculation
##       - a. Show disk statistics
##       - b. Read log sense page 0xC0 and get host_writes and physical_writes
##       - c. Stop statit
##       - d. Stop disk statistics
##   11. Check for write chain
##       - a. Check statit for write chain of data drives is less than five
##       - b. Display ontap disk statistics
##             - Host_writes
##             - Physical_writes
##             - WAF
##   12.Check filersio
##   13.Stop the filersio command
##   In Cleanup
##   Delete the aggregate, vserver, volume created for the test
##
## @param FILER        - optional for cluster setup
##                     - required for Dual path setup      Name of filer to be used
## @param FILERA       - required for cluster setup
##                     - optional for Dual path setup      Name of filer to be used
## @param FILERB       - required for cluster setup
##                     - optional for Dual path setup      Name of the partner node
## @param TEST_CONFIG  - optional Type of the setup, C for cluster, D for dual path (default)
## @param TEST_SETUP   - optional Default value set to 'FC'
## @param FILER_CONN   - optional Type of connection, default is set to 'console' for this test
## @param MODE         - optional Filer mode maint/normal, default set to 'maint' for this test
## @param FILER_PROMPT - optional prompt of the filer
## @param LOGDIR       - optional This is required to generate 'END-LOG'. Default set to main log directory.
## @param EOE          - optional 'default' is set to default value
## @param BOOT_MODE    - optional Boot mode for filer , default set to '1' for this test
## @param BOOT_TYPE    - optional Boot type, default set to 'A' (automatic) for this test
## @param SSD          - optional Setup is SSD or not, default set to 'no'
## @param EMAIL        -  optional Default value set to 'Y'
## @param EMAIL_TO     - required for setting to address sending email to recipient
## @param EMAIL_FROM   - required for setting  from address for sending email
## @param FILER_TYPE   - Required to set DataONTAP version    DataONTAP version is either IC or BR
## @param DRIVE_ID     - required to select the disks for the test
## @param DRIVES_NUMBR - required to create the aggregate
##
## @status  Automated
## @author  khosur@netapp.com
## @burt    1115437
## @change  YY/MM/DD from author: Description of the change
##
#############################################################################
#############################################################################
# Compiler directives.
use strict;
use Storage::Common_Lib;
use Storage::NVMe_Common_Lib;

use TCD;
use POSIX;
use boot;
use NATE::BaseException qw(:try);
use List::MoreUtils qw(each_array);

#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $ARMADILLO
  $DRIVE_ID
  $VOL_SIZE
  $DRIVES_NUMBR
  $DRIVE_TYPE
  $ENV_FIL_LOC
  $skt
);
my $params = NATE::ParamSet->new( global => 1 );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',                       default => 'D' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME',                    default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',                            default => undef );
$EOE            = $params->get( 'EOE',                               default => 'default' );
$EMAIL          = $params->get( 'EMAIL',                             default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',                           default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',                         default => 'Email from' );
$FILER_PROMPT   = $params->get( 'FILER_PROMPT',                      default => '\*>' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',                        default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',                         default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',                         default => 'A' );
$SSD            = $params->get( 'SSD',                               default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',                         default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',                        default => 'BR' );
$DRIVE_ID       = $params->get( 'DRIVE_ID_NVMe_WAF_Dual_port',       default => undef );
$DRIVES_NUMBR   = $params->get( 'DRIVES_NUMBR_NVMe_WAF_Dual_port',   default => '6' );
$DRIVE_TYPE     = $params->get( 'DRIVE_TYPE',                        default => undef );
$ENV_FIL_LOC    = $params->get( 'ENV_FIL_LOC',                       default => '/home/nvme/NDATE/FILER_INFO/NVMe_ENV_VAR_166_167' );
$skt            = $params->get( 'SKTRACE',                           default => '0' );
################
# Testcase name
################
my $TC_name;
$TC_name = "814_NADQ02_WAF_Dual_port";

########################################
### Initialization/declaration
#########################################
my $test_status = 0;
my %nodes_filer = {};
my @Nodes;
my $filer_names;
my $Mode;
my ( $email_subject, $email_body );
$EOE = 1;
my $timeout;
my @Node_use;
my %fil_owner          = {};
my %Fil_aggr_disk      = {};
my %Fil_dis_use        = {};
my %autoassign         = {};
my %info_waf_plot_file = {};
my $partition;
my $Node_use;
my $norun_flag = 0;
my @USER_SEL_DRV;
my %usr_sel_drvs;

#########################################
# Testcase available for execution
#########################################
## Pre-test processes
#########################################

my @Testcases = ( NVME_WAF_Dual_port => "NVME_WAF_Dual_port - CLI" );

&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status   = 1;
        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }
    exit(0);
} ## end sub main

########## INIT  ###################################################
# init automatically called before executing tests
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    logcomment("Drive type is $DRIVE_TYPE");
    logcomment("Model ID which user has selected : $DRIVE_ID");
    logcomment("Number of drives which user ahs selected for aggregate creation : $DRIVES_NUMBR");

    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node objects used for test execution.
    grep { logcomment( "Filers passed to the script are " . $_->name() ) } @Nodes;
    logcomment("Checking for execution mode");
    $Mode = $Nodes[0]->mode();

    version_test( node_present => \@Nodes, tc_name => $TC_name );
    logcomment( "Filer- $filer_names : $TC_name : started at : " . scalar( localtime() ) );
    logcomment( "Filer- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}

########## SETUP  ###################################################
# Setup automatically called before executing tests
#####################################################################
sub setup() {

    $Test->description("Setup the environment for the test exectuion ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref          = \@Nodes;
    my $filer_env_details = {};
    my @SEL_DRV;

    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my @DRVS_2_USE  = nvme_get_drv_by_user_sel_mod_id( API_con => $Api_Set_Obj, Node => $Node, model => $DRIVE_ID, local_name => 'no', Filer => $FILER_C );
        logcomment("$FILER_C : Drives from selection : @DRVS_2_USE");
        $usr_sel_drvs{$FILER_C} = [@DRVS_2_USE];
    }

    #####################################################################
    # PRE -CONDITION ALL DRIVES
    #####################################################################
    nvme_eye_catcher( Test => $Test, string => "PRE-CONDITION : START" );
    nvme_pre_condition_drvs( Nodes => \@Nodes, Filer_drv => \%usr_sel_drvs );
    nvme_eye_catcher( Test => $Test, string => "PRE-CONDITION : COMPLETED" );

    #####################################################################
    # Unpartiton and re-assign all drive
    #####################################################################
    remove_partition( Noderef => \@Nodes );
    logcomment("Partition removed, Re-assign drive");
    nvme_drv_assign( Nodes => \@Nodes, force => 1, assign => 'yes' );
    logcomment("Drive assigne to both nodes");

    # REINITILIZING FILERS BEFORE STARTING TEST , TO WIPEOUT METADATA
    logcomment("REINITILIZING FILERS BEFORE STARTING TEST , TO WIPEOUT METADATA");

    #READ ENV FILE
    logcomment("Read ENV variables");
    my $Home = $ENV{HOME};
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        $ENV_FIL_LOC = "$Home/NDATE/FILER_INFO/$FILER_C";
        open( ENV_FL, "$ENV_FIL_LOC" ) || die "Can not open file $ENV_FIL_LOC";
        my @env_data = <ENV_FL>;
        logcomment("Get path for filer ");
        my ( $env, $nb );
        foreach my $line (@env_data) {
            chomp($line);
            if ( $line =~ /FILER=(\S+)/ ) {
                print "Filer : $1\n";
                $FILER_C = $1;
                $filer_env_details->{$FILER_C} = { 'filer' => $1 };
            }
            if ( $line =~ /PATH=(\S+)/ ) {
                $env = $1;
                print "ENV : $env\n";
                $filer_env_details->{$FILER_C} = { 'env_p' => $env };
            }
            if ( $line =~ /NETBOOT_IMG=(\S+)/ ) {
                $nb = $1;
                print "NB : $nb\n";
            }
        }
    }
    logcomment( "Dumper " . Dumper($filer_env_details) );

    #Clean_config
    my @sub_clean;

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $env_data;
        foreach my $fil ( keys %$filer_env_details ) {
            logcomment("Get Env path to be set for filer $fil");
            if ( $fil =~ /$FILER_C/ ) {
                $env_data = $filer_env_details->{$fil}->{'env_p'};
                logcomment("Env variable to be used :  $env_data");
            }
        }
        push( @sub_clean, subtest( \&nvme_clean_config, -runid, "clean_config_node_$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) );
    }
    Subtest::wait_finish( subtest => [@sub_clean] );
    logcomment("Re-initilization of filers compelts, Proceed with cluster Create and Join");
    create_join_cluster( node_present => [@Nodes], nvme_setup => "yes" );
    logcomment("NVMe Setup compelted");

    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );

    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        my $output      = $Api_Set_Obj->execute_raw_command( 'command' => "storage disk option show -fields autoassign -node $FILER_C" );
        ( $autoassign{$FILER_C} ) = $output =~ /$FILER_C\s*(\S+)/;
        if ( $autoassign{$FILER_C} =~ /on/ ) {
            logcomment("$FILER_C : Set disk autoassign to OFF");
            $Api_Set_Obj->execute_raw_command( 'command' => "storage disk option modify -node $FILER_C  -autoassign off" );
        }
    }

    return $TCD::PASS;
}

########## TEST 1 ###################################################
#NADQ02_Homo_Perf_Test
#####################################################################
sub NVME_WAF_Dual_port {
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filers are $filer_names" );
    $Test->description("WAF test for SSD dual port use case");

    #####TEST VARIABLES#####
    my @disk_used = ();
    my $FILERB;
    my $loop      = 0;
    my $flag_fail = 0;
    my $check     = 2;
    #######################

    ## TEST STARTS
    my @subtests;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&NVME_WAF_Dual_port_sub, -runid, "WAF_Dual_port_$FILER_C", -bg, "--", $Node, $DRIVE_ID, $DRIVES_NUMBR ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $return = status_return( subtest_ref => [@subtests] );

###########################################################################################
## Test log/results
###########################################################################################
    logcomment("Test $TC_name is completed.");
    if ( $return == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub NVME_WAF_Dual_port_sub {
    $Test->description("WAF test for NVMe dual port use case");

    my $Node             = shift(@_);
    my $DRIVE_ID         = shift(@_);
    my $DRIVES_NUMBR     = shift(@_);
    my $FILERSIO_THREADS = 10;
    my $VOL_SIZE;
    my $FILER_C = $Node->name();

    ################################################
    #STEP 1
    ################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 1 of 13 : List all drives " );
    my $disk_list = nvme_list_only_nvme( node_present => [$Node] );

    ### Getting Model Name, Drive Model and FW Version for WAF plt file
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    my $output = $Api_Set_Obj->execute_command( 'command' => "sysconfig -v" );
    my @arr1 = split( /\n/, $output );
    my @arr_disk = ();
    my $model_name;
    my $fw;

    foreach my $line (@arr1) {
        my $disk;
        my $model;
        if ( $line =~ /Model Name:\s*(\S+)/ig ) {
            $info_waf_plot_file{$FILER_C}{'Model_Name'} = $1;
        }
        $info_waf_plot_file{$FILER_C}{'FILERSIO_THREADS'} = $FILERSIO_THREADS;
        if ( $line =~ /(\d+)\s*\:\s*\S+\s*(\S+)\s*(\S+)\s*\S+\s*/ ) {
            $disk  = $1;
            $model = $2;
            $fw    = $3;
            if ( $model =~ /$DRIVE_ID/i ) {
                $info_waf_plot_file{$FILER_C}{'FW'}         = $fw;
                $info_waf_plot_file{$FILER_C}{'DRIVE_MDEL'} = $DRIVE_ID;
            }
        }
    }

    ################################################
    #STEP 2
    ################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 2 of 13: $FILER_C : Check for Spare Drives available and sufficient to create aggregate" );
    logcomment("User selected $DRIVES_NUMBR drives to create aggregate");
    my $spare_list  = nvme_sto_agg_show_spare_dsk( Node => $Node );
    my @spare_drvs  = keys %$spare_list;
    my $spare_count = @spare_drvs;
    my @drives_use;
    my @aggr_c_drv;

    @drives_use = @{ $usr_sel_drvs{$FILER_C} };
    logcomment("drives to use are : @drives_use");
    logcomment("Spare drive available are : @spare_drvs");
    logcomment("Total spare count is $spare_count");
    logcomment("Drive Belongs to Filer : $FILER_C : @spare_drvs");
    foreach my $sdrv (@spare_drvs) {
        my ($chk) = $sdrv =~ /\w+\.(\S+)/;
        foreach my $usr_sel (@drives_use) {
            my ($matc) = $usr_sel =~ /\w+\.(\S+)/;
            if ( $matc == $chk ) {
                push( @aggr_c_drv, $sdrv );
            }
        }
    }
    @aggr_c_drv = unique(@aggr_c_drv);

    logcomment("Consider drives @aggr_c_drv which was pre-conditioned to create aggregate");
    my $spare_count_mod = @aggr_c_drv;
    if ( $spare_count_mod >= $DRIVES_NUMBR ) {
        logcomment("User selected drive count is $DRIVES_NUMBR, Sufficient spares are available to create aggregate");
    } elsif ( $spare_count_mod < $DRIVES_NUMBR ) {
        $norun_flag = 1;
        logcomment("User selected $DRIVES_NUMBR, but spare count is $spare_count_mod");
        logcomment("**FAIL** : Enough spare drives are NOT present to create aggregate spare count is $spare_count_mod");
        logresult( "FATAL", "Enough spare drives are NOT present to create aggregate spare count is $spare_count_mod" );
    }
    my @drives_for_aggr = splice( @aggr_c_drv, 0, $DRIVES_NUMBR );

    #################################################################################
    ## STEP 3 - Create an aggregate with the disks identified using step #1 and #2 ##
    ## Pass/Fail criteria: N/A						                               ##
    #################################################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 3 of 13: $FILER_C : Create an aggregate with the disks identified using step #1 and #2" );
    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
    $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    logcomment( "As the number of drives which user has selected is $DRIVES_NUMBR, drives that will be used for aggregate creation " . scalar @drives_for_aggr . " and the drives are @drives_for_aggr" );

    my $aggregate_name = "aggr_WAF_test_" . "$FILER_C";
    $aggregate_name =~ s/\-/_/g if ( $aggregate_name =~ /\-/ );
    logcomment("$FILER_C : Creating aggregate $aggregate_name... ");
    my $aggr_create = nvme_create_aggregate( Node => $Node, aggr_name => $aggregate_name, drive_to_use => \@drives_for_aggr );
    logcomment("Aggr Status - $aggr_create");
    my $output = $Api_Set_Obj->execute_raw_command( 'command' => "aggr show $aggregate_name -fields availsize" );

    if ( $output =~ /failed|does not exist/i ) {
        logcomment("**FAIL** : $FILER_C : Aggregate $aggregate_name was NOT created, So exiting from test");
        return logresult( "INFO", msg => 1 );
         logcomment("$FILER_C : Aggregate $aggregate_name created on Node $FILER_C");
    } else {
        logcomment("$FILER_C : Aggregate $aggregate_name created on Node $FILER_C");
    }
    logcomment(" Execute aggregate show ");
    $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate show -aggregate $aggregate_name -fields disklist" );

    ## Disabling aggregate snapshot
    logcomment("Disabling aggregate snapshot");
    $Api_Set_Obj->execute_raw_command( 'command' => "run local aggr options $aggregate_name nosnap on" );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local snap sched -A $aggregate_name 0 0 0" );

    ## Checking aggregate size
    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    my $output = $Api_Set_Obj->execute_raw_command( 'command' => "aggr show $aggregate_name -fields availsize", 'connectrec-match_table' => $prompts_answers1 );
    my $aggr_size;
    my $vol_size;

    if ( $output =~ /$aggregate_name/i ) {
        logcomment("$FILER_C : Aggregate $aggregate_name created on Node $FILER_C");
        foreach my $line ( split( /\n/, $output ) ) {
            if ( $line =~ /^$aggregate_name\s+(\S+)[GT]B/i ) {
                $aggr_size = $1;
                $aggr_size = $aggr_size * 1024 if ( $line =~ /^$aggregate_name\s+(\S+)TB/i );
                my ($val) = readable_disk_size( GB => $aggr_size );
                logcomment("$FILER_C : Size of created aggregate  --> $val");
                $info_waf_plot_file{$FILER_C}{'AGGR_SIZE'} = $val;
                $vol_size = floor( $aggr_size * 0.98 );
                my ($res) = readable_disk_size( GB => $vol_size );
                logcomment("98% of the aggregate size $val that will be used for creating the volume : $res");
                $VOL_SIZE = $vol_size;
            }
        }
    } else {
        logcomment("**FAIL** : $FILER_C : Aggregate $aggregate_name was NOT created, so exiting from test");
        return logresult( "INFO", msg => 1 );
    }

    ## Recording out of all the settings of aggregate
    logcomment("Recording out of all the settings aggregate");
    $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;aggr options $aggregate_name\"" );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;snap reserve -A $aggregate_name 0\"" );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;snap sched -A $aggregate_name\"" );

    ###########################################################
    ## STEP 4 - Create Vserver
    ## Pass/Fail criteria: N/A
    ###########################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 4 of 13: $FILER_C : Create vserver" );
    my $vserver_name = "vser_test_" . "$FILER_C";
    $vserver_name =~ s/\-/_/g if ( $vserver_name =~ /\-/ );
    logcomment("$FILER_C : Creating vserver $vserver_name...");
    my $root = "vol0";
    my $ns   = "file";
    my $sec  = "unix";
    logcomment("Create Vserver $vserver_name with root volume $root on filer $FILER_C");
    my $prompts_answers = [ ".*Do you want to continue??.*" => 'y' ];
    my $vser_o = $Api_Set_Obj->execute_raw_command( 'command' => "vserver create -vserver $vserver_name -rootvolume $root", 'connectrec-match_table' => $prompts_answers );
    logcomment("DEBUGL : $vser_o");

    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    $output = $Api_Set_Obj->execute_raw_command( 'command' => "vserver show", 'connectrec-match_table' => $prompts_answers1 );

    if ( $output =~ /$vserver_name/i ) {
        logcomment("$FILER_C : vserver $vserver_name created on Node $FILER_C");
    } else {
        logcomment("**FAIL** : $FILER_C : vserver $vserver_name has NOT been created properly, so exiting the test");
        return logresult( "INFO", msg => 1 );
    }

    ###########################################################
    ## STEP 5 - Create a volume of 98% aggr size
    ## Pass/Fail criteria: N/A
    ###########################################################
    my $volsize_g = $VOL_SIZE . "g";
    nvme_eye_catcher( Test => $Test, string => "STEP 5 of 13: $FILER_C : Create a volume of 98% aggr size : $volsize_g" );
    my $vol_name = "vol_of_waf_test_" . "$FILER_C";
    $vol_name =~ s/\-/_/g if ( $vol_name =~ /\-/ );
    my $cmd = "vol create -vserver $vserver_name -volume $vol_name -aggregate $aggregate_name -size $volsize_g -state online -policy default -unix-permissions ---rwxr-xr-x";
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command(
        'command'                => "$cmd",
        'connectrec-match_table' => $prompts_answers
    );
    sleep(10);
    my $prompts_answers1 = [ ".*to page down, .*" => ' ' ];
    $output = $Api_Set_Obj->execute_raw_command( 'command' => "vol show", 'connectrec-match_table' => $prompts_answers1 );

    if ( $output =~ /$vol_name/i ) {
        my ($res) = readable_disk_size( GB => $VOL_SIZE );
        logcomment("$FILER_C : Volume $vol_name of size $res created on Node $FILER_C");
    } else {
        logcomment("**FAIL** : $FILER_C -  $vol_name could NOT be created on node $FILER_C");
        return logresult( "INFO", msg => 1 );
    }
    $info_waf_plot_file{$FILER_C}{'VOL_SIZE'} = $volsize_g;

    ###########################################################
    # STEP 6 - Disable volume snapshot ploicy
    ###########################################################
    nvme_eye_catcher( Test => $Test, string => "STEP 6 of 13: $FILER_C : Disable volume snapshot policy" );
    ### Disabling snapshot policy
    logcomment("disabling volume policy");
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule daily -newcount 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule weekly -newcount 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule hourly -newcount 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;snap reserve -V $vol_name\"" );
    $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify -policy default -enabled false" );
    logcomment("Disabled snapshot policy");

    ## Recording out of all the settings of aggregate
    logcomment("Recording out of all the settings aggregate");
    $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;vol options $vol_name\"" );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;snap reserve -V $vol_name\"" );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;snap sched $vol_name\"" );

    logcomment("$FILER_C : Executing sysconfig -r to check drive assignment and layout of drives");
    $Api_Set_Obj->execute_command( 'command' => "sysconfig -r", 'connectrec-match_table' => $prompts_answers1 );

    ###########################################################
    ## STEP 7 - Start filersio thread on the volume created.
    ## Pass/Fail criteria: N/A
    ###########################################################
    nvme_eye_catcher(
        Test   => $Test,
        string => "STEP 7 of 13 : $FILER_C : Start filersio thread on the volume $vol_name",
    );

    ## Getting data drives from aggr status command
    my @drives_7Mode = ();    ## Datadrive array
    my $response;
    my $cmd        = "aggr status $aggregate_name -r";
    my $raw_output = $Api_Set_Obj->execute_command( 'command' => $cmd, 'connectrec-match_table' => $prompts_answers1 );
    my @lines      = split( '\n', $raw_output );
    foreach my $line (@lines) {

        if ( $line =~ /\s*data\s*(\w+\.\w+)\s*(.*)/i ) {
            my $drive = $1;
            logcomment("Drive is : $drive\n");
            $drive =~ s/P\d+$//g;
            push( @drives_7Mode, $drive );
        }
    }
    logcomment("Data drives in 7mode are : @drives_7Mode");

    ##Starting filersio thread on the volume created
    my $output_df = $Api_Set_Obj->execute_raw_command( 'command' => "run local df" );
    my ($vol_size_df) = ( $output_df =~ /\/vol\/$vol_name\/\s*\d+\s*\d+\s*(\d+)/i );
    logcomment("DEBUG : Volume : $vol_name size is : $vol_size_df");
    my $vol_size_df = $vol_size_df / ($FILERSIO_THREADS);
    my ( $res, $res1 ) = readable_disk_size( KB => $vol_size_df );
    $res =~ s/( |B|\.\d+)//ig;
    logcomment("DEBUG : Volume $vol_name size is $res - $res1");

    my $Api_Set_Obj_Sio = $Node->get_7m_or_nodescope_apiset( connid => "console" );
    $Api_Set_Obj_Sio->set_timeout( "connectrec-timeout" => 14400 );

    for ( my $i = 0 ; $i < $FILERSIO_THREADS ; $i++ ) {
        $output = $Api_Set_Obj_Sio->execute_command( 'command' => "filersio asyncio_active 0 -r 100 64k 0 $res -1 16 /vol/$vol_name/f$i -create" );
        if ( $output =~ /failed to start/ig ) {
            logcomment("**FAIL** : $FILER_C : filersio instanace $i FAILED to start");
            return logresult( "INFO", msg => 1 );
        } else {
            logcomment(" Filersio threads is started on instance $i");
        }

        ## Checking the filersio status
        my $filersio_stat = $Api_Set_Obj_Sio->execute_command( 'command' => "filersio status" );
        if ( $filersio_stat =~ /filersio: instance $i: asyncio_active running/i ) {
            logcomment( "$FILER_C : Thread " . ($i) . " : filersio is running" );
        } else {
            logcomment( "Filersio Thread " . ($i) . " is completed " );
        }
    }
    logcomment("Sleeping for a while before starting statistics");
    sleep(100);

    ## Check Loop Start
    my $start_time   = time;
    my $time_start   = 0;
    my $loop_count   = 1;
    my $step_counter = 1;
    my %waf_hash;
    my %ontap_waf_hash;
    my %time_value;
    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
    $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );

    while (1) {
        ######################################################################
        ## STEP 8 - Pre Data collection for WAF calculation
        ## Pass/Fail criteria: N/A
        ######################################################################
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 8($step_counter) of 13 : $FILER_C : Pre Data collection WAF calculation",
        );
        my %chain_hash;
        my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
        my $logfile = $actual_log_dir . '/' . "WAF_Plot_File_$FILER_C" . ".plt";
        logcomment("DEBUG: $FILER_C : combined log file path is : $logfile");
        open( INPUT, ">>", $logfile )
          or die "Could not open file $logfile: $!";

        ################## PRE DATA COLLECTION ##################
        ## 8a. Start statit
        logcomment("Starting statit");
        try {
            ## Beginning the statit
            my $response = $Api_Set_Obj->statit( 'beginning' => 1 );
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            my $exception_object = shift;
            my $resp             = $exception_object->get_response_object();
            logcomment( "Caught a " . ref($exception_object) . "!" );
            logcomment( "Error is: " . $exception_object->text() );
            logcomment( "Raw output was:\n" . $resp->get_raw_output() );
        };

        ## 8b. Start Disk Statistics
        logcomment("Starting Disk Statistics");
        my $sample_id;
        my $filr = $FILER_C;
        $filr =~ s/-/_/g;
        my $disks;
        my @drive_stat = ();

        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate show -aggregate $aggregate_name  -fields disklist", 'connectrec-match_table' => $prompts_answers );
        my $disk_list;
        if ( $out =~ /$aggregate_name\s*(.*)/ ) {
            $disk_list = $1;
        }
        my @disks_tmp = split( ',', $disk_list );
        my @data_disks = @disks_tmp[ 2 .. ( scalar(@disks_tmp) - 1 ) ];
        logcomment("DEBUG : $FILER_C : Data disks are : @data_disks");

        ## Deleting sample ids if present already
        my $smple_id = "sample_$filr";
        $Api_Set_Obj->execute_raw_command( 'command' => "statistics samples delete -sample-id $smple_id -force true" );

        foreach my $dsk (@data_disks) {
            $disks = $disks . $dsk . "|";
        }
        $disks =~ s/\|$//;    ## Removing last |

        $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "statistics start -object disk -sample-id sample_$filr -instance $disks",
            'connectrec-match_table' => $prompts_answers
        );
        if ( $output =~ /Statistics collection is being started for sample-id:\s+(\S+)/ ) {
            logcomment("$FILER_C : Statistics command for the selected disks has been started");
            $sample_id = $1;
            logcomment("DEBUG: Captured Sample -id $sample_id to Stop the Statistics");
        } else {
            logcomment("**FAIL** : $FILER_C - statistics command for the selected disks could NOT BE STARTED");
            return logresult( "INFO", msg => 1 );
        }

        logcomment("$FILER_C : Waiting for Statistics object to start on disk");
        sleep(60);

        ## 8c. Show Disk Statistics
        logcomment("Disk Statistics Show");
        $prompts_answers = [ ".*to page down, .*" => ' ' ];
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );
        $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "set -rows 0; statistics show -object disk -instance $disks -sample-id $sample_id",
            'connectrec-match_table' => $prompts_answers
        );
        if ( $output =~ /show failed/ ) {
            logcomment("**FAIL** : FAILED to show the statistics for disks $disks");
        }

        ## 8d. Read log sense page 0xC0 from data drives
        logcomment("Reading log sense page 0xC0 from data drives");
        my $return_before = scsi_logsense_page_data(
            'node'       => $Node,
            'drive_list' => \@drives_7Mode
        );
        if ( $return_before == 1 ) {
            logcomment("**FAIL** : $FILER_C : Scsi logsense error while fetching values from page 0xC0 at STEP 8d");
            return logresult( "INFO", msg => 1 );
        }

        ######################################################################
        ## STEP 9 - Check filersio status.Fail if not running
        ## Pass/Fail criteria: N/A
        ######################################################################
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 9($step_counter) of 13 : $FILER_C : Check filersio status.Fail if not running",
        );
        ## Wait for 15 mins
        logcomment("Soak time of 15 min");
        sleep 900;

        ## Check filersio status
        ## Checking volume size
        logcomment("Checking volume size");
        my $output_df = $Api_Set_Obj->execute_raw_command( 'command' => "run local df" );
        my ($vol_size_df) = ( $output_df =~ /\/vol\/$vol_name\/\s*\d+\s*\d+\s*(\d+)/i );
        logcomment("DEBUG : Volume : $vol_name size is : $vol_size_df");
        $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
        $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );
        my $vol_size_df = $vol_size_df / ($FILERSIO_THREADS);
        my ( $res, $res1 ) = readable_disk_size( KB => $vol_size_df );
        $res =~ s/( |B|\.\d+)//ig;
        logcomment("DEBUG : Volume $vol_name size is $res");
        $info_waf_plot_file{$FILER_C}{'FILER_IO_FILE_SIZE'} = $res;
        sleep 5;
        $Api_Set_Obj->execute_raw_command( 'command' => " " );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
        $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );
        sleep 3;

      filersio_retry_9:
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 14400 );
        $Api_Set_Obj->execute_command( 'command' => " " );
        my $filersio_stat = $Api_Set_Obj->execute_raw_command( 'command' => "filersio status" );
        for ( my $i = 0 ; $i < $FILERSIO_THREADS ; $i++ ) {
            if ( $filersio_stat =~ /filersio: instance $i: asyncio_active running/i ) {
                logcomment( "$FILER_C : Thread " . $i . " : filersio is running" );
            } elsif ( $filersio_stat !~ /filersio/ ) {
                logcomment("Filersio output is not proper, Re-execute ...");
                goto filersio_retry_9;
            } else {
                logcomment("$FILER_C :  FILERSIO_FAILURE/Volume Full ");
                my $params = NATE::ParamSet->new( global => 1 );
                my $LOGDIR = $params->get( 'LOGDIR', default => undef );
                my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
                my $log_dir = "$actual_log_dir\/FILERSIO_FAILURE_SYSTEM_LOGS\/" . $FILER_C;
                if ( !( -d "$log_dir" ) ) {
                    system("sudo mkdir -p $log_dir");
                }

                get_system_logs( Node => $Node, logdir => $log_dir );
                logcomment("$FILER_C : System logs copied to the directory : $log_dir");
                logcomment("**FAIL** : $FILER_C : filersio is not running");
                return logresult( "INFO", msg => 1 );
            }
        }

        ###################### POST DATA COLLECTION ######################
        ######################################################################
        ## STEP 10 - Post Data collection for WAF calculation
        ## Pass/Fail criteria: N/A
        ######################################################################
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 10($step_counter) of 13 : $FILER_C : Post Data collection for WAF calculation",
        );

        # 10.a Show Disk Statistics
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );
        logcomment("Show Disk Statistics");
        $prompts_answers = [ ".*to page down, .*" => ' ' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "exit" );
        $Api_Set_Obj->execute_raw_command( 'command' => "set -privilege test" );
        $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "set -rows 0;statistics show -object disk -instance $disks -sample-id $sample_id",
            'connectrec-match_table' => $prompts_answers
        );

        if ( $output =~ /show failed/ ) {
            logcomment("**WARNING** : FAILED to show the statistics for disks $disks");
        }

        ## Executing again to get host_write_blocks,physical_write_blocks and write_amplification_factor
        ## which will be displayed in step 11

        ## host_write_blocks to display
        my %ontap_statistics_hash;
        $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "set -rows 0;statistics show -object disk -instance $disks -sample-id $sample_id -fields value host_write_blocks",
            'connectrec-match_table' => $prompts_answers
        );
        my @output_lines = split( '\n', $output );
        logcomment("Disk statistics : $output");
        foreach my $line (@output_lines) {
            if ( $line =~ /disk\s*(\d+\.\d+\.\d+)\s*host_write_blocks\s*(\d+)/i ) {
                logcomment("Disk : $1 and host_write_blocks : $2");
                $ontap_statistics_hash{$1}{'host_write_blocks'} = $2;
            }
        }

        ## physical_write_blocks
        $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "set -rows 0;statistics show -object disk -instance $disks -sample-id $sample_id -fields value physical_write_blocks",
            'connectrec-match_table' => $prompts_answers
        );
        my @output_lines = split( '\n', $output );
        logcomment("Disk statistics : $output");
        foreach my $line (@output_lines) {
            my $disk;
            if ( $line =~ /disk\s*(\d+\.\d+\.\d+)\s*physical_write_blocks\s*(\d+)/i ) {
                logcomment("Disk : $1 and physical_write_blocks : $2");
                my $temp_disk   = $1;
                my $temp_pywrbl = $2;
                $ontap_statistics_hash{$temp_disk}{'physical_write_blocks'} = $temp_pywrbl;
            }
        }

        ## write_amplification_factor
        $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "set -rows 0;statistics show -object disk -instance $disks -sample-id $sample_id -fields value write_amplification_factor",
            'connectrec-match_table' => $prompts_answers
        );
        my @output_lines = split( '\n', $output );
        logcomment("Disk statistics : $output");
        foreach my $line (@output_lines) {
            if ( $line =~ /disk\s*(\d+\.\d+\.\d+)\s*write_amplification_factor\s*(\d+)/i ) {
                logcomment("Disk : $1 and write_amplification_factor : $2");
                my $temp_disk     = $1;
                my $temp_wrampfac = $2;
                $ontap_statistics_hash{$temp_disk}{'write_amplification_factor'} = $temp_wrampfac;
                my $disk;
                my @temp = split( '\.', $temp_disk );
                foreach my $dsk (@drives_7Mode) {
                    my @temp1 = split( '\.', $dsk );
                    if ( $temp[2] eq $temp1[1] ) {
                        $disk = $dsk;
                        last;
                    }
                }
                push( @{ $ontap_waf_hash{$disk}{'write_amplification_factor'} }, $temp_wrampfac );
            }
        }

        ## user_write_chain
        $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "set -rows 0;statistics show -object disk -instance $disks -sample-id $sample_id -fields value user_write_chain",
            'connectrec-match_table' => $prompts_answers
        );
        my @output_lines = split( '\n', $output );
        logcomment("Disk statistics : $output");
        foreach my $line (@output_lines) {
            if ( $line =~ /disk\s*(\d+\.\d+\.\d+)\s*user_write_chain\s*(\d+)/i ) {
                logcomment("Disk : $1 and user_write_chain : $2");
                my $temp_disk       = $1;
                my $temp_usrwrchain = $2;
                $ontap_statistics_hash{$temp_disk}{'user_write_chain'} = $temp_usrwrchain;
                my @temp = split( '\.', $temp_disk );
                my $disk;
                foreach my $dsk (@drives_7Mode) {
                    my @temp1 = split( '\.', $dsk );
                    if ( $temp[2] eq $temp1[1] ) {
                        $disk = $dsk;
                        last;
                    }
                }
                push( @{ $ontap_waf_hash{$disk}{'user_write_chain'} }, $2 );
            }
        }

        ## 10.b Read log page 0xC0 to get post_host_writes and post_physical writes
        my $return_after = scsi_logsense_page_data(
            'node'       => $Node,
            'drive_list' => \@drives_7Mode
        );
        if ( $return_after == 1 ) {
            logcomment("**FAIL** : $FILER_C : Scsi logsense error while fetching values from page 0xC0 at STEP 10b");
            return logresult( "INFO", msg => 1 );
        }

        ## Collecting stats
        # 10.c Stop statit
        $response = $Api_Set_Obj->statit(
            'ending' => 1,
            'report' => 1,
        );
        my $parsed_output = $response->get_parsed_output();
        foreach ( 0 .. ( $DRIVES_NUMBR - 1 ) ) {
            my ( $writes, $chain, $usecs );
            my $aggr_name          = '/' . $aggregate_name . '/plex0/rg0';
            my $disk               = $parsed_output->[0]->{$aggr_name}->[$_]->{'disk'};
            my $writes_chain_usecs = $parsed_output->[0]->{$aggr_name}->[$_]->{'writes_chain_usecs'};
            if ( $writes_chain_usecs =~ /(\S+)\s*(\S+)\s*(\S+)/ ) {
                $writes = $1;
                $chain  = $2;
                $usecs  = $3;
                logcomment("Disk : $disk writes : $writes chain : $chain usecs : $usecs");
                push( @{ $waf_hash{$disk}{'chain'} }, $chain );
                $chain_hash{$disk} = $chain;
            }
        }

        # 10.d Stop Disk Statistics
        $output = $Api_Set_Obj->execute_raw_command( 'command' => "statistics stop -sample-id $sample_id" );
        if ( $output =~ /(stopped\s*for\s*sample-id.*$sample_id)|("$sample_id"\s*already\s*stopped)/ig ) {
            logcomment("Statistics collection is being stopped for sample-id: $sample_id");
        } else {
            logcomment("**FAIL** : FAILED to stop the statstics  for sample-ID $sample_id");
            return logresult( "INFO", msg => 1 );
        }

        ######################################################################
        ## STEP 11 - Check for write chain on data drives less than 5
        ## Pass/Fail criteria: N/A
        ######################################################################
        nvme_eye_catcher(
            Test   => $Test,
            string => "STEP 11($step_counter) of 13 : $FILER_C : Check for write chain on data drives less than 5",
        );
        ## 11.a Check for write chain on data drives less than 5
        my $flag = 0;
        foreach my $drv (@drives_7Mode) {
            $flag++ if ( int( $chain_hash{$drv} ) < 5 );
        }

        ## 11.b Display ontap statistics
        logcomment("Displaying Ontap statistics");
        my @keys = keys %ontap_statistics_hash;
        @keys = unique(@keys);
        foreach my $dsk (@keys) {
            my $hostwriteblocks     = $ontap_statistics_hash{$dsk}{'host_write_blocks'};
            my $physicalwriteblocks = $ontap_statistics_hash{$dsk}{'physical_write_blocks'};
            my $amp_fact            = $ontap_statistics_hash{$dsk}{'write_amplification_factor'} . '%';
            logcomment("Disk : $dsk | host_write_blocks : $hostwriteblocks | physical_write_blocks : $physicalwriteblocks | write_amplification_factor : $amp_fact");
        }

        ## 11.c Calculating logpage WAF
        logcomment("Calculating logpage WAF");
        my $time_at_WAF_cal = scalar( localtime() );
        logcomment("Calculating WAF for data drives @drives_7Mode");
        foreach my $dsk (@drives_7Mode) {
            my $Pre_Host_writes      = $return_before->{$dsk}->{'Host_writes'};
            my $Post_Host_writes     = $return_after->{$dsk}->{'Host_writes'};
            my $Pre_Physical_writes  = $return_before->{$dsk}->{'Physical_writes'};
            my $Post_Physical_writes = $return_after->{$dsk}->{'Physical_writes'};
            logcomment("Pre_Host_writes:$Pre_Host_writes |Post_Host_writes:$Post_Host_writes |Pre_Physical_writes:$Pre_Physical_writes|Post_Physical_writes:$Post_Physical_writes");
            my $WAF = Waf_Calulation(
                'drive'                  => $dsk,
                'Host_writes_before'     => $Pre_Host_writes,
                'Host_writes_after'      => $Post_Host_writes,
                'Physical_writes_before' => $Pre_Physical_writes,
                'Physical_writes_after'  => $Post_Physical_writes
            );
            push( @{ $waf_hash{$dsk}{'waf'} }, $WAF );
        }
        my $end_time = time;
        $time_start = int( ( $end_time - $start_time ) / 3600 );
        if ( $loop_count == 1 ) {
            push( @{ $time_value{'time_lapsed'} }, "0" );
        } else {
            push( @{ $time_value{'time_lapsed'} }, $time_start );
        }

        my $resp1          = `dd if=/dev/null of=$logfile`;
        my $write_loop_cnt = 0;
        foreach my $tmp (@drives_7Mode) {
            if ( $write_loop_cnt == 0 ) {
                print INPUT "#DRIVE_MODEL = $info_waf_plot_file{$FILER_C}{'DRIVE_MDEL'}\n";
                print INPUT "#DRIVE_FW = $info_waf_plot_file{$FILER_C}{'FW'}\n";
                print INPUT "#SYSTEM = $info_waf_plot_file{$FILER_C}{'Model_Name'}\n";
                print INPUT "#FILER = $FILER_C\n";
                print INPUT "#FILER_IO_FILE_SIZE = $info_waf_plot_file{$FILER_C}{'FILER_IO_FILE_SIZE'}\n";
                print INPUT "#AGGR_SIZE = $info_waf_plot_file{$FILER_C}{'AGGR_SIZE'}\n";
                print INPUT "#VOLUME_SIZE = $info_waf_plot_file{$FILER_C}{'VOL_SIZE'}\n";
                print INPUT "#FILER_SIO_THREADS = $FILERSIO_THREADS\n";
                print INPUT "#FILER_IO = READ/WRITE(0/1)\n";
                $write_loop_cnt++;
            }
            print INPUT "\n# $tmp :\n";
            print INPUT 'ONTAP_WR_CHAIN [', join( ', ', @{ $ontap_waf_hash{$tmp}{'user_write_chain'} } ),           "\]\n";
            print INPUT 'ONTAP_WAF [',      join( ', ', @{ $ontap_waf_hash{$tmp}{'write_amplification_factor'} } ), "\]\n";
            print INPUT 'CHAIN=[',          join( ', ', @{ $waf_hash{$tmp}{'chain'} } ),                            "\]\n";
            print INPUT 'WAF=[',            join( ', ', @{ $waf_hash{$tmp}{'waf'} } ),                              "\]\n";
            print INPUT 'TIME=[', join( ', ', @{ $time_value{'time_lapsed'} } ), "\]\n";
            print INPUT 'Loop[', join( ', ', 1 .. $loop_count ), "\]\n\n";
        }
        close(INPUT);
        $step_counter++;
        $loop_count++;

        ## Last while(1) when write chain value for all drives becomes less than five
        if ( $flag eq scalar(@drives_7Mode) ) {
            logcomment("Write chain value of all drives is less than five so exiting from while loop");
            last;
        }

        ## Sleeping for 2hr before looping again
        logcomment("Sleeping for 2hr before looping again");
        sleep(7200);

    }    ## End of while(1)

    ###########################################################
    ## STEP 12 - Check filersio status
    ## Pass/Fail criteria: N/A
    ###########################################################
    nvme_eye_catcher(
        Test   => $Test,
        string => "STEP 12 of 13 : $FILER_C : Check filersio status",
    );
    $output = $Api_Set_Obj->execute_command( 'command' => "filersio status" );
    if ( $output =~ /(asyncio_active running|running)/ig ) {
        logcomment("filersio threads are running");
    }
    sleep 5;

    ###########################################################
    ## STEP 13 - Stop the filersio instance running
    ## Pass/Fail criteria: N/A
    ###########################################################
    nvme_eye_catcher(
        Test   => $Test,
        string => "STEP 13 of 13 : $FILER_C : Stop the filersio instance",
    );
    $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
    $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );
    $output = $Api_Set_Obj->execute_raw_command( 'command' => "filersio stop" );
    logcomment("Waiting for 5 minutes");
    sleep(300);

    $output .= $Api_Set_Obj->execute_raw_command( 'command' => "filersio status" );

    if ( $output =~ /(filersio:\s*aborting\s*instance\s*)|/ ) {
        logcomment("Filersio threads are stopped Successfully");
    } else {
        logcomment("**WARNING** : Filersio threads are NOT stopped ");
    }
    $Api_Set_Obj->execute_raw_command( 'command' => "exit" );
    return logresult( "INFO", msg => 0 );
}

#####################################################################
## Cleanup - Post Process
#####################################################################
sub cleanup() {
    logcomment("$filer_names - Clean up and post test process");

    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;
    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ##########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 14400 );
        logcomment("Stop the filersio instances if not stopeed");
        $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
        $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );
        my $output = $Api_Set_Obj->execute_raw_command( 'command' => "filersio stop" );
        logcomment("Waiting for 5 minutes");
        sleep(300);

        $output .= $Api_Set_Obj->execute_raw_command( 'command' => "filersio status" );
        if ( $output =~ /(filersio:\s*aborting\s*instance\s*)|/ ) {
            logcomment("Filersio threads are stopped Successfully");
        } else {
            logcomment("**WARNING** : Filersio threads are NOT stopped ");
        }
        $Api_Set_Obj->execute_raw_command( 'command' => "exit" );
        $Api_Set_Obj->execute_raw_command( 'command' => "statistics samples delete -sample-id * -force true" );
    }

    ## Deleting the non root aggregate created for this test
    logcomment("Deleting the non root aggregate created for this test");
    foreach my $Node (@Nodes) {
        aggregate_volume_delete( Node => $Node );
        vserver_delete( Node => $Node );
    }

    logcomment("Set disk autoassign to default");
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->execute_raw_command( 'command' => "storage disk option modify -node $FILER_C  -autoassign $autoassign{$FILER_C}" );
    }

    # nvme_post_test
    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );

    return $TCD::PASS;
}
########################## Private Functions #############################
### Subroutine to get logsense page data
sub scsi_logsense_page_data {
    my %options = @_;
    my $Node    = $options{'node'};
    my @drives  = @{ $options{'drive_list'} };
    my $dsk_result;
    my $Host_writes;
    my $Physical_writes;

    my $drv_result = {};
    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
    logcomment("Check Current Privilage Level ");
    $Api_Set_Obj->execute_raw_command( 'command' => "set" );
    logcomment("Set privilage level to diag");
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command( 'command' => "set diag", 'connectrec-match_table' => $prompts_answers );
    logcomment("Privilage level diag set");
    $Api_Set_Obj->execute_raw_command( 'command' => "set" );
    logcomment("Get password to login in system shell");
    my $FILER_C = $Node->name();
    my $Host    = host($FILER_C);
    my $Passwd  = $Host->default_password();
    logcomment("Filer password : $Passwd");
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    my $Api_Set_Obj_SS = NACL::APISet->new(
        hostobj   => $Host,
        category  => "Node",
        interface => "CLI",
        set       => "Systemshell"
    );
    logcomment("System shell API created, getting log pages ");
    my $ss_disk_list = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol devlist" );

    foreach my $line ( split( /\n/, $ss_disk_list ) ) {
        my ( $drv, $ns, $nsz );
        if ( $line =~ /(\S+)\:\s+(\S+)/ ) {
            $drv = $1;
            $drv_result->{$drv}->{'PRDID'} = $2;
        }
        if ( $line =~ /(\S+)ns\d+/ ) {
            $drv = $1;
            if ( $line =~ /(\S+)\s+\((\S+)\)/ ) {
                $ns                              = $1;
                $nsz                             = $2;
                $drv_result->{$drv}->{'NAME_SP'} = $ns;
                $drv_result->{$drv}->{'NS_SIZE'} = $nsz;
            }
        }
    }
    my @nv_drv = keys(%$drv_result);
    logcomment("System Shell Drive(s) : @nv_drv");
    my @data_drives_for_logsense;
    foreach my $nvdrv (@nv_drv) {
        foreach my $drv (@drives) {
            my @nvdrvsplit = split( 'peg_nvme', $nvdrv );
            my @drvsplit   = split( '0n.',      $drv );
            if ( $nvdrvsplit[1] eq $drvsplit[1] ) {
                push( @data_drives_for_logsense, $nvdrv );
            }
        }
    }
    @data_drives_for_logsense = unique(@data_drives_for_logsense);

    ### Getting Host writes and Physical writes
    my $it = each_array( @data_drives_for_logsense, @drives );
    while ( my ( $nvdrv, $drive ) = $it->() ) {
        logcomment("Recording 0xC0 log page data for drive $drive ");
        my $output = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol logpage -p 0xC0 $nvdrv " );
        logcomment("Recorded Firmware log page data for drive $drive ");

        ### Host_writes
        my $val_HW;
        if ( $output =~ /000:\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)/ ) {
            $val_HW = ( $5 . $6 );
        } elsif ( $output =~ /Scsi logsense error/i ) {
            return 1;
        }
        if ( $val_HW ne "" ) {
            no warnings;
            $Host_writes = hex("$val_HW");
            logcomment("For Drive $drive: Host Writes BL Value (HEX) = $val_HW and Host Writes BL Value (DEC) = $Host_writes");
            $dsk_result->{$drive}->{'Host_writes'} = $Host_writes;
        }

        ### Physical_writes
        my $val_PW;
        if ( $output =~ /000:\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)\s*(\S+)/ ) {
            $val_PW = ( $7 . $8 );
        } elsif ( $output =~ /Scsi logsense error/i ) {
            return 1;
        }
        if ( $val_PW ne "" ) {
            no warnings;
            $Physical_writes = hex("$val_HW");
            logcomment("For Drive $drive: Host Writes BL Value (HEX) = $val_HW and Physical Writes BL Value (DEC) = $Physical_writes");
            $dsk_result->{$drive}->{'Physical_writes'} = $Physical_writes;
        }
    }
    return $dsk_result;
}

####### Subroutine to calculating WAF #######
sub Waf_Calulation {
    my %options                = @_;
    my $drive                  = $options{'drive'};
    my $Host_writes_after      = $options{'Host_writes_after'};
    my $Host_writes_before     = $options{'Host_writes_before'};
    my $Physical_writes_after  = $options{'Physical_writes_after'};
    my $Physical_writes_before = $options{'Physical_writes_before'};
    my $PW_HR_15;

    ### Computing WAF
    if ( ( $Host_writes_after - $Host_writes_before ) != 0 ) {
        $PW_HR_15 = ( ( $Physical_writes_after - $Physical_writes_before ) / ( $Host_writes_after - $Host_writes_before ) );
        $PW_HR_15 = sprintf( "%.3f", $PW_HR_15 );
        logcomment("WAF Value for drive $drive = $PW_HR_15");
    } else {
        logcomment("WAF Value for drive $drive is null");
    }

    return $PW_HR_15;
}

sub unique {
    my @arr    = (@_);
    my @unique = do {
        my %seen;
        grep { !$seen{$_}++ } @arr;
    };
    return @unique;
}