#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/TEST_SCRIPTS/CMODE/NADQ02_NVMe_Th_Disk_IO_Latency.thpl#1 $
# Copyright (c) 2005 Network Appliance, Inc.
# All rights reserved
##
## @summary Mode
##  Performance testing and for sequential & random write/read latency
##  assessment - aligned and unaligned cases
##
## @description
##  Performance testing and for sequential & random write/read latency
##  assessment - aligned and unaligned cases
##
## @Test Mode
##  Maintenance system
##
## @Test bed setup
##  FC   : Dual Path Fabric, Cluster,  Cluster Fabric
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of an STEST.
##
## @steps
## The test will execute commands mentioned below:
##   1) read capacity of test drive
##
##   example:
##   *> scsi capacity 4a.00.20
##   30005842608 blocks x 520 bytes/blk = 15603038156160 bytes
##   max_cap = 15603038156160
##
##   3) Create two node access ranges
##
##   <node_1_lo> = 50000
##   <node_1_hi> = max_cap/2
##
##   <node_2_lo> = max_cap/2 + 2048
##   <node_2_hi> = max_cap - 100000
##
##   #######################################
##   4) Sequential Precondition
##   #######################################
##
##   disk_qual start -t 9 -d <drive_id> -o noverify nopattern unmap
##   disk_qual start -t 5 -n 512 -p 2 -d <drive_id> -o noverify nopattern
##
##   #######################################
##   ### Single Port Test (seq-read testing)
##   #######################################
##
##   5) Run Seq_Read disk_latency on node A:
##   disk_latency -p seq_read -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_1_lo> -h <node_1_hi> -s <blk_size> -f <drive_id>
##
##   #######################################
##   ### Single Port Test (seq-write testing)
##   #######################################
##
##   6) Run Seq_Write disk_latency on node A:
##   disk_latency -p seq_write -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_1_lo> -h <node_1_hi> -s <blk_size> -f <drive_id>
##
##   #######################################
##   ### Dual Port Test (seq-read testing)
##   ### Run nodes A and B at same time
##   #######################################
##
##   7) run seq-read disk_latency on node A:
##   disk_latency -p seq_read -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_1_lo> -h <node_1_hi> -s <blk_size> -f <drive_id>
##
##   8) run seq-read disk_latency on node B:
##   disk_latency -p seq_read -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_2_lo> -h <node_2_hi> -s <blk_size> -f <drive_id>
##
##   #######################################
##   ### Dual Port Test (seq-write testing)
##   ### Run nodes A and B at same time
##   #######################################
##
##   9) run seq-write disk_latency on node A:
##   disk_latency -p seq_write -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_1_lo> -h <node_1_hi> -s <blk_size> -f <drive_id>
##
##   10) run seq-write disk_latency on node B:
##   disk_latency -p seq_write -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_2_lo> -h <node_2_hi> -s <blk_size> -f <drive_id>
##
##   #######################################
##   11) Random Precondition
##   #######################################
##
##   disk_qual start -t 9 -d <drive_id> -o noverify nopattern unmap
##   disk_qual start -t 7 -n 1 -p 1 -d <drive_id> -o noverify nopattern
##
##   #######################################
##   ### Single Port Test (rand-read testing)
##   #######################################
##
##   12) run rand-read disk_latency on node A:
##   disk_latency -p rand_read -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_1_lo> -h <node_1_hi> -s <blk_size> -f <drive_id>
##
##   #######################################
##   ### Single Port Test (rand-write testing)
##   #######################################
##
##   13) run rand-write disk_latency on node A:
##   disk_latency -p rand_write -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_1_lo> -h <node_1_hi> -s <blk_size> -f <drive_id>
##
##   #######################################
##   ### Dual Port Test (rand-read testing)
##   # Run nodes A and B at same time
##   #######################################
##
##   14) run rand-read disk_latency on node A:
##   disk_latency -p rand_read -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_1_lo> -h <node_1_hi> -s <blk_size> -f <drive_id>
##
##   15) run rand-read disk_latency on node B:
##   disk_latency -p rand_read -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_2_lo> -h <node_2_hi> -s <blk_size> -f <drive_id>
##
##   #######################################
##   ### Dual Port Test (rand-write testing)
##   ### Run nodes A and B at same time
##   #######################################
##
##   16) run rand-write disk_latency on node A:
##   disk_latency -p rand_write -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_1_lo> -h <node_1_hi> -s <blk_size> -f <drive_id>
##
##   17) run rand-write disk_latency on node B:
##   disk_latency -p rand_write -q <que_lvl> -t 120 -a 8 -m $SGL -l <node_2_lo> -h <node_2_hi> -s <blk_size> -f <drive_id>
##
# @param FILER       - optional for cluster setup
##                    - required for Dual path setup      Name of filer to be used
## @param FILERA      - required for cluster setup
##                    - optional for Dual path setup      Name of filer to be used
## @param FILERB      - required for cluster setup
##                    - optional for Dual path setup      Name of the partner node
## @param TEST_CONFIG - optional Type of the setup, C for cluster, D for dual path (default)
## @param TEST_SETUP  - optional Default value set to 'FC'
## @param FILER_CONN  - optional Type of connection, default is set to 'console' for this test
## @param Mode        - optional Filer mode maint/normal, default set to 'maint' for this test
## @param FILER_PROMPT - optional prompt of the filer
## @param LOGDIR      - optional This is required to generate 'END-LOG'. Default set to main log directory.
## @param EOE         - optional 'default' is set to default value
## @param BOOT_MODE   - optional Boot mode for filer , default set to '1' for this test
## @param BOOT_TYPE   - optional Boot type, default set to 'A' (automatic) for this test
## @param SSD  	 - optional Setup is SSD or not, default set to 'no'
## @param EMAIL  - optional. This will indicate if the user wants to recieve mails for FAIL scenarios
## @param MAIL_TO - The email ids of the receiving users
## @param MAIL_FROM - The email id from where the above email will be send.
## @param FILER_TYPE - This is used to differentiate between BR and IC filers
##
## @status  Automated
## @author  khosur@netapp.com
## @burt    1215187
## @change  YY/MM/DD from author: Description of the change
##
##

#############################################################################
#########################################
#### Library functions
###########################################
# Compiler directives.
use strict;
use Subtest;

## Module imports
use TCD;
use NACL::APISet;
use NACL::C::Node;
use Storage::Common_Lib;
use Storage::Scsi qw(scsi_reserve);
use NATE::ParamSet;
use List::MoreUtils qw(each_array);
use Storage::NVMe_Common_Lib;
use Storage::Tahiti_Common_Lib;

## Global parameters
use vars qw(
  $FILER
  $TEST_CONFIG
  $TEST_SETUP
  $MODE
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $ARMADILLO
  $USER_SERIAL_NO
);
my $params = NATE::ParamSet->new( global => 1 );
$LOGDIR         = $params->get( 'LOGDIR',         default => '' );
$FILER          = $params->get( 'FILER',          default => '' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'D' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'FC' );
$MODE           = $params->get( 'MODE',           default => 'normal' );
$FILER_PROMPT   = $params->get( 'FILER_PROMPT',   default => '\*>' );
$EOE            = $params->get( 'EOE',            default => 'default' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'filer type' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$USER_SERIAL_NO = $params->get( 'USER_SERIAL_NO', default => '' );
$SSD            = $params->get( 'SSD',            default => '' );

my ( $email_subject, $email_body );
my $TC_name;

$TC_name = "701_NVMe_Th_Disk_IO_Latency";
if ( $ARMADILLO eq "1" ) {
    $TEST_SETUP = "FC";
}

########################################
### Initialization/declaration
#########################################
######################## Global variables###############################
my @Nodes;
my $filer_names;
my $Mode;
my $user_sel_drive;
my $drive;

# End log parameters

#Set test fail parameter to 0
my $test_status = 0;
my @temp_channels;
my %nodes_filer = ();
my %ApiSet_Objs = ();
my %disk_to_use = ();
my %channels;
my $drive_type;
my $pre_post_flag = 0;
my $node_1_lo;
my $node_1_hi;
my $node_2_lo;
my $node_2_hi;
my $target_filer;
my %system_ids = ();

my @q_level = qw(1 2 3 4 8 16 32);

#my @blk_size = qw(4160 8320 16640 33280 66560 133120 266240);
my @blk_size = qw(4160 8320 16640 33280 66560 133120 262144);
my $SGL      = "bcs";

########################################
# Testcase available for execution
#########################################
my @Testcases = ( NVME_Th_DISK_IO_LAT => "GENERAL PERFORMANCE TESTING" );

#########################################
## Pre-test processes
##########################################
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );    # Testcases will be executed using TCD object.
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main

########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    version_test( node_present => \@Nodes, tc_name => $TC_name );
    logcomment("User entered serial number  - $USER_SERIAL_NO");
    my @serial_nos = split( /\,/, $USER_SERIAL_NO );
    logcomment( "Filer- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "Filer- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        Tahiti          => "yes",
        filer_mode      => $Mode,
    );

    return $TCD::PASS;
}

########## TEST 1 ##################################################
#Scsi_PRO
####################################################################
sub NVME_Th_DISK_IO_LAT {
    $Test->description("GENERAL PERFORMANCE TESTING");
    logcomment("Starting execution of test steps. Total steps to be executed as part of the script is: 10");
    my @node_1_lo_hi;
    my @node_2_lo_hi;

    ## Checking the drive type
    my $disk_type = disk_type_maint( Node => $Nodes[0] );
    my $d_type;
    foreach my $dsk ( keys %$disk_type ) {
        $d_type = $disk_type->{$dsk}->{'Type'};
        if ($d_type) {
            logcomment("Disk type is $d_type");
            $SSD = 'Y';
            last;
        }
    }
    ###########################################################################################
    #Step 1 - Record kernel version
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        if ( $SSD =~ /Y/ ) {
            tahiti_eye_catcher( Test => $Test, string => "STEP 1 of 10 : $FILER_C : Get kernel version" );
        } else {
            tahiti_eye_catcher( Test => $Test, string => "STEP 1 of 8 : $FILER_C : Get kernel version" );
        }
        my $version_status = display_version( Node => $Node );
    }

    ###########################################################################################
    #Step 2 - check for 8020 model
    ###########################################################################################
    my $Node_test;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        if ( $SSD =~ /Y/ ) {
            tahiti_eye_catcher( Test => $Test, string => "STEP 2 of 10 : Check for System Model" );
        } else {
            tahiti_eye_catcher( Test => $Test, string => "STEP 2 of 8 Check for System Model" );
        }

        # my $disk_type = disk_type_maint( Node => $Node );
        # my $d_type;
        # foreach my $dsk ( keys %$disk_type ) {
        # $d_type = $disk_type->{$dsk}->{'Type'};
        # if ($d_type) {
        # logcomment("Disk type is $d_type");
        # last;
        # }
        # }
        my $return = sysconfig_v_data( node => $Node );
        $system_ids{$FILER_C} = $return->{system_id};
        if ( $d_type =~ /SSD/i ) {
            my $model = $return->{model};
            if ( ( $model =~ /[A-Za-z][-](\w+\d+\w+)/i ) || ( $model =~ /[A-Za-z](\d+)/ ) ) {
                my $model_id = $1;
                if ( $model_id < "8020" ) {
                    if ( $model_id =~ /(\D+)/i ) {
                        goto print_model;
                    }
                    logcomment("**WARNING** : $FILER_C: Model No. : $model . System model should be 8020 or GREATER THAN 8020");
                } else {
                  print_model:
                    logcomment("Model of $FILER_C is $model");
                }
            }
        } else {
            logcomment( "$FILER_C : Model No. : " . $return->{model} );
        }
    }

    ###########################################################################################
    #Step 3 -  Print Drive serial number user entered
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        if ( $SSD =~ /Y/ ) {
            tahiti_eye_catcher( Test => $Test, string => "STEP 3 of 10 : Print Serial number" );
        } else {
            tahiti_eye_catcher( Test => $Test, string => "STEP 3 of 8 : Print Serial number" );
        }
        logcomment("Serial number Entered by user - $USER_SERIAL_NO");
        last;
    }

    ###########################################################################################
    #Step 4 - ## GET DRIVE TYPE - SSD or SAS/OTHERS
    ###########################################################################################

    my $serno_exists = 0;
    my $flag         = 0;
    ### Creating ApiSet objects
    foreach my $Node (@Nodes) {
        my $FILER_C    = $Node->name();
        my $Host       = host($FILER_C);
        my $API_Object = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance"
        );
        $API_Object->set_timeout( "connectrec-timeout" => 900000 );
        $ApiSet_Objs{$FILER_C} = $API_Object;
    }

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Host    = host($FILER_C);
        if ( $SSD =~ /Y/ ) {
            tahiti_eye_catcher( Test => $Test, string => "STEP 4 of 10 : Get drive type and filer connection to be used" );
        } else {
            tahiti_eye_catcher( Test => $Test, string => "STEP 4 of 8 : Get drive type and filer connection to be used" );
        }

        ## Checking for USER_SERIAL_NO on filer
        logcomment("DEBUG : Checking for $USER_SERIAL_NO on filer :$FILER_C");

        my $disk_list_result = tahiti_list_only_tahiti_drv( node_present => [$Node] );
        my @disk_info = keys(%$disk_list_result);
        foreach my $dsk (@disk_info) {
            if ( $disk_list_result->{$dsk}->{'SERIAL'} eq $USER_SERIAL_NO ) {
                $disk_to_use{$Node} = $dsk;
            }
        }

        ## Get disks owned by node
        my @fil_drvs = tahiti_get_disk_owned( Node => $Node );

        my $disk_type = tahiti_get_all_disk( node_present => [$Node] );
        foreach my $dsk (@fil_drvs) {
            if ( $disk_list_result->{$dsk}->{'SERIAL'} eq $USER_SERIAL_NO ) {
                $serno_exists   = 1;
                $target_filer   = $FILER_C;
                $Node_test      = $Node;
                $user_sel_drive = $disk_list_result->{$dsk}->{'DISK'};
                $drive_type     = $disk_type->{$dsk}->{'TYPE'};
                $flag           = 1;
                last;
            }
        }
        last if $flag == 1;
    }
    if ( $flag == 1 ) {
        logcomment("Drive - $user_sel_drive has been selected by user which belongs to $target_filer system and of type $drive_type");
    } else {
        $test_status = 1;
        logcomment("**FAIL**: Serial number Entered by user: $USER_SERIAL_NO does not belongs to any node");
        return $TCD::FAIL;
    }

    ## Fetching user selected drive on both nodes
    foreach my $Node (@Nodes) {
        my $disk_list_result = tahiti_list_only_tahiti_drv( node_present => [$Node] );
        my @disk_info = keys(%$disk_list_result);
        foreach my $dsk (@disk_info) {
            if ( $disk_list_result->{$dsk}->{'SERIAL'} eq $USER_SERIAL_NO ) {
                $disk_to_use{ $Node->name() } = $dsk;
            }
        }
    }

    ###########################################################################################
    ## read capacity of test drive
    ###########################################################################################
    logcomment("Read capacity of test drive");
    my $max_cap;
    $drive = $disk_to_use{$target_filer};
    my $output = $ApiSet_Objs{$target_filer}->execute_raw_command( command => "scsi capacity $drive" );
    if ( $output =~ /=\s*(\d+) bytes/ ) {
        $max_cap = $1;
    } else {
        logcomment("**FAIL**: Capacity is not displayed properly. Cmnd output is : $output");
        return $TCD::FAIL;
    }
    logcomment("Drive capacity is : $max_cap");

    ###########################################################################################
    ## Create two node access ranges
    ###########################################################################################
    logcomment("Create two node access ranges");
    $node_1_lo    = 50000;
    $node_1_hi    = int( $max_cap / 2 );
    @node_1_lo_hi = ( $node_1_lo, $node_1_hi );
    logcomment("node 1 lo and hi values : @node_1_lo_hi");

    $node_2_lo    = int( $max_cap / 2 ) + 2048;
    $node_2_hi    = $max_cap - 100000;
    @node_2_lo_hi = ( $node_2_lo, $node_2_hi );
    logcomment("node 2 lo and hi values : @node_2_lo_hi");

    my $FILER_C = $target_filer;
    ###########################################################################################
    #     Step 5 - If SSD, Sequential Precondition
    ###########################################################################################
    if ( $drive_type =~ /SSD/ ) {
        tahiti_eye_catcher( Test => $Test, string => "STEP 5 of 10 : If drive type is SSD, Sequential Precondition drive" );
        logcomment("Drive type is SSD, Preconditioning drive - $drive");
        my $SGL = "bcs";
        logcomment("Available q size - @q_level\n");
        logcomment("Available blk_size - @blk_size\n");
        logcomment("SGL value $SGL will be used");

        # SEQUENTIAL PRECONDITION SSD DRIVE
        logcomment("Sequential Pre condition user entered drive");
        my @subtests;
        push( @subtests, subtest( \&seq_pre_condition_ssd_drive, -runid, "seq_pre_condition_ssd_drive_$FILER_C", -bg, "--", disks => [$drive], node => $Node_test ) );
        Subtest::wait_finish( subtest => [@subtests] );
        logcomment("Waiting for sometime to complete subtest");
        sleep 120;

        ## Remove ownership and assign user selected drive -s 999999999
        my $return_val = remove_ownership_and_assign_user_selected_drive();
        if ($return_val) {
            return $TCD::FAIL;
        }

        ###########################################################################################
        #  Performance Testing (loops q level and blksz level times)
        ###########################################################################################
        logcomment("Performance Testing (loops q level and blksz level times)");
        logcomment("Available q size - @q_level\n");
        logcomment("Available blk_size - @blk_size\n");
        logcomment("SGL value $SGL will be used");

        ###########################################################################################
        # SEQUENTIAL READ AND WRITE
        ###########################################################################################
        #######################################
        ### Single Port Test (seq-read testing)
        #######################################
        tahiti_eye_catcher( Test => $Test, string => "STEP 6 of 10 - Single Port SEQ READ and WRITE Performance Testing" );
        logcomment("Single Port Test seq-read");
        seq_read_ssd(
            API_Object => $ApiSet_Objs{$target_filer},
            disk       => $disk_to_use{$target_filer},
            lo         => $node_1_lo,
            hi         => $node_1_hi
        );

        logcomment("Single Port Test seq-write");
        seq_write_ssd(
            API_Object => $ApiSet_Objs{$target_filer},
            disk       => $disk_to_use{$target_filer},
            lo         => $node_1_lo,
            hi         => $node_1_hi
        );

        #######################################
        ### Dual Port Test (seq-read testing)
        ### Run nodes A and B at same time
        #######################################
        #run seq-read disk_latency on node A and node B:
        tahiti_eye_catcher( Test => $Test, string => "STEP 7 of 10 - Dual Port SEQ READ and WRITE Performance Testing" );
        logcomment("Dual Port Test seq-read");
        my @subtests;
        my $it = each_array( @Nodes, @node_1_lo_hi, @node_2_lo_hi );
        while ( my ( $Node, $lo, $hi ) = $it->() ) {
            my $FILER_C = $Node->name();
            logcomment("Node : $FILER_C | Node low range : $lo | Node high range : $hi");
            push(
                @subtests,
                subtest(
                    \&seq_read_ssd, -runid, "seq_read_ssd_$FILER_C", -bg, "--",
                    API_Object => $ApiSet_Objs{$FILER_C},
                    disk       => $disk_to_use{$FILER_C},
                    lo         => $lo,
                    hi         => $hi
                )
            );
        }
        Subtest::wait_finish( subtest => [@subtests] );

        logcomment("Dual Port Test seq-write");
        my @subtests;
        my $it = each_array( @Nodes, @node_1_lo_hi, @node_2_lo_hi );
        while ( my ( $Node, $lo, $hi ) = $it->() ) {
            my $FILER_C = $Node->name();
            push(
                @subtests,
                subtest(
                    \&seq_write_ssd, -runid, "seq_write_ssd_$FILER_C", -bg, "--",
                    API_Object => $ApiSet_Objs{$FILER_C},
                    disk       => $disk_to_use{$FILER_C},
                    lo         => $lo,
                    hi         => $hi
                )
            );
        }
        Subtest::wait_finish( subtest => [@subtests] );

        ## Assign drive back to node
        logcomment("Assign drive back to node");
        $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk remove_ownership $drive -f" );
        sleep(10);
        my $output = $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk show -n" );
        if ( $output !~ /$drive\s+Not Owned\s+/i ) {
            logcomment("**FAIL**: $target_filer : Failed to remove ownership of drive $drive.");
            return $TCD::FAIL;
        }
        my $sysid = $system_ids{$target_filer};
        $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk assign $drive -s $sysid" );

        # ################################################
        # If drive type is SSD, Random Precondition drive
        # ################################################
        tahiti_eye_catcher( Test => $Test, string => "STEP 8 of 10 : If drive type is SSD, Random Precondition drive" );
        logcomment("Drive type is SSD, Preconditioning drive - $drive");

        # RANDOM PRECONDITION SSD DRIVE
        logcomment("Random Pre condition user entered drive");
        my @subtests;
        push( @subtests, subtest( \&rand_pre_condition_ssd_drive, -runid, "rand_pre_condition_ssd_drive_$FILER_C", -bg, "--", disks => [$drive], node => $Node_test ) );
        Subtest::wait_finish( subtest => [@subtests] );
        logcomment("Waiting for sometime to complete subtest");
        sleep 120;

        ## Remove ownership and assign user selected drive -s 999999999
        my $return_val = remove_ownership_and_assign_user_selected_drive();
        if ($return_val) {
            return $TCD::FAIL;
        }

        ###########################################################################################
        # SNGLE PORT RANDOM READ AND WRITE
        ###########################################################################################
        tahiti_eye_catcher( Test => $Test, string => "STEP 9 of 10 - Single Port RAND READ and WRITE Performance Testing" );
        logcomment("Single Port Test rand-read");
        rand_read_ssd(
            API_Object => $ApiSet_Objs{$target_filer},
            disk       => $disk_to_use{$target_filer},
            lo         => $node_1_lo,
            hi         => $node_1_hi
        );

        logcomment("Single Port Test rand-write");
        rand_write_ssd(
            API_Object => $ApiSet_Objs{$target_filer},
            disk       => $disk_to_use{$target_filer},
            lo         => $node_1_lo,
            hi         => $node_1_hi
        );

        #######################################
        ### Dual Port Test (rand-read testing)
        ### Run nodes A and B at same time
        #######################################
        #run seq-read disk_latency on node A and node B:
        tahiti_eye_catcher( Test => $Test, string => "STEP 10 of 10 - Dual Port RAND READ and WRITE Performance Testing" );
        logcomment("Dual Port Test rand-read");
        my @subtests;

        my $it = each_array( @Nodes, @node_1_lo_hi, @node_2_lo_hi );
        while ( my ( $Node, $lo, $hi ) = $it->() ) {
            my $FILER_C = $Node->name();
            push(
                @subtests,
                subtest(
                    \&rand_read_ssd, -runid, "rand_read_ssd_$FILER_C", -bg, "--",
                    API_Object => $ApiSet_Objs{$FILER_C},
                    disk       => $disk_to_use{$FILER_C},
                    lo         => $lo,
                    hi         => $hi
                )
            );
        }
        Subtest::wait_finish( subtest => [@subtests] );

        #######################################
        ### Dual Port Test (rand-write testing)
        ### Run nodes A and B at same time
        #######################################
        logcomment("Dual Port Test rand-write");
        my @subtests;
        my $it = each_array( @Nodes, @node_1_lo_hi, @node_2_lo_hi );
        while ( my ( $Node, $lo, $hi ) = $it->() ) {
            my $FILER_C = $Node->name();
            push(
                @subtests,
                subtest(
                    \&rand_write_ssd, -runid, "rand_write_ssd_$FILER_C", -bg, "--",
                    API_Object => $ApiSet_Objs{$FILER_C},
                    disk       => $disk_to_use{$FILER_C},
                    lo         => $lo,
                    hi         => $hi
                )
            );
        }
        Subtest::wait_finish( subtest => [@subtests] );
        logcomment("Completed GENERAL PERFORMANCE testing for $drive");
    } else {    #End of SSD
        logcomment("Drive is NOT of SSD type, Performing SAS/SATA/FSAS performing test");
        logcomment("Drive $drive is of $drive_type type belongs to $target_filer");
        my @SATA_blk = qw(4096 8192 16384 32768 65536 131072 262144);

        #my @SAS_blk  = qw(4160 8320 16640 33280 66560 133120 266240);
        my @SAS_blk = qw(4160 8320 16640 33280 66560 133120 262144);
        my @q_level = qw(1 2 3 4 5 6 7 8);
        my $SGL;
        $SGL = "bcs" if ( ( $drive_type =~ /^SAS$/ ) || ( $drive_type =~ /^FSAS$/ ) );
        $SGL = "8_9_bcs" if ( $drive_type =~ /^BSAS$/ );
        $SGL = "4k"      if ( $drive_type =~ /^MSATA$/ );
        my @blk_ary;
        @blk_ary = @SAS_blk  if ( ( $drive_type =~ /^SAS$/i )  || ( $drive_type =~ /^FSAS$/i ) );
        @blk_ary = @SATA_blk if ( ( $drive_type =~ /^BSAS$/i ) || ( $drive_type =~ /^MSATA$/i ) );

        logcomment(" Drive type - $drive_type, Hence SGL value - $SGL");
        ###########################################################################################
        # STEP 5 - SEQ READ
        ###########################################################################################
        tahiti_eye_catcher( Test => $Test, string => "STEP 5 of 8 : Sequential Read" );
        foreach my $que (@q_level) {
            logcomment("Sequential Read for Queue - $que");
            foreach my $blk (@blk_ary) {
                logcomment("Sequential Read for queue $que and block size $blk for drive - $drive");
                $ApiSet_Objs{$target_filer}->execute_raw_command( command => "disk_latency -p seq_read -q $que -t 60 -i 0 -m $SGL -l 832000 -s $blk $drive" );
                sleep 3;
            }
        }

        ###########################################################################################
        # Step 6 - RND READ
        ###########################################################################################
        tahiti_eye_catcher( Test => $Test, string => "STEP 6 of 8 : Random Read " );
        foreach my $que (@q_level) {
            logcomment("Random Read for Queue - $que");
            foreach my $blk (@blk_ary) {
                logcomment("Random Read for queue $que and block size $blk for drive - $drive");
                $ApiSet_Objs{$target_filer}->execute_raw_command( command => "disk_latency -p rand_read -q $que -t 60 -i 0 -m $SGL -a 8 -s $blk $drive" );
                sleep 3;
            }
        }
        ###########################################################################################
        # Step 7 - SEQ WRITE
        ###########################################################################################
        tahiti_eye_catcher( Test => $Test, string => "STEP 7 of 8 : Sequential Write " );
        foreach my $que (@q_level) {
            logcomment("Sequential write for Queue - $que ");
            foreach my $blk (@blk_ary) {
                logcomment("Sequential Read for Que - $que and block size $blk for drive - $drive");
                $ApiSet_Objs{$target_filer}->execute_raw_command( command => "disk_latency -p seq_write -q $que -t 60 -i 0 -m $SGL -l 832000 -s $blk -f $drive" );
                sleep 3;
            }
        }
        ###########################################################################################
        # Step 8 - RND WRITE
        ###########################################################################################
        tahiti_eye_catcher( Test => $Test, string => "STEP 8 of 8 : Random Write " );
        foreach my $que (@q_level) {
            logcomment("Random Read for Queue - $que ");
            foreach my $blk (@blk_ary) {
                logcomment("Random Read for Que - $que and block size $blk for drive - $drive");
                $ApiSet_Objs{$target_filer}->execute_raw_command( command => "disk_latency -p rand_write -q $que -t 60 -i 0 -m $SGL -a 8 -s $blk -f $drive" );
                sleep 3;
            }
        }
    }

###########################################################################################
## Test log/results
###########################################################################################

    if ( $test_status == 0 ) {
        logcomment( "$filer_names : $TC_name : completed : " . scalar( localtime() ) );
        return $TCD::PASS;
    } else {
        logcomment("FAIL : test status - $test_status");
        logcomment( "$filer_names : FAIL : test status - $test_status" . scalar( localtime() ) );
        return $TCD::FAIL;
    }
}
####################################################################
#Cleanup
####################################################################
sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    $Test->description("Preforming Post test process - Check filer state, Check disk, check root volume,halt hammer, halt scrub, filer config");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;
    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ###########################################################################################
    ## Assign drive back to node
    logcomment("Assign drive back to node");
    $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk remove_ownership $drive -f" );
    sleep(10);
    my $output = $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk show -n" );
    if ( $output !~ /$drive\s+Not Owned\s+/i ) {
        logcomment("**FAIL**: $target_filer : Failed to remove ownership of drive $drive.");
        return $TCD::FAIL;
    }
    logcomment("output : $output");
    my $sysid = $system_ids{$target_filer};
    my $output = $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk assign $drive -s $sysid" );
    if ( $output !~ /Changing ownership of disk/ ) {
        logcomment("**FAIL**: $target_filer : Failed to assign drive $drive.");
        return $TCD::FAIL;
    }
    nvme_post_test(
        filer_mode      => $Mode,
        node_present    => [@Nodes],
        Test            => $Test,
        Tahiti          => "yes",
        change_state_to => "MAINT",
    );

    return $TCD::PASS;
}

## Private Functions
####################################################################
#    seq_read_ssd
####################################################################
sub seq_read_ssd {
    my %options    = @_;
    my $API_Object = $options{API_Object};
    my $disk_used  = $options{disk};
    my $node_lo    = $options{lo};
    my $node_hi    = $options{hi};

    if ( $drive_type =~ /SSD/ ) {
        logcomment("Available q size - @q_level\n");
        logcomment("Available blk_size - @blk_size\n");
        logcomment("SGL value $SGL will be used");
        logcomment("Node low range $node_lo will be used");
        logcomment("Node hign range $node_hi will be used");

        foreach my $que (@q_level) {
            logcomment("Sequential Read for Queue - $que");
            foreach my $blk (@blk_size) {
                logcomment("Sequential Read for queue $que and block size $blk for drive - $disk_used");
                my $output = $API_Object->execute_raw_command( command => "disk_latency -p seq_read_dd -e -q $que -t 120 -a 8 -m $SGL -l $node_lo -h $node_hi -s $blk -f $disk_used" );
                sleep 3;
                if ( $output =~ /did not match any disks/i ) {
                    logcomment("**FAIL**: seq_read disk_latency command failed for queue $que and block size $blk for drive - $disk_used");
                    return logresult( "INFO", msg => 1 );
                }
            }
        }
    }
}

####################################################################
#    seq_write_ssd
####################################################################
sub seq_write_ssd {
    my %options    = @_;
    my $API_Object = $options{API_Object};
    my $disk_used  = $options{disk};
    my $node_lo    = $options{lo};
    my $node_hi    = $options{hi};
    if ( $drive_type =~ /SSD/ ) {
        logcomment("Available q size - @q_level\n");
        logcomment("Available blk_size - @blk_size\n");
        logcomment("SGL value $SGL will be used");
        logcomment("Node low range $node_lo will be used");
        logcomment("Node hign range $node_hi will be used");

        foreach my $que (@q_level) {
            logcomment("Sequential Read for Queue - $que");
            foreach my $blk (@blk_size) {
                logcomment("DEBUG : Sequential Read for queue $que and block size $blk for drive - $disk_used");
                my $output = $API_Object->execute_raw_command( command => "disk_latency -p seq_write_dd -e -q $que -t 120 -a 8 -m $SGL -l $node_lo -h $node_hi -s $blk -f $disk_used" );
                sleep 3;
                if ( $output =~ /did not match any disks/i ) {
                    logcomment("**FAIL**: seq_write disk_latency command failed for queue $que and block size $blk for drive - $disk_used");
                    return logresult( "INFO", msg => 1 );
                }
            }
        }
    }
}

####################################################################
#    rand_read_ssd
####################################################################
sub rand_read_ssd {
    my %options    = @_;
    my $API_Object = $options{API_Object};
    my $disk_used  = $options{disk};
    my $node_lo    = $options{lo};
    my $node_hi    = $options{hi};
    if ( $drive_type =~ /SSD/ ) {
        logcomment("Available q size - @q_level\n");
        logcomment("Available blk_size - @blk_size\n");
        logcomment("SGL value $SGL will be used");
        logcomment("Node low range $node_lo will be used");
        logcomment("Node hign range $node_hi will be used");

        foreach my $que (@q_level) {
            logcomment("DEBUG : Sequential Read for Queue - $que");
            foreach my $blk (@blk_size) {
                logcomment("Sequential Read for queue $que and block size $blk for drive - $disk_used");
                my $output = $API_Object->execute_raw_command( command => "disk_latency -p rand_read_dd -e -q $que -t 120 -a 8 -m $SGL -l $node_lo -h $node_hi -s $blk -f $disk_used" );
                sleep 3;
                if ( $output =~ /did not match any disks/i ) {
                    logcomment("**FAIL**: rand_read disk_latency command failed for queue $que and block size $blk for drive - $disk_used");
                    return logresult( "INFO", msg => 1 );
                }
            }
        }
    }
}

####################################################################
#    rand_write_ssd
####################################################################
sub rand_write_ssd {
    my %options    = @_;
    my $API_Object = $options{API_Object};
    my $disk_used  = $options{disk};
    my $node_lo    = $options{lo};
    my $node_hi    = $options{hi};
    if ( $drive_type =~ /SSD/ ) {
        logcomment("Available q size - @q_level\n");
        logcomment("Available blk_size - @blk_size\n");
        logcomment("SGL value $SGL will be used");
        logcomment("Node low range $node_lo will be used");
        logcomment("Node hign range $node_hi will be used");

        foreach my $que (@q_level) {
            logcomment("Sequential Read for Queue - $que");
            foreach my $blk (@blk_size) {
                logcomment("Sequential Read for queue $que and block size $blk for drive - $disk_used");
                my $output = $API_Object->execute_raw_command( command => "disk_latency -p rand_write_dd -e -q $que -t 120 -a 8 -m $SGL -l $node_lo -h $node_hi -s $blk -f $disk_used" );
                sleep 3;
                if ( $output =~ /did not match any disks/i ) {
                    logcomment("**FAIL**: rand_write disk_latency command failed for queue $que and block size $blk for drive - $disk_used");
                    return logresult( "INFO", msg => 1 );
                }
            }
        }
    }
}

####################################################################
#    remove_ownership_and_assign_user_selected_drive
####################################################################
sub remove_ownership_and_assign_user_selected_drive {

    ## Remove ownership and assign user selected drive -s 999999999
    my $sysid;
    $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk remove_ownership $drive -f" );
    sleep(10);
    my $output = $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk show -n" );
    logcomment("output : $output");
    if ( $output !~ /$drive\s+Not Owned\s+/i ) {
        logcomment("$target_filer : Drive $drive ownership is not yet removed.");
        logcomment("Check again after sometime");
    }
    sleep(30);
    ## Check disk ownership again
    $output = $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk show -n" );
    if ( $output !~ /$drive\s+Not Owned\s+/i ) {
        logcomment("**FAIL**: $target_filer : Failed to remove ownership of drive $drive.");
        return 1;
    }
    $ApiSet_Objs{$target_filer}->execute_raw_command( 'command' => "disk assign $drive -s 999999999" );

    ## Check drive ownership on Filer A and Filer B
    foreach my $node (@Nodes) {
        my $nodename = $node->name();
        my $op_of_diskshow = $ApiSet_Objs{$nodename}->execute_raw_command( 'command' => "disk show -a" );
        my $drv;
        foreach my $line ( split( /\n/, $op_of_diskshow ) ) {
            $drv = $disk_to_use{$nodename};
            if ( ( $line =~ /$drv\s+\((\d+)\)/ ) ) {
                $sysid = $1;
                last;
            }
        }
        if ( $sysid ne '999999999' ) {
            logcomment("**FAIL**: $nodename : Drive $drv is not assigned to sysid 999999999, Please check");
            return 1;
        }
    }

    return 0;
}