#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_4_2/NADQ02_Multi_Stream_Wrt_Perf.thpl#1 $
# Copyright (c) 2005 Network Appliance, Inc.
# All rights reserved
##
## @summary Mode
##  SCSI and Persistent Reservation Operations
##
## @description
##  Verify that the drive supports the SCSI Reserve/Release commands and the
##  complete Persistent Reservation command set properly, as required for use in a
##  Network Appliance Filer
##
## @Test Mode
##  File system
##
## @Test bed setup
##  FC   : Dual Path Fabric, Cluster,  Cluster Fabric
##  SATA : NA
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of an STEST.
##
# Test overview:
# - verify basic VPD and stream control support
# - Main test concept:
#   - divide drive into 8 equal size partitons based on max-capacity
#   - The lower 4 partitions are accessed on by filer-A and the other 4 by filer-
#   - filer-A access via pri-PHY and filer-B access via sec-PHYs (or vice-versa - no restrictions here)
#	- Create 12 different ssh and issue all command in parlel
#
# Overview of test sequences:
#     test           PRI    SEC (PHYs being accessed)
# -----------------------------------
# non_MSW_test_1:     1      0
# non_MSW_test_1:     1      1
# non_MSW_test_1:     2      2
# non_MSW_test_1:     3      3
# non_MSW_test_1:     4      4
# disk zero spares
#     MSW_test_1:     1      0
#     MSW_test_1:     1      1
#     MSW_test_1:     2      2
#     MSW_test_1:     3      3
#     MSW_test_1:     4      4
#=======================================================================================================

##
## @status  Automated
## @author  arunak@netapp.com
## @burt    986510
## @change  YY/MM/DD from author: Description of the change
## @change  khosur : Modified for Burt 827153
##

#############################################################################
#########################################
#### Library functions
###########################################
# Compiler directives.
use strict;
use Subtest;

## Module imports
use TCD;
use NACL::APISet;
use NACL::C::Node;
use NACL::Transit;
use Storage::Common_Lib;
use Storage::NVMe_Common_Lib;
use Storage::Scsi qw(scsi_test scsi_inquiry scsi_capacity scsi_readdefect_b scsi_readdefect scsi_reset scsi_format scsi_reserve);
use NATE::ParamSet;
use List::MoreUtils qw(each_array);

## Global parameters
use vars qw(
  $FILER
  $TEST_CONFIG
  $TEST_SETUP
  $MODE
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $ARMADILLO
  $MSW_SERIAL_NO
);
my $params = NATE::ParamSet->new( global => 1 );
$LOGDIR         = $params->get( 'LOGDIR',         default => '' );
$FILER          = $params->get( 'FILER',          default => '' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'D' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'FC' );
$MODE           = $params->get( 'MODE',           default => 'normal' );
$FILER_PROMPT   = $params->get( 'FILER_PROMPT',   default => '\*>' );
$EOE            = $params->get( 'EOE',            default => 'default' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$SSD            = $params->get( 'SSD',            default => 'no' );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'filer type' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$MSW_SERIAL_NO  = $params->get( 'MSW_SERIAL_NO',  default => 'RANDOM' );

my ( $email_subject, $email_body );
my $TC_name;
my $flag = 0;

$TC_name = "521_NADQ02_NVMe_Multi_Stream_Wrt_Perf";

########################################
### Initialization/declaration
#########################################
######################## Global variables###############################
my @Nodes;
my $filer_names;
my $Mode;

my $test_status = 0;
my %spare       = ();
my %nodes_filer = ();
my $NODES_FILER = {};
my $Spare_to_test;
my @NODE;
my %sel_filer_drv  = ();
my %prt_off_drv    = ();
my %drvie_with_prt = ();

################################################################
#Variables for Disk Latency ssh command to be executed later
#non multi-stream
my ( $non_msw_p1_1, $non_msw_p1_2, $non_msw_p1_3, $non_msw_p1_4 );
my ( $non_msw_p2_1, $non_msw_p2_2, $non_msw_p2_3, $non_msw_p2_4 );

#multi stream
my ( $msw_p1_1, $msw_p1_2, $msw_p1_3, $msw_p1_4 );
my ( $msw_p2_1, $msw_p2_2, $msw_p2_3, $msw_p2_4 );

#Values -  PART 1
my ( $msw1_p1_s, $msw2_p1_s, $msw3_p1_s, $msw4_p1_s );
my ( $msw1_p1_e, $msw2_p1_e, $msw3_p1_e, $msw4_p1_e );

#Values -  PART 2
my ( $msw1_p2_s, $msw2_p2_s, $msw3_p2_s, $msw4_p2_s );
my ( $msw1_p2_e, $msw2_p2_e, $msw3_p2_e, $msw4_p2_e );

#Support values
my ( $wr_size, $tetris_per_LAA, $TWO_GB, $tsec );
################################################################
################################################################
#SSH GLB variable for each filer
my ( $SSH_FILER_A_1, $SSH_FILER_A_2, $SSH_FILER_A_3, $SSH_FILER_A_4 );
my ( $SSH_FILER_B_1, $SSH_FILER_B_2, $SSH_FILER_B_3, $SSH_FILER_B_4 );

################################################################
########################################
# Testcase available for execution
#########################################
my @Testcases = ( Multi_Stream_Wrt_Perf => "MULTI-STREAM WRITE PERFORMANCE TESTING" );

#########################################
## Pre-test processes
##########################################
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );    # Testcases will be executed using TCD object.
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main

########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");

    version_test( node_present => \@Nodes, tc_name => $TC_name );

    logcomment( "Filer- $filer_names : $TC_name : started, expected max completion time 2 Hrs : " . scalar( localtime() ) );
    logcomment( "Filer- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );
    logcomment("Filer- $filer_names : $TC_name will be executed on $MSW_SERIAL_NO spare drive");

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");

    #Brinf to maintenance and set storage release
    logcomment("Bring filer to maintenance mode and release ownership of all drives");
    my @subtests;
    foreach my $node (@Nodes) {
        my $filer_name = $node->name();
        my $Transit_obj = NACL::Transit->new( name => $filer_name );
        push(
            @subtests,
            subtest(
                \&nvme_boot_in_req_mode,                    -runid,
                "boot_filer_in_req_state_" . "$filer_name", -bg,
                "--",
                Trans   => $Transit_obj,
                boot_to => 'MAINT',
            )
        );
    }
    Subtest::wait_finish( subtest => [@subtests] );

    foreach my $node (@Nodes) {
        my $filer_name  = $node->name();
        my $Transit_obj = NACL::Transit->new( name => $filer_name );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("DEBUG:Filer $filer_name is in state : $filer_state");
        my $API_Obj_Maint;
        my $Host = host($filer_name);
        $API_Obj_Maint = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance",
        );

        $API_Obj_Maint->set_timeout( "connectrec-timeout" => 1200 );
        logcomment("Filer - $filer_name : Release all disk ownership");
        $API_Obj_Maint->execute_raw_command( 'command' => "storage release disks -f" );
    }

    logcomment("Halt filer and \"setenv use-resv? false\" ");
    my @subtests_1;
    foreach my $node (@Nodes) {
        my $filer_name = $node->name();
        my $Transit_obj = NACL::Transit->new( name => $filer_name );
        push(
            @subtests_1,
            subtest(
                \&nvme_boot_in_req_mode,                          -runid,
                "boot_filer_in_req_state_loader" . "$filer_name", -bg,
                "--",
                Trans   => $Transit_obj,
                boot_to => 'FIRMWARE',
            )
        );
    }
    Subtest::wait_finish( subtest => [@subtests_1] );

    foreach my $node (@Nodes) {
        my $FILER = $node->name();
        my $Transit_obj = NACL::Transit->new( name => $FILER );
        $Transit_obj->set( 'timeout' => 600, maxcycle => 20 );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("Filer - $FILER state is $filer_state");
        if ( $filer_state =~ /FIRMWARE/ ) {
            my $host = Tharn::host($FILER);
            my $prompt_fw;
            my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ];
            my $prompt_array   = $host->prompt_fw();
            my $prompts        = join( "|", @$prompt_array );
            $prompt_fw = "($prompts)";
            logcomment("DEBUG:This is the prompt regex: $prompt_fw");
            my $Server = new Server( no_cli => 1 );
            $Server->set_execution_server( -Server => $FILER );    #set the execution server to the current node
            logcomment("Filer is in Loader prompt, setting use-resv? to false");
            $Server->console(
                cmd                        => 'setenv use-resv? false;saveenv',
                additional_prompts_timeout => 60,
                ignore                     => $ignore_strings,
                additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
            );

            logcomment("$FILER : Environment variable saved");
        } else {
            logcomment("Filer is NOT in Loader, Cehck state");
        }
    }
    logcomment("Boot filer in CLI and proceed with test");

    my @subtests_11;
    foreach my $node (@Nodes) {
        my $filer_name = $node->name();
        my $Transit_obj = NACL::Transit->new( name => $filer_name );
        push(
            @subtests_11,
            subtest(
                \&nvme_boot_in_req_mode,                       -runid,
                "boot_filer_in_req_state_cli" . "$filer_name", -bg,
                "--",
                Trans   => $Transit_obj,
                boot_to => 'CLI',
            )
        );
    }
    Subtest::wait_finish( subtest => [@subtests_11] );
    foreach my $node (@Nodes) {
        my $FILER = $node->name();
        logcomment("Check state of filers ");
        my $Transit_obj = NACL::Transit->new( name => $FILER );
        $Transit_obj->set( 'timeout' => 600, maxcycle => 20 );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("Filer - $FILER state is $filer_state");
        logcomment("Set privilage set ");
        if ( $filer_state =~ /CLI/ ) {
            my $API_Obj = $node->get_7m_or_nodescope_apiset( connid => "console" );
            $API_Obj->execute_raw_command( command => " " );
            $API_Obj->set_timeout( "connectrec-timeout" => 1200 );
            my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
            $API_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        } else {
            logcomment("Filers is NOT in CLI or UP state");
        }
    }

####### Get Selected Drvie ##########

    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    my $spar_chek;
    logcomment("Check for user selected $MSW_SERIAL_NO spare drive");

    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $Api_Set_Obj  = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $FILER_C      = $Node->name();
        my @spare_drives = nvme_get_local_spares( API_con => $Api_Set_Obj, Filer => $FILER_C );
        if ( $MSW_SERIAL_NO =~ /RANDOM/i ) {
            logcomment("User has selected random spare drive for testing");
            logcomment("NDATE will pick random single drive from available spare pool");
            $Spare_to_test = $spare_drives[0];
            $spar_chek     = 0;
        } else {
            logcomment("Check drive with serial number $MSW_SERIAL_NO provided by user is available");
            logcomment("User has provided spare drive serial number : $MSW_SERIAL_NO");
            logcomment("Test will be executed on drive with $MSW_SERIAL_NO serial number ");
            logcomment("Get all drive serial number");
            my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
            my $disk_list_result = nvme_get_all_disk( node_present => [$Node] );
            logcomment("Checking drive with serial number $MSW_SERIAL_NO in available spare pool");
            foreach my $sp_drv (@spare_drives) {

                if ( $disk_list_result->{$sp_drv}->{'SERIAL'} eq $MSW_SERIAL_NO ) {
                    logcomment("$FILER_C : Drive - $sp_drv with serial number $MSW_SERIAL_NO found");
                    logcomment("$FILER_C : Drive - $sp_drv will be selected");
                    $Spare_to_test = $sp_drv;
                    $spar_chek     = 0;
                    last;
                } else {
                    $spar_chek = 1;
                }
            }
        }
        if ( $spar_chek == 0 ) {
            logcomment( "$FILER_C : Spare Drives present are " . scalar @spare_drives . " and the drives are : @spare_drives" ) if (@spare_drives);
            logcomment("$FILER_C : Spare drive $Spare_to_test will be used for testing");
            logcomment("$FILER_C : Spare drive $Spare_to_test belongs to $FILER_C");
            $NODES_FILER->{$FILER_C} = $Spare_to_test;
            last;
        } else {
            logcomment("Selected Spare drive serial $MSW_SERIAL_NO could NOT be found on $FILER_C, checking in other node");
        }
    }    #End of foreach
    if ( $spar_chek == 1 ) {
        logcomment("User entered serial $MSW_SERIAL_NO drive could NOT be found in spare list");
        logcomment("Test Cannot execute without spare drives");
        $test_status = 1;
        logcomment("**FATAL** :User entered serial $MSW_SERIAL_NO drive could NOT be found in spare list");
        logresult( "FATAL", "USER ENTERED SERIAL $MSW_SERIAL_NO DRIVE COULD NOT BE FOUND IN SPARE LIST" );
    }

    #### check primary port of drive selected
    ( my $sel_drv = $Spare_to_test ) =~ s/^(\w*\.)//;
    logcomment("Checking Primary port of the drive selected $sel_drv");
    my @prim_ports;
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C = $Node->name();
        logcomment("$FILER_C : Get Primary port for drive $sel_drv");
        my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        $API_Obj->execute_raw_command( command => " " );
        $API_Obj->set_timeout( "connectrec-timeout" => 1200 );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $API_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        my $disk_list_result = nvme_get_all_disk( node_present => [$Node] );
        my @disk_list = keys(%$disk_list_result);

        foreach my $temp_drv (@disk_list) {
            ( my $temp = $temp_drv ) =~ s/^(\w*\.)//;
            my $selected_drv_1 = $disk_list_result->{$temp_drv}->{'DISK'};
            if ( $temp =~ /$sel_drv/ ) {
                logcomment("$FILER_C : Selected drive ::::: $selected_drv_1");
                $prt_off_drv{$FILER_C} = $selected_drv_1;
                logcomment("Executing storage show disk -p $selected_drv_1 on filer : $FILER_C");
                my $storage_out = $API_Obj->execute_raw_command( command => "run local \"priv set -q tet;storage show disk -p $selected_drv_1\"" );
                foreach my $line ( split( /\n/, $storage_out ) ) {
                    if ( $line =~ /Pri\s+Port\:\s+(\S+)/ ) {
                        my $pri_port = $1;
                        logcomment("Drive $selected_drv_1 primary port is : $pri_port");
                        push( @prim_ports, $pri_port );
                        $drvie_with_prt{$selected_drv_1} = $pri_port;
                    }
                }
            }
        }
    }    # end of foreach nodes

    ( my $sel_drv = $Spare_to_test ) =~ s/^(\w*\.)//;
    logcomment("Drive : $sel_drv has @prim_ports as Primary ports");
    logcomment("NVMe Drive Type Selected, Hence there's no port for drives, Proceed with next steps");
    my $ret_prt_chk = check_port(@prim_ports);

    ##################################################################tt#
    # Pre test proces : call for pre_n_post test process
    #####################################################################
    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );
    if ( $test_status == 1 ) {
        return $TCD::FAIL;
    } else {
        return $TCD::PASS;
    }
}

########## TEST ####################################################
#Multi Stream Write
####################################################################
sub Multi_Stream_Wrt_Perf {
    my $JOFA = 0;
    $Test->description("MULTI-STREAM WRITE PERFORMANCE TESTING");
    logcomment("Starting execution of test steps. Total steps to be executed as part of the script is: X-X");

    ####################################################################
    #Disable media and raid srub before starting test
    ####################################################################
    eye_catcher( Test => $Test, string => "Disable raid-scrub and media-scrub on both nodes" );
    logcomment("Disable scrub on both nodes");
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C = $Node->name();
        logcomment("Disable raid and media scrub on filer $FILER_C ");
        my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        $API_Obj->execute_raw_command( command => " " );
        $API_Obj->set_timeout( "connectrec-timeout" => 1200 );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $API_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        logcomment("Disabling raid-scrub");
        $API_Obj->execute_raw_command( 'command' => "storage raid-options show -node $FILER_C -name raid.scrub.enable" );
        $API_Obj->execute_raw_command( command   => " " );
        $API_Obj->execute_raw_command( 'command' => "storage raid-options modify -node $FILER_C -name raid.scrub.enable off" );
        $API_Obj->execute_raw_command( 'command' => "storage raid-options show -node $FILER_C -name raid.scrub.enable" );
        logcomment("Disabled raid-scrub");

        logcomment("Disabling media-scrub");
        $API_Obj->execute_raw_command( 'command' => "storage raid-options show -node $FILER_C -name raid.media_scrub.enable" );
        $API_Obj->execute_raw_command( command   => " " );
        $API_Obj->execute_raw_command( 'command' => "storage raid-options modify -node $FILER_C -name raid.media_scrub.enable off" );
        $API_Obj->execute_raw_command( 'command' => "storage raid-options show -node $FILER_C -name raid.media_scrub.enable" );
        logcomment("Disabled media-scrub");

        logcomment("Disabling media-scrub on spares");
        $API_Obj->execute_raw_command( 'command' => "storage raid-options show -node $FILER_C -name raid.media_scrub.spares.enable" );
        $API_Obj->execute_raw_command( command   => " " );
        $API_Obj->execute_raw_command( 'command' => "storage raid-options modify -node $FILER_C -name raid.media_scrub.spares.enable off" );
        $API_Obj->execute_raw_command( 'command' => "storage raid-options show -node $FILER_C -name raid.media_scrub.spares.enable" );
        logcomment("Disabled spare media-scrub");
    }

    ####################################################################
    #STEP 1 - GET SPARE DRIVE TO TEST
    ####################################################################
    eye_catcher( Test => $Test, string => "STEP 1 of 14 : List user selected or random picked  spare drives used for testing" );
    logcomment(" Spare Drive : $Spare_to_test will be used for testing ");
    logcomment("Get owner of spare drive - $Spare_to_test ");
    my $owner;
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $found   = 0;
        my $FILER_C = $Node->name();
        my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        logcomment("Drive selected for test - $Spare_to_test");
        my ($selected_drv) = $Spare_to_test =~ /(\w+\.\w+)/;
        logcomment("Get selected drive for list of disk owned by $FILER_C - Selected drive $selected_drv");
        my $disk_list_result = nvme_get_all_disk( node_present => [$Node] );
        my @dsk              = keys %$disk_list_result;
        my @list_of_disk     = sort(@dsk);
        logcomment("Drives : @list_of_disk");

        foreach my $dsk (@list_of_disk) {
            if ( $dsk =~ /$selected_drv$/ ) {
                logcomment("Drive Found on $FILER_C : $dsk");
                $sel_filer_drv{$FILER_C} = $dsk;
                $found = 1;
            }
        }
        if ( $found == 0 ) {
            logcomment("slected drive $selected_drv CANNOT be found on filer $FILER_C");
        }
        logcomment("Filer :  $FILER_C Selected Drive - $sel_filer_drv{$FILER_C}");
    }

    my @check_adp_drv;
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C = $Node->name();
        my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        $API_Obj->set_timeout( "connectrec-timeout" => 30000 );
        logcomment("DEBUG : check console messages");
        $API_Obj->execute_raw_command( command => " " );
        $API_Obj->execute_raw_command( command => "\013" );
        logcomment("Execute sysconfig -v ");
        $API_Obj->execute_raw_command( command => "run local \"priv set -q test;sysconfig -v\"" );
        logcomment("sysconfig -v captured for filer $FILER_C");
    }
    logcomment("DEBUG - @Nodes will be used for execution");

####################################################################
    # CHECK FOR SELECTED SPARE DRIVE OWNER
####################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        logcomment("Drive selected for test - $sel_filer_drv{$FILER_C}");

    }

####################################################################
    #STEP 4 - DETERMINE PARTITION SIZES
####################################################################
    eye_catcher( Test => $Test, string => "STEP 4 of 14 : Determine partition sizes to use and create disk latency commands" );
    foreach my $Node (@Nodes) {
        my $maxlba;
        my $blcko;
        $Node->refresh_command_interface();
        my $FILER_C = $Node->name();
        my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );

        #Get MAX LBA
        logcomment("$FILER_C : Get max LBA ");
        my $drv = $sel_filer_drv{$FILER_C};
        my $output = $API_Obj->execute_raw_command( command => "run local \"priv set -q test;scsi capacity $drv\"" );

        #if ( $output =~ /(\S+)\s+blocks/ ) {
        if ( $output =~ /(\S+)\s+blocks\s+x\s+(\S+)\s+bytes\// ) {
            $maxlba = $1;
            $blcko  = $2;
            logcomment("Max LBA - $maxlba");
            logcomment("Bytes/blk - $blcko");
        } else {
            logcomment("FAILED to capture Max LBA ");
        }

        #Determine 50% max capacity of drive
        my $main_part1_s = 50000;
        my $main_part2_e = $maxlba - 500000;
        my $main_part1_e = int( ( $main_part2_e - $main_part1_s ) / 2 );
        my $main_part2_s = $main_part1_e + 1;
        logcomment("$FILER_C : Determined main partition values");
        logcomment("$FILER_C : \n\tMain PART_1 START : $main_part1_s\n\tMain PART_1 END :$main_part1_e\n\tMain PART_2 START : $main_part2_s\n\tMain PART_2 END: $main_part2_e");

        #Calculate partition size for 8 streams
        my $msw_lbas = int( ( $main_part1_e - $main_part1_s ) / 4 );
        logcomment("FILER_C : Partition size for 8 streams - $msw_lbas");

        #calculate stream start and end LBA's for lower 4 partition
        $msw1_p1_s = 50000;
        $msw1_p1_e = $msw1_p1_s + $msw_lbas;
        $msw2_p1_s = $msw1_p1_e + 1;
        $msw2_p1_e = $msw2_p1_s + $msw_lbas;
        $msw3_p1_s = $msw2_p1_e + 1;
        $msw3_p1_e = $msw3_p1_s + $msw_lbas;
        $msw4_p1_s = $msw3_p1_e + 1;
        $msw4_p1_e = $msw4_p1_s + $msw_lbas;
        logcomment("$FILER_C : Determined Start and end LBA for lower 4 partition");
        logcomment("Lower partition 1 : Start LBA - $msw1_p1_s : End LBA - $msw1_p1_e");
        logcomment("Lower partition 2 : Start LBA - $msw2_p1_s : End LBA - $msw2_p1_e");
        logcomment("Lower partition 3 : Start LBA - $msw3_p1_s : End LBA - $msw3_p1_e");
        logcomment("Lower partition 4 : Start LBA - $msw4_p1_s : End LBA - $msw4_p1_e");

        # calc stream start and end LBAs for upper 4 partitions
        $msw1_p2_s = $msw4_p1_e + 1;
        $msw1_p2_e = $msw1_p2_s + $msw_lbas;
        $msw2_p2_s = $msw1_p2_e + 1;
        $msw2_p2_e = $msw2_p2_s + $msw_lbas;
        $msw3_p2_s = $msw2_p2_e + 1;
        $msw3_p2_e = $msw3_p2_s + $msw_lbas;
        $msw4_p2_s = $msw3_p2_e + 1;
        $msw4_p2_e = $msw4_p2_s + $msw_lbas;
        logcomment("$FILER_C : Determined Start and end LBA for upper 4 partition");
        logcomment("Upper partition 1 : Start LBA - $msw1_p2_s : End LBA - $msw1_p2_e");
        logcomment("Upper partition 2 : Start LBA - $msw2_p2_s : End LBA - $msw2_p2_e");
        logcomment("Upper partition 3 : Start LBA - $msw3_p2_s : End LBA - $msw3_p2_e");
        logcomment("Upper partition 4 : Start LBA - $msw4_p2_s : End LBA - $msw4_p2_e");

        # Calculate LAA information for each partition - for reference
        # Current LAA is based on (2GB) TWO_GB size
        $wr_size = 266240;    # 256KB Full teris size
        $TWO_GB  = 2**31;     # align to 2GB
        $tsec    = 120;       # number of seconds to run test
        my $max_cap    = $maxlba * $blcko;         # capacity of drive in bytes
        my $max_cap_gb = int( $max_cap / 1e9 );    # convert to GB

        my $part_cap    = ( $main_part1_e - $main_part1_s ) * $blcko;    # this is the size of 1 (of 8) partitions in bytes
        my $part_cap_gb = int( $part_cap / 1e9 );                        # convert to GB

        my $msw_cap    = $msw_lbas * $blcko;
        my $msw_cap_gb = int( $msw_cap / 1e9 );

        my $LAA_part = int( ( $msw_cap * 4 ) / $TWO_GB );
        my $LAA_msw = int( $msw_cap / $TWO_GB );
        $tetris_per_LAA = int( $TWO_GB / 266240 );

        logcomment("$FILER_C : LAA Partition - $LAA_part : LAA MSW - $LAA_msw");
        ####################################################################
        #STEP 5 - Save test information to log
        ####################################################################

        eye_catcher( Test => $Test, string => "STEP 5 of 14 : Save test information to log" );
        my $main_log_dir;
        if ( $LOGDIR =~ /(\S+)\/HDD\/(\S+)/ ) {
            $main_log_dir = $1;
        } else {
            $main_log_dir = $LOGDIR;
        }
        logcomment("Logdir is $main_log_dir");
        my $test_info_log = "$main_log_dir/MSW_TEST_INFO.txt";
        logcomment("Test information log will be saved under - $test_info_log");

        #        open( FOUT, ">$test_info_log" ) || logresult( "FATAL", "Cannot open the file $test_info_log" );
        open( FOUT, ">$test_info_log" ) || logresult( "FATAL", "CANNOT OPEN THE FILE $test_info_log" );

        ####################################################################
        ####### Display test information
        ####################################################################
        print FOUT "\n\nMSW Test Information";
        print FOUT "\n------------------------------------------------------------------------";
        print FOUT "\nDrive capacity                        = $max_cap ($max_cap_gb GB)";
        print FOUT "\nMax Drive LBA                         = $maxlba";
        print FOUT "\nBytes/blk								= $blcko";
        print FOUT "\nMain Partition Capacity (2 partitions)= $part_cap ($part_cap_gb GB)";
        print FOUT "\n1 MSW Partition capacity              = $msw_cap ($msw_cap_gb GB)";
        print FOUT "\n2GB LAA's per 1 MSW partition         = $LAA_msw ";
        print FOUT "\nTetris per MSW (Full=256KB)           = $tetris_per_LAA ";

        #
        # display partition 1 and 2 LBAs for reference
        #
        print FOUT "\n\nStarting and Ending LBAs for all 8 partitions";
        print FOUT "\nmsw1_p1_s                             = $msw1_p1_s ";
        print FOUT "\nmsw1_p1_e                             = $msw1_p1_e ";
        print FOUT "\nmsw2_p1_s                             = $msw2_p1_s ";
        print FOUT "\nmsw2_p1_e                             = $msw2_p1_e ";
        print FOUT "\nmsw3_p1_s                             = $msw3_p1_s ";
        print FOUT "\nmsw3_p1_e                             = $msw3_p1_e ";
        print FOUT "\nmsw4_p1_s                             = $msw4_p1_s ";
        print FOUT "\nmsw4_p1_e                             = $msw4_p1_e ";

        print FOUT "\nmsw1_p2_s                             = $msw1_p2_s ";
        print FOUT "\nmsw1_p2_e                             = $msw1_p2_e ";
        print FOUT "\nmsw2_p2_s                             = $msw2_p2_s ";
        print FOUT "\nmsw2_p2_e                             = $msw2_p2_e ";
        print FOUT "\nmsw3_p2_s                             = $msw3_p2_s ";
        print FOUT "\nmsw3_p2_e                             = $msw3_p2_e ";
        print FOUT "\nmsw4_p2_s                             = $msw4_p2_s ";
        print FOUT "\nmsw4_p2_e                             = $msw4_p2_e ";

        ####################################################################
        ###### End of displaying test information
        ####################################################################
        logcomment("info file $test_info_log created");
        last;
    }    #End Of foreach STEP 4

    #build command for NON MSW -
    my $node_count = @Nodes;
    logcomment("Node count : $node_count");
    my $part = 0;
    for ( my $n = 0 ; $n <= $node_count ; $n++ ) {
        my $Node = $Nodes[$n];
        logcomment("aDEBUG=: node - $n");
        if ( $n == 0 ) {
            $part = 1;
            my $FILER_C = $Node->name();
            logcomment("Build NON-MSW commands for filer - $FILER_C");
            my $drv_2_use = $sel_filer_drv{$FILER_C};
            logcomment("Drive to use on filer $FILER_C is $drv_2_use");
            logcomment("DEBUG - Part - $part : $FILER_C");

            # Create disk latency command to execute on both filer in next step using ssh
            # non MSW part 1
            $non_msw_p1_1 = "run local \"priv set test; disk_latency -p rand_seq_write -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw1_p1_s -h $msw1_p1_e -t $tsec -f $drv_2_use\"";
            $non_msw_p1_2 = "run local \"priv set test; disk_latency -p rand_seq_write -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw2_p1_s -h $msw2_p1_e -t $tsec -f $drv_2_use\"";
            $non_msw_p1_3 = "run local \"priv set test; disk_latency -p rand_seq_write -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw3_p1_s -h $msw3_p1_e -t $tsec -f $drv_2_use\"";
            $non_msw_p1_4 = "run local \"priv set test; disk_latency -p rand_seq_write -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw4_p1_s -h $msw4_p1_e -t $tsec -f $drv_2_use\"";
            $part++;
        }
        if ( $n == 1 ) {
            my $FILER_C = $Node->name();
            logcomment("DEBUG - Part - $part : $FILER_C");
            logcomment("Build NON-MSW commands for filer - $FILER_C");
            my $drv_2_use = $sel_filer_drv{$FILER_C};
            logcomment("Drive to use on filer $FILER_C is $drv_2_use");
            logcomment("DEBUG - Part - $part : $FILER_C");

            # non MSW part
            $non_msw_p2_1 = "run local \"priv set test; disk_latency -p rand_seq_write -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw1_p2_s -h $msw1_p2_e -t $tsec -f $drv_2_use\"";
            $non_msw_p2_2 = "run local \"priv set test; disk_latency -p rand_seq_write -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw2_p2_s -h $msw2_p2_e -t $tsec -f $drv_2_use\"";
            $non_msw_p2_3 = "run local \"priv set test; disk_latency -p rand_seq_write -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw3_p2_s -h $msw3_p2_e -t $tsec -f $drv_2_use\"";
            $non_msw_p2_4 = "run local \"priv set test; disk_latency -p rand_seq_write -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw4_p2_s -h $msw4_p2_e -t $tsec -f $drv_2_use\"";
        }
    }    #End of NON MSW for loop

    logcomment("Command built for : FILER_A : Part 1");
    logcomment("p1_1 : $non_msw_p1_1");
    logcomment("p1_2 : $non_msw_p1_2");
    logcomment("p1_3 : $non_msw_p1_3");
    logcomment("p1_4 : $non_msw_p1_4");
    logcomment("Command built for : FILER_B : Part 2 ");
    logcomment("p2_1 : $non_msw_p2_1");
    logcomment("p2_2 : $non_msw_p2_2");
    logcomment("p2_3 : $non_msw_p2_3");
    logcomment("p2_4 : $non_msw_p2_4");

    logcomment("Completed building NON-MSW commands");

    ####################################################################
    #STEP 6 - Open 4 SSH sessions on each filer
    ####################################################################

    # $SSH_FILER_A_1 .. $SSH_FILER_A_4 - FILER A
    # $SSH_FILER_B_1 .. $SSH_FILER_B_4 - FILER B
    # Create individual variable for each connection (8 variables) in future each will have different connection parameters.
    eye_catcher( Test => $Test, string => "STEP 6 of 14 : Open 4 SSH session on each filer" );
    my $visited    = 1;
    my $node_count = @Nodes;
    for ( my $n = 0 ; $n <= $node_count ; $n++ ) {
        my $Node = $Nodes[$n];
        logcomment("aDEBUG=: node - $n");
        if ( $n == 0 ) {
            my $FILER_C  = $Node->name();
            my $time_out = 43200;           #12 Hrs timeout
            logcomment("$FILER_C : Creating 4  SSH connection for Filer $FILER_C");
            $SSH_FILER_A_1 = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
            $SSH_FILER_A_1->set_timeout( "connectrec-timeout" => $time_out );
            $Node->refresh_command_interface();
            logcomment("$FILER_C : Created SSH 1 - $SSH_FILER_A_1");
            $SSH_FILER_A_2 = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
            $SSH_FILER_A_2->set_timeout( "connectrec-timeout" => $time_out );
            $Node->refresh_command_interface();
            logcomment("$FILER_C : Created SSH 2 -  $SSH_FILER_A_2");
            $SSH_FILER_A_3 = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
            $SSH_FILER_A_3->set_timeout( "connectrec-timeout" => $time_out );
            $Node->refresh_command_interface();
            logcomment("$FILER_C : Created SSH 3 -  $SSH_FILER_A_3");
            $SSH_FILER_A_4 = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
            $SSH_FILER_A_4->set_timeout( "connectrec-timeout" => $time_out );
            $Node->refresh_command_interface();
            logcomment("$FILER_C : Created SSH 4 -  $SSH_FILER_A_4");
            logcomment("$FILER_C : Created 4 SSH sessions ");
            $visited = 2;
        }
        if ( $n == 1 ) {
            my $FILER_C  = $Node->name();
            my $time_out = 43200;           #12 Hrs timeout
            logcomment("$FILER_C : Creating 4  SSH connection for Filer $FILER_C");
            $SSH_FILER_B_1 = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
            $SSH_FILER_B_1->set_timeout( "connectrec-timeout" => $time_out );
            $Node->refresh_command_interface();
            logcomment("$FILER_C : Created SSH 1 - $SSH_FILER_B_1");
            $SSH_FILER_B_2 = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
            $SSH_FILER_B_2->set_timeout( "connectrec-timeout" => $time_out );
            $Node->refresh_command_interface();
            logcomment("$FILER_C : CCreated SSH 2 -  $SSH_FILER_B_2");
            $SSH_FILER_B_3 = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
            $SSH_FILER_B_3->set_timeout( "connectrec-timeout" => $time_out );
            $Node->refresh_command_interface();
            logcomment("$FILER_C : CCreated SSH 2 -  $SSH_FILER_B_3");
            $SSH_FILER_B_4 = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
            $SSH_FILER_B_4->set_timeout( "connectrec-timeout" => $time_out );
            logcomment("$FILER_C : Created SSH 4 -  $SSH_FILER_B_4");
            logcomment("$FILER_C : Created 4 SSH sessions ");
        }
    }
    logcomment("**DEBUG** : SSH connections created ");
    logcomment("**DEBUG** : Filer A - $SSH_FILER_A_1 | $SSH_FILER_A_2 | $SSH_FILER_A_3 | $SSH_FILER_A_4");
    logcomment("**DEBUG** : Filer B - $SSH_FILER_B_1 | $SSH_FILER_B_2 | $SSH_FILER_B_3 | $SSH_FILER_B_4");

####################################################################
    #STEP 7 - Execute NON-MSW test
####################################################################
    eye_catcher( Test => $Test, string => "STEP 7 of 14 : Execute NON-MSW tests" );
    foreach my $Node (@NODE) {
        my $FILER_C = $Node->name();
        my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        logcomment("$FILER_C : Make sure no streams are open on filer $FILER_C");
        my $output = $API_Obj->execute_raw_command( command => "run local \"priv set -q test;scsi stream_status $Spare_to_test\"" );
        if ( $output =~ /No Streams open/i ) {
            logcomment("$FILER_C : No Streams open on this device");
        } else {
            logcomment("$FILER_C : Some of the streams are open on this device");
            logcomment("**FATAL** : $FILER_C : Some of the STREAMS are OPEN on this device");
            $test_status = 1;
        }
    }

    #non-MSW Test 1: PRI=1, SEC=0
    eye_catcher( Test => $Test, string => "NON-MSW TEST 1 : PRI=1 , SEC=0" );
    logcomment("Starting NON-MSW Test 1 - PRI=1 , SEC=0");
    my $output = $SSH_FILER_A_1->execute_raw_command( command => "$non_msw_p1_1" );
    logcomment("End of NON-MSW Test 1 - PRI=1 , SEC=0");

    #non-MSW Test 2: PRI=1, SEC=1";
    eye_catcher( Test => $Test, string => "NON-MSW TEST 2 : PRI=1 , SEC=1" );
    my @m_subtest2;
    push( @m_subtest2, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_2_P1_S1_1", -bg, "--", $SSH_FILER_A_1, $non_msw_p1_1 ) );
    push( @m_subtest2, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_2_P1_S1_2", -bg, "--", $SSH_FILER_B_1, $non_msw_p2_1 ) );
    logcomment("Starting NON-MSW Test 2 : PRI=1 , SEC=1");
    Subtest::wait_finish( subtest => [@m_subtest2] );
    logcomment("End of NON-MSW Test 2 : PRI=1 , SEC=1");

    #non-MSW Test 3: PRI=2, SEC=2";
    eye_catcher( Test => $Test, string => "NON-MSW TEST 3 : PRI=2 , SEC=2" );
    my @m_subtest3;
    push( @m_subtest3, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_3_P2_S2_1", -bg, "--", $SSH_FILER_A_1, $non_msw_p1_1 ) );
    push( @m_subtest3, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_3_P2_S2_2", -bg, "--", $SSH_FILER_A_2, $non_msw_p1_2 ) );
    push( @m_subtest3, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_3_P2_S2_1", -bg, "--", $SSH_FILER_B_1, $non_msw_p2_1 ) );
    push( @m_subtest3, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_3_P2_S2_2", -bg, "--", $SSH_FILER_B_2, $non_msw_p2_2 ) );
    logcomment("Starting NON-MSW Test 3 : PRI=2 , SEC=2");
    Subtest::wait_finish( subtest => [@m_subtest3] );
    logcomment("End of NON-MSW Test 3 : PRI=2 , SEC=2");

    #non-MSW Test 4: PRI=3, SEC=3";
    eye_catcher( Test => $Test, string => "NON-MSW TEST 4 : PRI=3 , SEC=3" );
    my @m_subtest4;
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_4_P3_S3_1", -bg, "--", $SSH_FILER_A_1, $non_msw_p1_1 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_4_P3_S3_2", -bg, "--", $SSH_FILER_A_2, $non_msw_p1_2 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_4_P3_S3_3", -bg, "--", $SSH_FILER_A_3, $non_msw_p1_3 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_4_P3_S3_1", -bg, "--", $SSH_FILER_B_1, $non_msw_p2_1 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_4_P3_S3_2", -bg, "--", $SSH_FILER_B_2, $non_msw_p2_2 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_4_P3_S3_3", -bg, "--", $SSH_FILER_B_3, $non_msw_p2_3 ) );
    logcomment("Starting NON-MSW Test 4 : PRI=3 , SEC=3");
    Subtest::wait_finish( subtest => [@m_subtest4] );
    logcomment("End of NON-MSW Test 4 : PRI=3 , SEC=3");

    #non-MSW Test 5 : PRI=4 , SEC=4";
    eye_catcher( Test => $Test, string => "NON-MSW TEST 5 : PRI=4 , SEC=4" );
    my @m_subtest5;
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_5_P4_S4_1", -bg, "--", $SSH_FILER_A_1, $non_msw_p1_1 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_5_P4_S4_2", -bg, "--", $SSH_FILER_A_2, $non_msw_p1_2 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_5_P4_S4_3", -bg, "--", $SSH_FILER_A_3, $non_msw_p1_3 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_A_TST_5_P4_S4_4", -bg, "--", $SSH_FILER_A_4, $non_msw_p1_4 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_5_P4_S4_1", -bg, "--", $SSH_FILER_B_1, $non_msw_p2_1 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_5_P4_S4_2", -bg, "--", $SSH_FILER_B_2, $non_msw_p2_2 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_5_P4_S4_3", -bg, "--", $SSH_FILER_B_3, $non_msw_p2_3 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "NON_MSW_FILER_B_TST_5_P4_S4_4", -bg, "--", $SSH_FILER_B_4, $non_msw_p2_4 ) );
    logcomment("Starting NON-MSW Test 5 : PRI=4 , SEC=4");
    Subtest::wait_finish( subtest => [@m_subtest5] );
    logcomment("End of NON-MSW Test 4 : PRI=4 , SEC=4");

####################################################################
    #STEP 8 - Check for any failed drive
####################################################################
    eye_catcher( Test => $Test, string => "STEP 8 of 14 : Check for failed drives" );
    foreach my $Node (@Nodes) {
        my $FILER_C  = $Node->name();
        my $API_obj  = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $time_out = "6000000";
        $API_obj->set_timeout( "connectrec-timeout" => $time_out );
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );
        sleep 5;
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );

        logcomment("$FILER_C : Checking failed drive on filer $FILER_C ");
        my $sys_config_data = Storage::Common_Lib::sysconfig_v_data( node => $Node );
        my $failed_ar       = $sys_config_data->{'faileddisks'};
        my @failed_drvs     = @$failed_ar;
        my $normal_drv      = $sys_config_data->{'normaldisks'};
        my @nor_drv         = @$normal_drv;
        my $sys_id          = $sys_config_data->{'system_id'};
        logcomment("FILER : $FILER_C : System ID : $sys_id");
        logcomment("Drives present - @nor_drv");
        logcomment("Check for Failed Drives on FILER : $FILER_C");
        logcomment("FAILED DRIVES PRESENT, DRIVE ID(s) ::: @failed_drvs ") if (@failed_drvs);
        logcomment("No failed drives present, Proceed with next steps")    if !(@failed_drvs);
        logcomment("check for any unowned dirves");
        my $output = $API_obj->execute_raw_command( 'command' => "run local disk show -n" );
        my @unowned;

        foreach my $line ( split( /\n/, $output ) ) {
            if ( $line =~ /Not Owned|Unowned/ ) {
                if ( $line =~ /(\S+)\s+\S+\s+\S+\s+\S+/ ) {
                    logcomment("DEBUG DRIVE : $1");
                    push( @unowned, $1 );
                }
            }
        }
        logcomment("Un-Owned Drive - @unowned");
        foreach my $dv (@unowned) {
            logcomment("Assign drive $dv to $FILER_C");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => "run local disk assign $dv -s $sys_id -o $FILER_C -f" );
            logcomment("Drive assigned  ");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => " " );
            my $prompts_answers = [ ".*continue with disk unfail.*" => 'y', ".*y/n.*" => "y" ];
            $API_obj->execute_raw_command( 'command' => "run local disk unfail -s $dv", 'connectrec-match_table' => $prompts_answers );
        }
    }
####################################################################
    #STEP 9 - Disk zero spares - Clean up drive
####################################################################
    eye_catcher( Test => $Test, string => "STEP 9 of 14 : Clean up drive - disk zero spares" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        logcomment("$FILER_C : Executing disk zero spares");
        $API_obj->execute_raw_command( 'command' => "run local disk zero spares" );
        sleep 10;
        while (1) {
            my $out = $API_obj->execute_raw_command( 'command' => "run local aggr status -s" );
            if ( $out !~ /^not\s+zeroed|zeroing/ig ) {
                logcomment("Filer : $FILER_C : Disk zero spares completed.");
                $flag = 1;
                last;
            } else {
                logcomment("Filer : $FILER_C : Disk zero spares is in progress, waiting for disk zero spares to complete");
                sleep 300;
            }
        }
    }

####################################################################
    #STEP 10  - Check selected spare drive is available in spare pool
####################################################################
    eye_catcher( Test => $Test, string => "STEP 10 of 14 : Check selected spare drive $Spare_to_test is available in spare pool" );
    my $spare_found = 0;
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C      = $Node->name();
        my $drv_2_use    = $sel_filer_drv{$FILER_C};
        my @spare_drives = spare_count_cmode( Node => $Node, run_local => 'YES' );
        logcomment("Spare selected : $drv_2_use");
        ( my $sel_drv = $drv_2_use ) =~ s/^(\w*\.)//;
        logcomment("Checking for drive $sel_drv in spare pool");
        foreach my $spare (@spare_drives) {
            ( my $sp_drv = $spare ) =~ s/^(\w*\.)//;
            if ( $sp_drv =~ /$sel_drv$/ ) {
                logcomment("Spare drive selected $sp_drv is available in spare pool");
                $spare_found = 1;
            }
        }
    }
    if ( $spare_found == 1 ) {
        logcomment("Spare drive $Spare_to_test is available in spare pool, proceed executing MSW tests");
    } else {
        logcomment("Spare drive $Spare_to_test is NOT available in spare pool, MSW test cant be executed");
        $test_status = 1;
        logcomment("**FATAL** : Spare drive $Spare_to_test is NOT available in spare pool, MSW test can NOT be executed");
        logresult( 'FATAL', "SPARE DRIVE $Spare_to_test IS NOT AVAILABLE IN SPARE POOL, MSW test cant be executed" );
    }

####################################################################################################################################################
################################################################ MSW TEST ##########################################################################
####################################################################################################################################################
    #BUILD command for MSW
    my $node_count = @Nodes;
    logcomment("Node count : $node_count");
    my $part = 0;
    for ( my $n = 0 ; $n <= $node_count ; $n++ ) {
        my $Node = $Nodes[$n];
        logcomment("aDEBUG=: node - $n");
        if ( $n == 0 ) {
            $part = 1;
            my $FILER_C = $Node->name();
            my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
            logcomment("Build MSW commands for filer - $FILER_C");
            my $drv_2_use = $sel_filer_drv{$FILER_C};
            logcomment("Drive to use on filer $FILER_C is $drv_2_use");
            logcomment("DEBUG - Part - $part : $FILER_C");
            logcomment("$FILER_C : Check for streams starus on filer $FILER_C");
            my $output = $API_Obj->execute_raw_command( command => "run local \"priv set -q test;scsi stream_status $drv_2_use\"" );
            my @streams = qw ( 1 2 3 4 );
            logcomment("$FILER_C : Streams available : @streams");

            # MSW part 1
            $msw_p1_1 = "run local \"priv set test; disk_latency -p rand_seq_write -w $streams[0] -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw1_p1_s -h $msw1_p1_e -t $tsec -f $drv_2_use\"";
            $msw_p1_2 = "run local \"priv set test; disk_latency -p rand_seq_write -w $streams[1] -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw2_p1_s -h $msw2_p1_e -t $tsec -f $drv_2_use\"";
            $msw_p1_3 = "run local \"priv set test; disk_latency -p rand_seq_write -w $streams[2] -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw3_p1_s -h $msw3_p1_e -t $tsec -f $drv_2_use\"";
            $msw_p1_4 = "run local \"priv set test; disk_latency -p rand_seq_write -w $streams[3] -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw4_p1_s -h $msw4_p1_e -t $tsec -f $drv_2_use\"";

        }    #End of if cond
        if ( $n == 1 ) {
            $part = 2;
            my $FILER_C = $Node->name();
            logcomment("Build MSW commands for filer - $FILER_C");
            my $drv_2_use = $sel_filer_drv{$FILER_C};
            logcomment("Drive to use on filer $FILER_C is $drv_2_use");
            logcomment("DEBUG - Part - $part : $FILER_C");
            my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
            logcomment("Build MSW commands for filer - $FILER_C");
            my $drv_2_use = $sel_filer_drv{$FILER_C};
            logcomment("Drive to use on filer $FILER_C is $drv_2_use");
            logcomment("DEBUG - Part - $part : $FILER_C");
            logcomment("$FILER_C : Check for streams starus on filer $FILER_C");
            my $output = $API_Obj->execute_raw_command( command => "run local \"priv set -q test;scsi stream_status $drv_2_use\"" );

            my @streams = qw ( 5 6 7 8 );
            logcomment("$FILER_C : Streams available : @streams");

            # MSW part 2
            $msw_p2_1 = "run local \"priv set test; disk_latency -p rand_seq_write -w $streams[0] -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw1_p2_s -h $msw1_p2_e -t $tsec -f $drv_2_use\"";
            $msw_p2_2 = "run local \"priv set test; disk_latency -p rand_seq_write -w $streams[1] -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw2_p2_s -h $msw2_p2_e -t $tsec -f $drv_2_use\"";
            $msw_p2_3 = "run local \"priv set test; disk_latency -p rand_seq_write -w $streams[2] -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw3_p2_s -h $msw3_p2_e -t $tsec -f $drv_2_use\"";
            $msw_p2_4 = "run local \"priv set test; disk_latency -p rand_seq_write -w $streams[3] -q 8 -s $wr_size -n $tetris_per_LAA -a $TWO_GB -m bcs -l $msw4_p2_s -h $msw4_p2_e -t $tsec -f $drv_2_use\"";
        }    #End of if cond

    }    #End of for loop
    logcomment("Completed building MSW commands");

####################################################################
    #STEP 11 - Execute MSW tests
####################################################################
    eye_catcher( Test => $Test, string => "STEP 11 of 14 : Execute MSW tests" );
    foreach my $Node (@NODE) {
        my $FILER_C = $Node->name();
        my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        logcomment("$FILER_C : Make sure no streams are open on filer $FILER_C");
        my $output = $API_Obj->execute_raw_command( command => "run local \"priv set -q test;scsi stream_status $Spare_to_test\"" );
        if ( $output =~ /No Streams open/i ) {
            logcomment("$FILER_C : No Streams open on this device");
        } else {
            logcomment("$FILER_C : Some of the STREAMS are OPEN on this device");
            logresult( "FATAL", " $FILER_C : Some of the STREAMS are OPEN on this device" );
            $test_status = 1;
        }
    }

    #MSW Test 1: PRI=1, SEC=0
    eye_catcher( Test => $Test, string => "MSW TEST 1 : PRI=1 , SEC=0" );
    logcomment("Starting MSW Test 1 - PRI=1 , SEC=0");
    my $output = $SSH_FILER_A_1->execute_raw_command( command => "$msw_p1_1" );
    logcomment("End of MSW Test 1 - PRI=1 , SEC=0");
    foreach my $Node (@Nodes) {
        my $FILER_C  = $Node->name();
        my $API_obj  = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $time_out = "6000000";
        $API_obj->set_timeout( "connectrec-timeout" => $time_out );
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );
        sleep 5;
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );

        logcomment("$FILER_C : Checking failed drive on filer $FILER_C ");
        my $sys_config_data = Storage::Common_Lib::sysconfig_v_data( node => $Node );
        my $failed_ar       = $sys_config_data->{'faileddisks'};
        my @failed_drvs     = @$failed_ar;
        my $normal_drv      = $sys_config_data->{'normaldisks'};
        my @nor_drv         = @$normal_drv;
        my $sys_id          = $sys_config_data->{'system_id'};
        logcomment("FILER : $FILER_C : System ID : $sys_id");
        logcomment("Drives present - @nor_drv");
        logcomment("Check for Failed Drives on FILER : $FILER_C");
        logcomment("FAILED DRIVES PRESENT, DRIVE ID(s) ::: @failed_drvs ") if (@failed_drvs);
        logcomment("No failed drives present, Proceed with next steps")    if !(@failed_drvs);
        logcomment("check for any unowned dirves");
        my $output = $API_obj->execute_raw_command( 'command' => "run local disk show -n" );
        my @unowned;

        foreach my $line ( split( /\n/, $output ) ) {
            if ( $line =~ /Not Owned|Unowned/ ) {
                if ( $line =~ /(\S+)\s+\S+\s+\S+\s+\S+/ ) {
                    logcomment("DEBUG DRIVE : $1");
                    push( @unowned, $1 );
                }
            }
        }
        logcomment("Un-Owned Drive - @unowned");
        foreach my $dv (@unowned) {
            logcomment("Assign drive $dv to $FILER_C");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => "run local disk assign $dv -s $sys_id -o $FILER_C -f" );
            logcomment("Drive assigned  ");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => " " );
            my $prompts_answers = [ ".*continue with disk unfail.*" => 'y', ".*y/n.*" => "y" ];
            $API_obj->execute_raw_command( 'command' => "run local disk unfail -s $dv", 'connectrec-match_table' => $prompts_answers );
        }
    }

    #MSW Test 2: PRI=1, SEC=1";
    eye_catcher( Test => $Test, string => "MSW TEST 2 : PRI=1 , SEC=1" );
    my @m_subtest2;
    push( @m_subtest2, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_2_P1_S1_1", -bg, "--", $SSH_FILER_A_1, $msw_p1_1 ) );
    push( @m_subtest2, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_2_P1_S1_2", -bg, "--", $SSH_FILER_B_1, $msw_p2_1 ) );
    logcomment("Starting MSW Test 2 : PRI=1 , SEC=1");
    Subtest::wait_finish( subtest => [@m_subtest2] );
    logcomment("End of MSW Test 2 : PRI=1 , SEC=1");

    foreach my $Node (@Nodes) {
        my $FILER_C  = $Node->name();
        my $API_obj  = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $time_out = "6000000";
        $API_obj->set_timeout( "connectrec-timeout" => $time_out );
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );
        sleep 5;
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );

        logcomment("$FILER_C : Checking failed drive on filer $FILER_C ");
        my $sys_config_data = Storage::Common_Lib::sysconfig_v_data( node => $Node );
        my $failed_ar       = $sys_config_data->{'faileddisks'};
        my @failed_drvs     = @$failed_ar;
        my $normal_drv      = $sys_config_data->{'normaldisks'};
        my @nor_drv         = @$normal_drv;
        my $sys_id          = $sys_config_data->{'system_id'};
        logcomment("FILER : $FILER_C : System ID : $sys_id");
        logcomment("Drives present - @nor_drv");
        logcomment("Check for Failed Drives on FILER : $FILER_C");
        logcomment("FAILED DRIVES PRESENT, DRIVE ID(s) ::: @failed_drvs ") if (@failed_drvs);
        logcomment("No failed drives present, Proceed with next steps")    if !(@failed_drvs);
        logcomment("check for any unowned dirves");
        my $output = $API_obj->execute_raw_command( 'command' => "run local disk show -n" );
        my @unowned;

        foreach my $line ( split( /\n/, $output ) ) {
            if ( $line =~ /Not Owned|Unowned/ ) {
                if ( $line =~ /(\S+)\s+\S+\s+\S+\s+\S+/ ) {
                    logcomment("DEBUG DRIVE : $1");
                    push( @unowned, $1 );
                }
            }
        }
        logcomment("Un-Owned Drive - @unowned");
        foreach my $dv (@unowned) {
            logcomment("Assign drive $dv to $FILER_C");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => "run local disk assign $dv -s $sys_id -o $FILER_C -f" );
            logcomment("Drive assigned  ");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => " " );
            my $prompts_answers = [ ".*continue with disk unfail.*" => 'y', ".*y/n.*" => "y" ];
            $API_obj->execute_raw_command( 'command' => "run local disk unfail -s $dv", 'connectrec-match_table' => $prompts_answers );
        }
    }

    #SW Test 3: PRI=2, SEC=2";
    eye_catcher( Test => $Test, string => "MSW TEST 3 : PRI=2 , SEC=2" );
    my @m_subtest3;
    push( @m_subtest3, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_3_P2_S2_1", -bg, "--", $SSH_FILER_A_1, $msw_p1_1 ) );
    push( @m_subtest3, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_3_P2_S2_2", -bg, "--", $SSH_FILER_A_2, $msw_p1_2 ) );
    push( @m_subtest3, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_3_P2_S2_1", -bg, "--", $SSH_FILER_B_1, $msw_p2_1 ) );
    push( @m_subtest3, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_3_P2_S2_2", -bg, "--", $SSH_FILER_B_2, $msw_p2_2 ) );
    logcomment("Starting MSW Test 3 : PRI=2 , SEC=2");
    Subtest::wait_finish( subtest => [@m_subtest3] );
    logcomment("End of MSW Test 3 : PRI=2 , SEC=2");

    foreach my $Node (@Nodes) {
        my $FILER_C  = $Node->name();
        my $API_obj  = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $time_out = "6000000";
        $API_obj->set_timeout( "connectrec-timeout" => $time_out );
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );
        sleep 5;
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );

        logcomment("$FILER_C : Checking failed drive on filer $FILER_C ");
        my $sys_config_data = Storage::Common_Lib::sysconfig_v_data( node => $Node );
        my $failed_ar       = $sys_config_data->{'faileddisks'};
        my @failed_drvs     = @$failed_ar;
        my $normal_drv      = $sys_config_data->{'normaldisks'};
        my @nor_drv         = @$normal_drv;
        my $sys_id          = $sys_config_data->{'system_id'};
        logcomment("FILER : $FILER_C : System ID : $sys_id");
        logcomment("Drives present - @nor_drv");
        logcomment("Check for Failed Drives on FILER : $FILER_C");
        logcomment("FAILED DRIVES PRESENT, DRIVE ID(s) ::: @failed_drvs ") if (@failed_drvs);
        logcomment("No failed drives present, Proceed with next steps")    if !(@failed_drvs);
        logcomment("check for any unowned dirves");
        my $output = $API_obj->execute_raw_command( 'command' => "run local disk show -n" );
        my @unowned;

        foreach my $line ( split( /\n/, $output ) ) {
            if ( $line =~ /Not Owned|Unowned/ ) {
                if ( $line =~ /(\S+)\s+\S+\s+\S+\s+\S+/ ) {
                    logcomment("DEBUG DRIVE : $1");
                    push( @unowned, $1 );
                }
            }
        }
        logcomment("Un-Owned Drive - @unowned");
        foreach my $dv (@unowned) {
            logcomment("Assign drive $dv to $FILER_C");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => "run local disk assign $dv -s $sys_id -o $FILER_C -f" );
            logcomment("Drive assigned  ");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => " " );
            my $prompts_answers = [ ".*continue with disk unfail.*" => 'y', ".*y/n.*" => "y" ];
            $API_obj->execute_raw_command( 'command' => "run local disk unfail -s $dv", 'connectrec-match_table' => $prompts_answers );
        }
    }

    #MSW Test 4: PRI=3, SEC=3";
    eye_catcher( Test => $Test, string => "MSW TEST 4 : PRI=3 , SEC=3" );
    my @m_subtest4;
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_4_P3_S3_1", -bg, "--", $SSH_FILER_A_1, $msw_p1_1 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_4_P3_S3_2", -bg, "--", $SSH_FILER_A_2, $msw_p1_2 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_4_P3_S3_3", -bg, "--", $SSH_FILER_A_3, $msw_p1_3 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_4_P3_S3_1", -bg, "--", $SSH_FILER_B_1, $msw_p2_1 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_4_P3_S3_2", -bg, "--", $SSH_FILER_B_2, $msw_p2_2 ) );
    push( @m_subtest4, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_4_P3_S3_3", -bg, "--", $SSH_FILER_B_3, $msw_p2_3 ) );
    logcomment("Starting MSW Test 4 : PRI=3 , SEC=3");
    Subtest::wait_finish( subtest => [@m_subtest4] );
    logcomment("End of MSW Test 4 : PRI=3 , SEC=3");

    foreach my $Node (@Nodes) {
        my $FILER_C  = $Node->name();
        my $API_obj  = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $time_out = "6000000";
        $API_obj->set_timeout( "connectrec-timeout" => $time_out );
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );
        sleep 5;
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );

        logcomment("$FILER_C : Checking failed drive on filer $FILER_C ");
        my $sys_config_data = Storage::Common_Lib::sysconfig_v_data( node => $Node );
        my $failed_ar       = $sys_config_data->{'faileddisks'};
        my @failed_drvs     = @$failed_ar;
        my $normal_drv      = $sys_config_data->{'normaldisks'};
        my @nor_drv         = @$normal_drv;
        my $sys_id          = $sys_config_data->{'system_id'};
        logcomment("FILER : $FILER_C : System ID : $sys_id");
        logcomment("Drives present - @nor_drv");
        logcomment("Check for Failed Drives on FILER : $FILER_C");
        logcomment("FAILED DRIVES PRESENT, DRIVE ID(s) ::: @failed_drvs ") if (@failed_drvs);
        logcomment("No failed drives present, Proceed with next steps")    if !(@failed_drvs);
        logcomment("check for any unowned dirves");
        my $output = $API_obj->execute_raw_command( 'command' => "run local disk show -n" );
        my @unowned;

        foreach my $line ( split( /\n/, $output ) ) {
            if ( $line =~ /Not Owned|Unowned/ ) {
                if ( $line =~ /(\S+)\s+\S+\s+\S+\s+\S+/ ) {
                    logcomment("DEBUG DRIVE : $1");
                    push( @unowned, $1 );
                }
            }
        }
        logcomment("Un-Owned Drive - @unowned");
        foreach my $dv (@unowned) {
            logcomment("Assign drive $dv to $FILER_C");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => "run local disk assign $dv -s $sys_id -o $FILER_C -f" );
            logcomment("Drive assigned  ");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => " " );
            my $prompts_answers = [ ".*continue with disk unfail.*" => 'y', ".*y/n.*" => "y" ];
            $API_obj->execute_raw_command( 'command' => "run local disk unfail -s $dv", 'connectrec-match_table' => $prompts_answers );
            $API_obj->execute_raw_command( 'command' => " " );
        }
    }

    #non-MSW Test 5 : PRI=4 , SEC=4";
    eye_catcher( Test => $Test, string => "MSW TEST 5 : PRI=4 , SEC=4" );
    my @m_subtest5;
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_5_P4_S4_1", -bg, "--", $SSH_FILER_A_1, $msw_p1_1 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_5_P4_S4_2", -bg, "--", $SSH_FILER_A_2, $msw_p1_2 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_5_P4_S4_3", -bg, "--", $SSH_FILER_A_3, $msw_p1_3 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_A_TST_5_P4_S4_4", -bg, "--", $SSH_FILER_A_4, $msw_p1_4 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_5_P4_S4_1", -bg, "--", $SSH_FILER_B_1, $msw_p2_1 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_5_P4_S4_2", -bg, "--", $SSH_FILER_B_2, $msw_p2_2 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_5_P4_S4_3", -bg, "--", $SSH_FILER_B_3, $msw_p2_3 ) );
    push( @m_subtest5, subtest( \&exe_multiple_ssh, -runid, "MSW_FILER_B_TST_5_P4_S4_4", -bg, "--", $SSH_FILER_B_4, $msw_p2_4 ) );
    logcomment("Starting MSW Test 5 : PRI=4 , SEC=4");
    Subtest::wait_finish( subtest => [@m_subtest5] );
    logcomment("End of MSW Test 4 : PRI=4 , SEC=4");

####################################################################
    #STEP 12 - Check for any failed drive
####################################################################
    eye_catcher( Test => $Test, string => "STEP 12 of 14 : Check for failed drives" );
    foreach my $Node (@Nodes) {
        my $FILER_C  = $Node->name();
        my $API_obj  = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my $time_out = "6000000";
        $API_obj->set_timeout( "connectrec-timeout" => $time_out );
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );
        sleep 5;
        $API_obj->execute_raw_command( 'command' => "\013" );
        $API_obj->execute_raw_command( 'command' => " " );

        logcomment("$FILER_C : Checking failed drive on filer $FILER_C ");
        my $sys_config_data = Storage::Common_Lib::sysconfig_v_data( node => $Node );
        my $failed_ar       = $sys_config_data->{'faileddisks'};
        my @failed_drvs     = @$failed_ar;
        my $normal_drv      = $sys_config_data->{'normaldisks'};
        my @nor_drv         = @$normal_drv;
        my $sys_id          = $sys_config_data->{'system_id'};
        logcomment("FILER : $FILER_C : System ID : $sys_id");
        logcomment("Drives present - @nor_drv");
        logcomment("Check for Failed Drives on FILER : $FILER_C");
        logcomment("FAILED DRIVES PRESENT, DRIVE ID(s) ::: @failed_drvs ") if (@failed_drvs);
        logcomment("No failed drives present, Proceed with next steps")    if !(@failed_drvs);
        logcomment("check for any unowned dirves");
        my $output = $API_obj->execute_raw_command( 'command' => "run local disk show -n" );
        my @unowned;

        foreach my $line ( split( /\n/, $output ) ) {
            if ( $line =~ /Not Owned|Unowned/ ) {
                if ( $line =~ /(\S+)\s+\S+\s+\S+\s+\S+/ ) {
                    logcomment("DEBUG DRIVE : $1");
                    push( @unowned, $1 );
                }
            }
        }
        logcomment("Un-Owned Drive - @unowned");
        foreach my $dv (@unowned) {
            logcomment("Assign drive $dv to $FILER_C");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => "run local disk assign $dv -s $sys_id -o $FILER_C -f" );
            logcomment("Drive assigned  ");
            $API_obj->execute_raw_command( 'command' => " " );
            $API_obj->execute_raw_command( 'command' => " " );
            my $prompts_answers = [ ".*continue with disk unfail.*" => 'y', ".*y/n.*" => "y" ];
            $API_obj->execute_raw_command( 'command' => "run local disk unfail -s $dv", 'connectrec-match_table' => $prompts_answers );
        }
    }
####################################################################
    #STEP 13 - Disk zero spares - Clean up drive
####################################################################
    eye_catcher( Test => $Test, string => "STEP 13 of 14 : Clean up drive - disk zero spares" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        logcomment("$FILER_C : Executing disk zero spares");
        $API_obj->execute_raw_command( 'command' => "run local disk zero spares" );
        sleep 10;
        while (1) {
            my $out = $API_obj->execute_raw_command( 'command' => "run local aggr status -s" );
            if ( $out !~ /^not\s+zeroed|zeroing/ig ) {
                logcomment("Filer : $FILER_C : Disk zero spares completed.");
                $flag = 1;
                last;
            } else {
                logcomment("Filer : $FILER_C : Disk zero spares is in progress, waiting for disk zero spares to complete");
                sleep 300;
            }
        }
    }

####################################################################
    #STEP 14  - Check selected spare drive is available in spare pool
####################################################################
    eye_catcher( Test => $Test, string => "STEP 14 of 14 : Check selected spare drive $Spare_to_test is available in spare pool" );
    my $spare_found = 0;
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C      = $Node->name();
        my $Api_Set_Obj  = $Node->get_7m_or_nodescope_apiset( connid => "console" );
        my @spare_drives = nvme_get_local_spares( API_con => $Api_Set_Obj, Filer => $FILER_C );
        my $drv_2_use    = $sel_filer_drv{$FILER_C};
        $Spare_to_test = $drv_2_use;
        logcomment("Spare selected : $Spare_to_test");
        ( my $sel_drv = $drv_2_use ) =~ s/^(\w*\.)//;
        logcomment("Checking for drive $sel_drv in spare pool");

        foreach my $spare (@spare_drives) {
            ( my $sp_drv = $spare ) =~ s/^(\w*\.)//;
            if ( $sp_drv =~ /$sel_drv$/ ) {
                logcomment("Spare drive selected $sp_drv is available in spare pool");
                $spare_found = 1;
            }
        }
    }
    if ( $spare_found == 1 ) {
        logcomment("Spare drive $Spare_to_test is available in spare pool");
    } else {
        logcomment("Spare drive $Spare_to_test is NOT available in spare pool, After Executing MSW test");
        $test_status = 1;
        logcomment("**FATAL** : Spare drive $Spare_to_test is NOT available in spare pool, After Executing MSW test");
        logresult( 'FATAL', "SPARE DRIVE $Spare_to_test IS NOT AVAILABLE IN SPARE POOL, After Executing MSW test" );
    }

    if ( $test_status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }

}    # End of Multi_Stream_Wrt_Perf

sub exe_multiple_ssh() {
    my ( $ssh_conn, $command ) = @_;
    my $output = $ssh_conn->execute_raw_command( command => "$command" );
}

####################################################################
#Cleanup
####################################################################
sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    $Test->description("Preforming Post test process - Check filer state, Check disk, check root volume,halt hammer, halt scrub, filer config");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;
    my $result   = 0;
    ###########################################################################################
    ## Post Test process - Category : "post_test"
    ###########################################################################################
    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );
    if ($result) {
        return $TCD::FAIL;
    } else {
        return $TCD::PASS;
    }
}

sub check_port() {
    ( my @ary ) = @_;
    my %count;
    map { $count{$_}++ } @ary;
    my $i = 0;
    foreach ( keys %count ) {
        $i++;
    }
    return $i;
}