#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Hmr_Scrub.thpl#1 $
#
## @summary Mode
##  Disk Firmware Download in background/foreground mode
#
##
## @description
## Verify that the drive can download disk firmware in foreground and background mode properly
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
## @steps
## The test will execute steps mentioned below:
##
##
## @status  Automated
########################################
### Library functions
##########################################
use strict;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
use List::Util qw(shuffle);
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $USER_LEVL
  $ARMADILLO
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$USER_LEVL      = $params->get( 'USER_LEVL',      default => 'high' );
################
# Testcase name
################
my $TC_name;
$TC_name = "507_NADQ02_NVMe_Sanitize_Verify";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home = $ENV{HOME};
my $scrub_start;
my $scrub_end;

my $test_wait_time = 12 * 3600;    #12 Hrs test
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( NVMe_Sanitize => "NVME Sanitize verify on spare drives" );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );

    return $TCD::PASS;
}

sub NVMe_Sanitize {
    my @subtests;
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&exe_sanitize, -runid, "Exe_Sanitize_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $status = status_return( subtest_ref => [@subtests] );
    logcomment("Total test status is : $status");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub exe_sanitize {
    $Test->description("NVME : Exectuing Sanitize commands on available filers");
    my ( @Nodes, %filer_nvol );
    my $Home = $ENV{HOME};
    my @SPARE_DRIVES;
    my @NON_RT_VOLS;
    my %filer_nvol       = ();
    my $create_agg_n_vol = 0;
    my $san_drv_1;
    my $san_drv_2;
    my $san_drv_3;
    my $aggr_vol_found = 0;
    push( @Nodes, shift(@_) );
    logcomment("Total Nodes  - @Nodes");
    logcomment( "Filer passed to the subtest :: " . $Nodes[0]->name() );

    ################################################
    ##STEP 1
    nvme_eye_catcher( Test => $Test, string => "STEP 1 of 10 : Enabling the disk sanitization license." );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("$FILER_C : Enabling the disk sanitization license");

        #my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" );
        logcomment("Enable sanitize options");
        $Api_Set_Obj->execute_command( 'command' => "options nodescope.disableoptions false" );
        $Api_Set_Obj->execute_command( 'command' => "options licensed_feature.disk_sanitization.enable" );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
        $Api_Set_Obj->execute_raw_command( 'command' => "priv set test" );
        logcomment("Set sanitize command");
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        sleep 10;
        my $san_prom = [ ".*are you sure you wish to continue.*" => 'y', "continue?" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "options licensed_feature.disk_sanitization.enable on", 'connectrec-match_table' => $san_prom );
        logcomment("Enabled sanitize option");
        $Api_Set_Obj->execute_raw_command( 'command' => "\cD" );
        $Api_Set_Obj->execute_command( 'command' => "options licensed_feature.disk_sanitization.enable" );
        $Api_Set_Obj->execute_command( 'command' => "options nodescope.disableoptions true" );
        sleep 20;
        logcomment("$FILER_C : Enabled all sanitize options");
    }
    ################################################
    ##STEP 2
    nvme_eye_catcher( Test => $Test, string => "STEP 2 of 10 : Run disk sanitize on all spares" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();

        #my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        logcomment("$FILER_C : Execute sanitize commands on all spare drives");
        logcomment("Check for spare drive availability, Minimum 3 spares are required to create aggregate");
        my @spare_drvs = nvme_get_local_spares( API_con => $Api_Set_Obj, Filer => $FILER_C );
        my $spare_count = @spare_drvs;
        @SPARE_DRIVES = @spare_drvs;
        logcomment("$FILER_C : Available spare are : @spare_drvs");
        logcomment("$FILER_C : Total Spares avaiable are $spare_count");

        if ( $spare_count < 3 ) {
            logcomment("**FAIL** : $FILER_C : Minimum spare required on filer is 3, Available spare on filer is $spare_count");
            return logresult( 'INFO', msg => 1 );
        }
        logcomment("$FILER_C : Minimum spares are available");
        foreach my $drv (@SPARE_DRIVES) {
            logcomment("Executing Sanitize command on drive :$drv");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            my $prompts_answers = [ ".*continue.*" => 'yes' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
            my $prompts_answers_1 = [ ".*Would you like to continue.*" => 'yes' , ".*Do you want to continue.*" => 'yes'];
            $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize start $drv", 'connectrec-match_table' => $prompts_answers_1 );
			
            logcomment("Started sanitixe on drive : $drv");
            sleep 10;
        }
        logcomment("$FILER_C : Sanitize commande executed on all spare dirves");
        logcomment("$FILER_C : Check Sanitize status of all drives");
        my $timeout = 172800;    #48 hrs
        my ( $sec, $min, $hour ) = gmtime($timeout);
        logcomment("MAX timeout is $timeout seconds i.e $hour Hours $min Minutes $sec Seconds");
        my $start_time = time();
        my $end_time   = time();
        $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
        $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );

        while ( ( $end_time - $start_time ) < $timeout ) {
            my $aggr_output = $Api_Set_Obj->execute_raw_command( command => "aggr status -r" );
            if ( $aggr_output =~ /sanitizing/ ) {
                logcomment("Sanitization is under progress. Waiting for the completion of sanitization");
                sleep(600);
            } else {
                logcomment("Sanitization is completed");
                last;
            }
            $end_time = time();
        }
        $Api_Set_Obj->execute_raw_command( 'command' => "exit" );
        logcomment("$FILER_C : Sanitization on all spare drives completed");
		my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
		$Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600000 );
		$Api_Set_Obj->execute_raw_command( command => " ");
    }

    ################################################
    ##STEP 3
    nvme_eye_catcher( Test => $Test, string => "STEP 3 of 10 : Restore the disks" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();

        #my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
		$Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600000 );
        my $after = $Api_Set_Obj->execute_raw_command( 'command' => "run local sysconfig -v" );
        my $sysid;
        if ( ( $after =~ /\s*System ID:\s+(\d+)\s+/ ) ) {
            $sysid = $1;
        }
        my @san_comp_drv;
        logcomment("$FILER_C : System ID : $sysid");
        my $san_o = $Api_Set_Obj->execute_raw_command( 'command' => "run local aggr status -r" );
        foreach my $line ( split /\n/, $san_o ) {
            if ( $line =~ /sanitized/ ) {
                my ($drv) = ( $line =~ /^sanitized\s+(\S+)\s+/ );
                logcomment("$FILER_C : Drive $drv Sanitized");
                push( @san_comp_drv, $drv );
            }
        }
        logcomment("$FILER_C : Release Sanitized drive : @san_comp_drv");
        foreach my $drv (@san_comp_drv) {
            logcomment("Releasing Sanitized  Drive : $drv ");
            $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize release $drv " );
            logcomment("Released drive $drv ");
            sleep 5;
            logcomment("Check drive ownership");
            my $unout = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk show -n $drv" );
            if ( $unout =~ /$drv\s+Not\s+Owned\s+/ ) {
                logcomment("$drv drive is Not Owned, Assign drives to $FILER_C ");
                logcomment("Filer : $FILER_C  : System ID : $sysid");
                $Api_Set_Obj->execute_raw_command( 'command' => "run local disk assign $drv -o $FILER_C -s $sysid" );
                logcomment("Drive $drv assigned to $FILER_C with serial number $sysid");
            } else {
                logcomment("Drive $drv has owenrship assigned, Proceed with next steps");
            }
        }
        logcomment("$FILER_C : Check for bad lable drive ");
        my @bad_drv;
        my $bad_o = $Api_Set_Obj->execute_raw_command( 'command' => "run local aggr status -r" );
        foreach my $line ( split /\n/, $bad_o ) {
            if ( $line =~ /bad label/ ) {
                my ($drv_b) = ( $line =~ /bad\s+label\s+(\S+)/ );
                logcomment("$FILER_C : Drive $drv_b is failed / has bad label");
                push( @bad_drv, $drv_b );
            }
        }
        logcomment("$FILER_C : Bad lable drives are : @bad_drv");
        logcomment("$FILER_C : Unfail all bad lable drives and issue disk zero spares");
        foreach my $d (@bad_drv) {
            logcomment("$FILER_C : Unfail $d ");
            my $prompts_answers = [ ".*you sure you want to continue with disk unfail.*" => 'yes' ];
            my $ou = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk unfail -s $d ", 'connectrec-match_table' => $prompts_answers );
            logcomment("Drive $d unfailing ...") if ( $ou =~ /unfailing/ );
            sleep 10;
        }
        logcomment("$FILER_C : Zero all spare drives");
        my $Api_Set_Objc = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
		$Api_Set_Obj->set_timeout( "connectrec-timeout" => 360000000 );
        $Api_Set_Objc->execute_raw_command( 'command' => " " );
		$Api_Set_Objc->execute_raw_command( 'command' => " " );
		sleep 60;
		$Api_Set_Objc->execute_raw_command( 'command' => "cd" );
		$Api_Set_Objc->execute_raw_command( 'command' => "\013" );
		$Api_Set_Objc->execute_raw_command( 'command' => "\013" );
        $Api_Set_Objc->execute_raw_command( 'command' => "run local disk zero spares" );
		$Api_Set_Objc->execute_raw_command( 'command' => "\013" );
        logcomment("$FILER_C : All drives were unfailed and zeroed");
        $Api_Set_Obj->execute_raw_command( command => " ");
    }

    ################################################
    ##STEP 4
    nvme_eye_catcher( Test => $Test, string => "STEP 4  of 10 : Execute disk sanitize on one drive	" );
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my @spare_drvs  = nvme_get_local_spares( API_con => $Api_Set_Obj, Filer => $FILER_C );
        logcomment("Execute disk sanitize on drive : $spare_drvs[0]");
        $san_drv_1 = $spare_drvs[0];
        $san_drv_2 = $spare_drvs[1];
        $san_drv_3 = $spare_drvs[2];
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $prompts_answers = [ ".*Would you like to continue.*" => 'yes' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        my $prompts_answers_1 = [ ".*continue.*" => 'yes' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize start $san_drv_1", 'connectrec-match_table' => $prompts_answers_1 );
        logcomment("Started sanitize on drive : $san_drv_1");
        logcomment("$FILER_C : Sanitization started on drive $san_drv_1");
        logcomment("$FILER_C : Check Sanitize status of all drives");
        logcomment("$FILER_C : Checking Sanititze completion status of all drives");
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );
        sleep 60;

        if ( $out =~ /sanitization for\s+(\S+)\s+is\s+(\S+)\s+\%\s+complete/ ) {
            logcomment("Drive $1 - $2% Sanitization is in progress");
            sleep 10;
        } else {
            logcomment("Santitzation of drives $san_drv_1 FAILED or Cannot start");
            logcomment("**FAIL** : Santitzation of drives $san_drv_1 FAILED or Cannot start");
        }
    }

    ################################################
    ##STEP 5
    nvme_eye_catcher( Test => $Test, string => "STEP 5 of 10 : Verify that the disks rejects scsi test " );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("Verify drive : $san_drv_1 rejects scsi test");
        my $sc_out = $Api_Set_Obj->execute_raw_command( 'command' => "run local scsi test $san_drv_1 " );
        if ( ( $sc_out =~ /scsi test error:/ ) && ( $sc_out =~ /key\:\s+0x2\s+code\:\s+0x4\s+qual\:\s+0x1b/ ) ) {
            logcomment("$FILER_C : Drive $san_drv_1 scsi test rejected");
        } else {
            logcomment("$FILER_C : Drive $san_drv_1 ready for scsi test");
            logcomment("**FAIL** : $FILER_C : Drive $san_drv_1 ready for scsi test");
        }
        sleep 10;
    }

    ################################################
    ##STEP  6
    nvme_eye_catcher( Test => $Test, string => "STEP 6 of 10 : Verify that the disks accepts scsi inquiry" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("Verify drive : $san_drv_1 accepts scsi inquiry");
        my $sc_o = $Api_Set_Obj->execute_raw_command( 'command' => "run local scsi inquiry  $san_drv_1" );
        sleep 20;
    }

    ################################################
    ##STEP 7
    nvme_eye_catcher( Test => $Test, string => "STEP 7 of 10 : Verify that the disks accepts snvmadmin reset on a single drive." );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("Verify drive : $san_drv_1 accepts snvmadmin reset");
        my $out = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;snvmadmin reset $san_drv_1 \"" );
        sleep 20;
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );
        sleep 20;
        if ( $out =~ /sanitization for\s+(\S+)\s+is\s+(\S+)\s+\%\s+complete/ ) {
            logcomment("Drive $1 - $2% Sanitization is in progress");
        }
    }

    ################################################
    ##STEP 8
    nvme_eye_catcher( Test => $Test, string => "STEP 8 of 10 : Abort the disk sanitize on drive $san_drv_1" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("$FILER_C : Abort disk sanitize on drive $san_drv_1");
        my $out = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk sanitize abort $san_drv_1\"" );
        logcomment("Sanitization aborted on drive $san_drv_1, drive will be in unfailed state");
        sleep 20;
        logcomment("Check for drive $san_drv_1 sanitize abort command is successful");
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );

        if ( $out =~ /sanitization for\s+($san_drv_1)\s+is\s+(\S+)\s+\%\s+complete/ ) {
            logcomment("Abort on drive $san_drv_1 is still in progress, Wait till drive is aborted");
            while (1) {
                logcomment("Check for drive $san_drv_1 abort ..");
                my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );
                if ( $out !~ /$san_drv_1/ ) {
                    logcomment("Drive $san_drv_1 Aborted ...");
                    last;
                } else {
                    logcomment("Waiting for abort on drive $san_drv_1");
                    sleep 60;
                }
            }
        } else {
            logcomment("Abort on drive $san_drv_1 successful, proceed with next stpes");
        }
        sleep 30;
        logcomment("Check for drive ownership after aborting sanitizing command on drive $san_drv_1");
        $Api_Set_Obj->execute_raw_command( command => "\013" );
        my $ad_out = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk show -n\"" );
        if ( $ad_out =~ /$san_drv_1/ ) {
            logcomment("Drive $san_drv_1 is unassigned, Assign to $FILER_C");
            my $after = $Api_Set_Obj->execute_raw_command( 'command' => "run local sysconfig -v" );
            my $sysid;
            if ( ( $after =~ /\s*System ID:\s+(\d+)\s+/ ) ) {
                $sysid = $1;
            }
            $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk assign $san_drv_1 -o $FILER_C -s $sysid\"" );
        } else {
            logcomment("Drive $san_drv_1 has ownership");
        }
        sleep 20;
    }

    ################################################
    ##STEP 9
    nvme_eye_catcher( Test => $Test, string => "STEP 9 of 10 : Execute disk sanitize on another drive." );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("$FILER_C : Execute disk sanitize on drive $san_drv_2");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $prompts_answers = [ ".*Would you like to continue.*" => 'yes' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        my $prompts_answers_1 = [ ".*continue.*" => 'yes' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize start $san_drv_2", 'connectrec-match_table' => $prompts_answers_1 );
        logcomment("$FILER_C : Sanitization started on drive $san_drv_2");
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );
        sleep 60;

        if ( $out =~ /sanitization for\s+(\S+)\s+is\s+(\S+)\s+\%\s+complete/ ) {
            logcomment("Drive $1 - $2% Sanitization is in progress");
            sleep 10;
        } else {
            logcomment("Santitzation of drives $san_drv_2 FAILED or Cannot start");
            logcomment("**FAIL** : Santitzation of drives $san_drv_2 FAILED or Cannot start");
        }
        sleep 20;
    }

    ################################################
    ##STEP 10
    nvme_eye_catcher( Test => $Test, string => "STEP 10 of 10 : Execute snvmadmin nssron drive $san_drv_2 where sanitize is in progress" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("$FILER_C : Execute snvmadmin nssr on drive  $san_drv_2 where sanitize is in progress");
        my $out = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;snvmadmin nssr $san_drv_2 \"" );
        sleep 20;
        logcomment("$FILER_C :Check for sanitization process");
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );
        sleep 20;

        if ( $out =~ /sanitization for\s+(\S+)\s+is\s+(\S+)\s+\%\s+complete/ ) {
            logcomment("Drive $1 - $2% Sanitization is in progress");
        }
        sleep 10;
    }

    ################################################
    ##STEP 11
    nvme_eye_catcher( Test => $Test, string => "STEP 11 of 10 : Abort the disk sanitize on drive $san_drv_2" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("$FILER_C : Abort disk sanitize on drive $san_drv_2");
        my $out = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk sanitize abort $san_drv_2\"" );
        logcomment("Sanitization aborted on drive $san_drv_2, drive will be in unfailed state");
        sleep 20;
        logcomment("Check for drive $san_drv_2 sanitize abort command is successful");
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );

        if ( $out =~ /sanitization for\s+($san_drv_2)\s+is\s+(\S+)\s+\%\s+complete/ ) {
            logcomment("Abort on drive $san_drv_2 is still in progress, Wait till drive is aborted");
            while (1) {
                logcomment("Check for drive $san_drv_2 abort ..");
                my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );
                if ( $out !~ /$san_drv_2/ ) {
                    logcomment("Drive $san_drv_2 Aborted ...");
                    last;
                } else {
                    logcomment("Waiting for abort on drive $san_drv_2");
                    sleep 60;
                }
            }
        } else {
            logcomment("Abort on drive $san_drv_2 successful, proceed with next stpes");
        }
        sleep 30;
        logcomment("Check for drive ownership after aborting sanitizing command on drive $san_drv_2");
        $Api_Set_Obj->execute_raw_command( command => "\013" );
        my $ad_out = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk show -n\"" );
        if ( $ad_out =~ /$san_drv_2/ ) {
            logcomment("Drive $san_drv_2 is unassigned, Assign to $FILER_C");
            my $after = $Api_Set_Obj->execute_raw_command( 'command' => "run local sysconfig -v" );
            my $sysid;
            if ( ( $after =~ /\s*System ID:\s+(\d+)\s+/ ) ) {
                $sysid = $1;
            }
            $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk assign $san_drv_2 -o $FILER_C -s $sysid\"" );
        } else {
            logcomment("Drive $san_drv_2 has ownership");
        }
        sleep 10;
    }

    ################################################
    ##STEP 12
    nvme_eye_catcher( Test => $Test, string => "STEP 12 of 10 : Execute disk sanitize on another drive." );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("$FILER_C : Execute disk sanitize on drive $san_drv_3");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $prompts_answers = [ ".*Would you like to continue.*" => 'yes' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        my $prompts_answers_1 = [ ".*continue.*" => 'yes' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize start $san_drv_3", 'connectrec-match_table' => $prompts_answers_1 );
        logcomment("$FILER_C : Sanitization started on drive $san_drv_3");
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );
        sleep 60;

        if ( $out =~ /sanitization for\s+(\S+)\s+is\s+(\S+)\s+\%\s+complete/ ) {
            logcomment("Drive $1 - $2% Sanitization is in progress");
            sleep 10;
        } else {
            logcomment("Santitzation of drives $san_drv_3 FAILED or Cannot start");
            logcomment("**FAIL** : Santitzation of drives $san_drv_3 FAILED or Cannot start");
        }
        sleep 20;
    }

    ################################################
    ##STEP 13
    nvme_eye_catcher( Test => $Test, string => "STEP 13 of 10 :  Execute snvmadmin freset on drive $san_drv_3 where sanitize is in progress" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("$FILER_C : Execute snvmadmin nssr on drive  $san_drv_3 where sanitize is in progress");
        my $out = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;snvmadmin freset $san_drv_3 \"" );
        sleep 20;
        logcomment("$FILER_C :Check for sanitization process");
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );
        sleep 20;

        if ( $out =~ /sanitization for\s+(\S+)\s+is\s+(\S+)\s+\%\s+complete/ ) {
            logcomment("Drive $1 - $2% Sanitization is in progress");
        }
        sleep 10;
    }

    ################################################
    ##STEP 14
    nvme_eye_catcher( Test => $Test, string => "STEP 14 of 10 : Abort the disk sanitize on drive $san_drv_3" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("$FILER_C : Abort disk sanitize on drive $san_drv_3");
        my $out = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk sanitize abort $san_drv_3\"" );
        logcomment("Sanitization aborted on drive $san_drv_3, drive will be in unfailed state");
        sleep 20;
        logcomment("Check for drive $san_drv_3 sanitize abort command is successful");
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );

        if ( $out =~ /sanitization for\s+($san_drv_3)\s+is\s+(\S+)\s+\%\s+complete/ ) {
            logcomment("Abort on drive $san_drv_3 is still in progress, Wait till drive is aborted");
            while (1) {
                logcomment("Check for drive $san_drv_3 abort ..");
                my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk sanitize status" );
                if ( $out !~ /$san_drv_3/ ) {
                    logcomment("Drive $san_drv_3 Aborted ...");
                    last;
                } else {
                    logcomment("Waiting for abort on drive $san_drv_3");
                    sleep 60;
                }
            }
        } else {
            logcomment("Abort on drive $san_drv_3 successful, proceed with next stpes");
        }
        sleep 20;
        logcomment("Check for drive ownership after aborting sanitizing command on drive $san_drv_3");
        $Api_Set_Obj->execute_raw_command( command => "\013" );
        my $ad_out = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk show -n\"" );
        if ( $ad_out =~ /$san_drv_3/ ) {
            logcomment("Drive $san_drv_3 is unassigned, Assign to $FILER_C");
            my $after = $Api_Set_Obj->execute_raw_command( 'command' => "run local sysconfig -v" );
            my $sysid;
            if ( ( $after =~ /\s*System ID:\s+(\d+)\s+/ ) ) {
                $sysid = $1;
            }
            $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk assign $san_drv_3 -o $FILER_C -s $sysid\"" );
        } else {
            logcomment("Drive $san_drv_3 has ownership");
        }
    }

    ################################################
    ## STEP 15
    nvme_eye_catcher( Test => $Test, string => "STEP 15 of 15 : Clean up all Sanitized drives" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("$FILER_C : Check for bad lable drive ");
        my @bad_drv;
        my $bad_o = $Api_Set_Obj->execute_raw_command( 'command' => "run local aggr status -r" );
        foreach my $line ( split /\n/, $bad_o ) {
            if ( $line =~ /bad label/ ) {
                my ($drv_b) = ( $line =~ /bad\s+label\s+(\S+)/ );
                logcomment("$FILER_C : Drive $drv_b is failed / has bad lable");
                push( @bad_drv, $drv_b );
            }
        }
        logcomment("$FILER_C : Bad lable drives are : @bad_drv");
        logcomment("$FILER_C : Unfail all bad lable drives and issue disk zero spares");
        foreach my $d (@bad_drv) {
            logcomment("$FILER_C : Unfail $d ");
            my $prompts_answers = [ ".*you sure you want to continue with disk unfail.*" => 'yes' ];
            my $ou = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk unfail -s $d ", 'connectrec-match_table' => $prompts_answers );
            logcomment("Drive $d unfailing ...") if ( $ou =~ /unfailing/ );
            sleep 10;
        }
        logcomment("$FILER_C : Zero all spare drives");
        my $Api_Set_Objc = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Objc->execute_raw_command( 'command' => " " );
        $Api_Set_Objc->execute_raw_command( 'command' => "run local disk zero spares" );
        logcomment("$FILER_C : All drives were unfailed and zeroed");
    }

}

#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;
    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "MAINT",
        filer_mode      => $Mode
    );
    return $TCD::PASS;
}