#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Compliance.thpl#1 $
#
## @summary Mode
##  Disk Firmware Download in background/foreground mode
#
##
## @description
## Verify that the drive can download disk firmware in foreground and background mode properly
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground
##
## @steps
## The test will execute steps mentioned below:
##
##
## @status  Automated
########################################
### Library functions
##########################################
use strict;
use Data::Dumper;
use Storage::Tahiti_Common_Lib;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
  $SFO_LOOP_TIME
  $FILER_SIO
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$SFO_LOOP_TIME  = $params->get( 'SFO_LOOP_TIME',  default => 'INFINITE' );
$FILER_SIO      = $params->get( 'FILER_SIO',      default => '15' );

################
# Testcase name
################
my $TC_name;
$TC_name = "422_NADQ02_NVMe_Th_Enum_GB_TC";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home = $ENV{HOME};
my $READ_LOG_FILE;

my $status    = 0;
my %FILER_API = ();

my $Partner;
my $Client;

my $test_wait_time = 12 * 3600;    #12 Hrs test
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( NVMe_Th_Enum_GB => "NVMe Tahiti Enumeration stress testing - Filer giveback takeover command." );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );
    return $TCD::PASS;
}

########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode,
        Tahiti          => "yes",
    );

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000000 );
        my $Host = host($FILER_C);
        logcomment(" Disabling autogiveback ");
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $prompts_answers = [ '.*Do you want to enable auto-giveback?' => 'y', ".*command?.*" => "y" , ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback true", 'connectrec-match_table' => $prompts_answers );
        sleep 20;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $prompts_answers_1 = [ "Do you want to continue" => 'y' ];
        my $return = $Api_Set_Obj->execute_raw_command( 'command' => 'cluster ha show', 'connectrec-match_table' => $prompts_answers_1 );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        logcomment("Check SFO status");
        sleep 20;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $output = $Api_Set_Obj->execute_raw_command( command => "storage fail show" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        sleep 20;
    }
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("Filer $FILER_C is in : $filer_state");
        if ( $filer_state =~ /UP|CLI/ ) {
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
            logcomment("Set AUTOBOOT as true");
            $Api_Set_Obj->execute_command( command => 'priv set test;bootargs set AUTOBOOT true' );
            logcomment("Autoboot set to true");
        }
        sleep 10;
    }

    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000000 );
        my $Host = host($FILER_C);
        logcomment(" Enable autogiveback ");
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $prompts_answers = [ '.*Do you want to enable auto-giveback?' => 'y', ".*command?.*" => "y" , ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback true", 'connectrec-match_table' => $prompts_answers );
        sleep 20;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $prompts_answers_1 = [ ".*Do you want to continue.*" => 'y' ];
        my $return = $Api_Set_Obj->execute_raw_command( 'command' => 'cluster ha modify -configured true', 'connectrec-match_table' => $prompts_answers_1 );

        my $return = $Api_Set_Obj->execute_raw_command( 'command' => 'cluster ha show' );
        sleep 20;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $output = $Api_Set_Obj->execute_raw_command( command => "storage fail show" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        sleep 20;
    }
    sleep 30;
    logcomment("Check for cluster health");
    logcomment("Check for port status");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name;
        $Node->refresh_command_interface();

        #my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $output = $Api_Set_Obj->execute_raw_command( 'command' => "cluster show" );
        sleep 30;
        my $output_net = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;net int show" );
        sleep 60;
        if ( ( $output =~ /$FILER_C/ ) && ( $output_net =~ /$FILER_C/ ) ) {
            if ( ( $output =~ /true/ ) && ( $output_net !~ /false/ ) ) {
                logcomment("Cluster is enabled on $FILER_C.");
            } else {
                logcomment("\cluster IS NOT ENABLED on $FILER_C.");
                return $TCD::FAIL;
            }
        }
        $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;net port show" );
        sleep 90;
        $Api_Set_Obj->execute_raw_command( 'command' => " " );
    }

    return $TCD::PASS;
}

sub NVMe_Th_Enum_GB {
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    logcomment("Create aggregates and initiate filer sio threads on all aggregate");
    my $filersio_thrds_all;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        $Node->refresh_command_interface();

        #my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000000 );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $Host = host($FILER_C);
        my $fsio = 10;
        sleep 30;
        my $spare_list  = tahiti_sto_agg_show_spare_dsk( Node => $Node );
        my @spare_drvs  = keys %$spare_list;
        my $spare_count = @spare_drvs;
        logcomment("$FILER_C : Total spare are : $spare_count and Spare Drive are : @spare_drvs");
        logcomment("Create new vol");
        logcomment("DEBUG : FILERSIO THREADS : $FILER_SIO");
        sleep 30;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $return_filersio_thrds = nvme_multi_aggr_with_rd_wr_vol_n_fsio(
            API_con  => $Api_Set_Obj,
            filer    => $FILER_C,
            fsio     => $FILER_SIO,
            loc_drvs => \@spare_drvs,
            Node     => $Node,
        );
        my @loc_all_threads;

        foreach my $fil ( keys %$return_filersio_thrds ) {
            my $sio_th = $return_filersio_thrds->{$fil}->{'SIO_CMD'};
            @loc_all_threads = @$sio_th;
        }
        $filersio_thrds_all->{$FILER_C} = [@loc_all_threads];
        logcomment(" $FILER_C completed");
        sleep 30;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        sleep 30;
    }

    $READ_LOG_FILE = "$LOGDIR" . "/" . "$TC_name" . ".log";
    logcomment(" LOG : $READ_LOG_FILE");
    logcomment(" User selected : $SFO_LOOP_TIME Loop");
    $SFO_LOOP_TIME = 9999999999 if ( $SFO_LOOP_TIME =~ /INFINITE/ );
    my $loop = 1;
    for ( $loop = 1 ; $loop <= $SFO_LOOP_TIME ; $loop++ ) {
        logcomment("Create API for both nodes");
        @Nodes = shuffle_array( \@Nodes );
        my $NODE_A_CON = $Nodes[0];
        my $NODE_B_CON = $Nodes[1];
        $NODE_A_CON->refresh_command_interface();
        $NODE_B_CON->refresh_command_interface();
        my $FILER_A = $Nodes[0]->name();
        my $FILER_B = $Nodes[1]->name();
        $Test->nlog("************************************************************");
        $Test->nlog("** Loop $loop : FILER A : $FILER_A  | FILER B : $FILER_B ***");
        $Test->nlog("************************************************************");
        ####### STEP 1 #########
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            tahiti_eye_catcher( Test => $Test, string => "STEP 1 : Set Privilage level to test : Loop $loop" );
            $Node->refresh_command_interface();

            #my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
            logcomment(" Privilage Level ");
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            $Api_Set_Obj->execute_raw_command( 'command' => "set" );
            my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
            $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0" );
        }
        sleep 10;
        ####### STEP 2 #########
        tahiti_eye_catcher( Test => $Test, string => "STEP 2  : Print user selected loop count : Loop $loop" );
        logcomment("User SELECTED LOOP COUNT : $SFO_LOOP_TIME");

        ####### STEP 3 #########
        tahiti_eye_catcher( Test => $Test, string => "STEP 3  : Check Storage failover status : Loop $loop" );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            $Node->refresh_command_interface();
            logcomment("$FILER_C : Check storage failover status");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( 'command' => "storage fail show" );
            $Api_Set_Obj->execute_raw_command( 'command' => " " );
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            sleep 10;
        }

        ####### STEP 4 #########
        tahiti_eye_catcher( Test => $Test, string => "STEP 4 : Check drives owned be each node : Loop $loop" );
        foreach my $Node (@Nodes) {
            $Node->refresh_command_interface();
            my $FILER_C = $Node->name();
            logcomment("$FILER_C : Check storage failover status");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( 'command' => " " );
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            sleep 10;
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            my @failed_drv;
            my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" );
            my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" ) if ( $out eq '' );

            foreach my $line ( split /\n/, $out ) {
                my ( $drv, $path );
                next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change|\[\S+\]/ );
                if ( ( $line =~ /$FILER_A/ ) && ( $line =~ /$FILER_B/ ) ) {
                    my ( $drv, $path ) = $line =~ /(\S+)\s+(\S+)/;
                    logcomment("Drive $drv has path $path");
                } else {
                    my ( $drv, $path ) = $line =~ /\S+\:(\S+)\s+(\S+)/;
                    logcomment("Drive $drv path is incomplete");
                    logcomment("**FATAL** : $FILER_C Drive $drv path is INCOMPLETE");
                    push( @failed_drv, $drv );
                    $status = 1;
                }
            }
            if ( ( $status == 1 ) && (@failed_drv) ) {
                logcomment("**FATAL** : $FILER_C : @failed_drv drives path is INCOMPLETE");
                logresult( 'FATAL', "$FILER_C : @failed_drv drives path is INCOMPLETE" );
            }
            sleep 10;
        }
        #######  STEP 5 #######
        tahiti_eye_catcher( Test => $Test, string => "STEP 5 : check filersio status : Loop $loop" );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            $Node->refresh_command_interface();
            logcomment("$FILER_C : Check filersio status");
            my $Api_Set_Obj     = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            my $i               = 0;
            sleep 2;
            $Api_Set_Obj->execute_command( 'command' => "\013");
            sleep 2;
            my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
            sleep 1;
            $Api_Set_Obj->execute_command( 'command' => "\013");
            $Api_Set_Obj->execute_command( 'command' => "\013");
            my $filersio_stat = $Api_Set_Obj->execute_command( 'command' => "filersio status" );

            if ( $filersio_stat =~ /filersio: instance $i: asyncio_active running/i ) {
                $i++;
                logcomment( "$FILER_C : Thread " . ( $i + 1 ) . " : filersio is running" );
            } else {
                logcomment("$FILER_C : Filer sio is not executing, Re-initiating all filersio threads");
                my $get_fil          = $filersio_thrds_all->{$FILER_C};
                my @all_filersio_thd = @$get_fil;
                foreach my $cmd (@all_filersio_thd) {
                    logcomment("$FILER_C : Re-Start Filer sio threads");
                    logcomment("$cmd");
                    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
                    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    sleep 1;
                    $Api_Set_Obj->execute_raw_command( 'command' => "$cmd" );
                    logcomment("RE-STARTED FILER SIO THREADS");
                }
                logcomment("$FILER_C : All filersio threads re-started");
                logcomment("$FILER_C : Check filersio status");
                my $Api_Set_Obj     = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
                my $i               = 0;
                my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
                $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
                my $filersio_stat = $Api_Set_Obj->execute_command( 'command' => "filersio status" );

                if ( $filersio_stat =~ /filersio: instance $i: asyncio_active running/i ) {
                    $i++;
                    logcomment( "$FILER_C : Thread " . ( $i + 1 ) . " : filersio is running" );
                } else {
                    logcomment("**WARNING** : $FILER_C : filersio cannot be re-started");
                }
            }
        }
        ####### STEP 7 #######
        tahiti_eye_catcher( Test => $Test, string => "STEP 6 : Initiating giveback takeover... : Loop $loop" );
        foreach my $node (@Nodes) {
            $node->refresh_command_interface();
            my $FILER_C = $node->name();
            eye_catcher(
                Test   => $Test,
                string => "$FILER_C :Checking the state of filers and initiating giveback and takeover "
            );
            logcomment("Set rows to 0");

            #my $Api_Set_Obj = $node->get_7m_or_nodescope_apiset( connid => 'console' );
            my $Api_Set_Obj = $node->get_7m_or_nodescope_apiset();
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0" );
            sleep 30;
            logcomment("Verify stroage failover status");
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            my $cmd_out = $Api_Set_Obj->execute_raw_command( 'command' => "storage failover show -node * -fields node,state-description,partner-name" );
            sleep 10;
            my $loop = 1;

            if ( $cmd_out =~ /Connected to/i ) {
                logcomment("Filers connected to each other, Proceed with next steps");
            } else {
                logcomment("FILERS Not connected, Check status");
            }
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            logcomment("Check storage failover autogive back option");
            my $cmd_out = $Api_Set_Obj->execute_raw_command( 'command' => "storage failover show -node * -fields auto-giveback" );
            my $node_name;
            foreach my $line ( split /\n/, $cmd_out ) {
                next if ( $line =~ /---/ );
                if ( $line =~ /(\S+)\s(\S+)/ ) {
                    $node_name = $1;
                    if ( $2 =~ /-|False/i ) {
                        logcomment("auto-giveback for node $node_name is False or Not available , sleeping for 5 min and checking again");
                        sleep(300);
                        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                        $Api_Set_Obj->execute_raw_command( 'command' => "storage failover show -node * -fields auto-giveback" );
                    } else {
                        logcomment("**DEBUG** : Auto-givack is true");
                    }
                }
            }
            sleep 10;
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );

            tahiti_eye_catcher( Test => $Test, string => "$FILER_C : Proceed with giveback take over" );
            $Partner = NACL::STask::Node->get_partner_obj( command_interface => $node );
            my $Partner_name = $Partner->name;
            logcomment("Partner Node NAME is : $Partner_name");
            my $prompts_answers = [ '.*Do you want to continue.*' => 'y' ];
            my $gout = $Api_Set_Obj->execute_raw_command( 'command' => "storage failover takeover -ofnode $Partner_name -option immediate -skip-lif-migration-before-takeover true", 'connectrec-match_table' => $prompts_answers );
            logcomment("Issued takeover command from $FILER_C to $Partner_name");
            logcomment("Wait for few min");
            sleep 300;
            logcomment("Check State of both nodes");
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            my $flag       = 0;
            my $sleep_time = 300;
            my $temp       = 0;

            while (1) {
                logcomment("Checking State of both nodes");
                my $out = $Api_Set_Obj->execute_raw_command( 'command' => "storage failover show -node * -fields node,state-description,partner-name" );
                if ( $out =~ /Auto giveback will be initiated in (\d+) seconds/i ) {
                    logcomment("Auto Give back will be initatied in $1 seconds, Wait for few more min");
                } elsif ( $out =~ /Waiting for cluster applications to come online on the local node/i ) {
                    logcomment("Waiting for cluster applications to come online on the local node");
                    sleep 20;
                    logcomment("Check State of partner node ");
                    my $Transit_obj = NACL::Transit->new( name => $Partner_name );
                    my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
                    logcomment("Filer $Partner_name is in : $filer_state");
                    if ( $filer_state =~ /USERNAME||PASSWORD/i ) {
                        logcomment("$Partner_name is in : $filer_state state, Change to CLI");
                        $Transit_obj->change_state( to => "CLI" );
                        logcomment("Changed to CLI");
                    }
                } elsif ( $out =~ /Connected to/i ) {
                    logcomment("Filer appears to be connected to each other, Verify on both node");
                    foreach my $Node (@Nodes) {
                        my $fl_name = $Node->name;
                        logcomment("Check State of filer : $fl_name");
                        my $API_NODE_IN_CLI = $node->get_7m_or_nodescope_apiset( connid => 'console' );
                        $API_NODE_IN_CLI->execute_raw_command( 'command' => "\013" );
                        my $s_out = $API_NODE_IN_CLI->execute_raw_command( 'command' => "storage failover show -node $fl_name -fields node,state-description" );
                        if ( $s_out =~ /connected to (\S+)/i ) {
                            logcomment("Filer : $fl_name Connected to $1\n");
                            $flag++;
                        } else {
                            logcomment("FILER :$fl_name is not connected to partner");
                        }
                    }
                } else {
                    logcomment("$out");
                }

                $temp = $temp + $sleep_time;

                if ( $flag == 2 ) {
                    last;
                } else {
                    if ( $temp <= 2700 ) {
                        $flag = 0;
                        sleep $sleep_time;
                    } else {
                        logcomment("Checking clusetr port status");
                        my $FILER_C = $Nodes[1]->name;
                        $Partner->refresh_command_interface();
                        my $Api_Set_Obj_Partner = $Partner->get_7m_or_nodescope_apiset();
                        my $output = $Api_Set_Obj_Partner->execute_raw_command( 'command' => "net port show -ipspace Cluster -fields link" );
                        sleep 90;
                        $Api_Set_Obj_Partner->execute_raw_command( 'command' => " " );
                        logcomment("output : $output");
                        my $master_name = $node->name();

                        foreach my $line ( split( /\n/, $output ) ) {
                            next if ( $line =~ /^node/ );
                            if ( $line =~ /$Partner_name\s+(\S+)\s+(\S+)/ || $line =~ /$master_name\s+(\S+)\s+(\S+)/ ) {
                                logcomment("port : $1 link status : $2");
                                my $link_status = $2;
                                if ( $link_status =~ /down/i ) {
                                    logcomment("**FAIL** : Cluster port is down.So exiting from the test");
                                }
                            }

                        }

                        logcomment("Nodes are still in Partial giveback even after waiting for 45 min");
                        logcomment("**FAIL** : Nodes are still in partial giveback even after 45 min");
                        logresult( 'FAIL', "Nodes are still in partial giveback even after 45 min" );
                        return $TCD::FAIL;
                    }
                }

            }    #end of while

            logcomment("After giveback ");
            my $manual_gvbck = 0;
            my $FILER_IN_CLI;
            my $API_NODE_IN_CLI;
            my $FILER_IN_GB;

            foreach my $node (@Nodes) {
                $node->refresh_command_interface();
                my $FILER_C        = $node->name();
                my $Transit_obj    = NACL::Transit->new( name => $FILER_C );
                my $Original_State = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
                logcomment("Filer  $FILER_C  is in  $Original_State\n");
                if ( $Original_State =~ /USERNAME|PASSWORD|CLI|UP/i ) {
                    $Transit_obj->change_state( to => "CLI" ) if ( $Original_State =~ /USERNAME|PASSWORD/ );
                    logcomment("$FILER_C state changed to CLI or was in CLI");
                    $Original_State = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
                    logcomment("Filer  $FILER_C  is in  $Original_State\n");
                    $API_NODE_IN_CLI = $node->get_7m_or_nodescope_apiset( connid => 'console' );
                    $FILER_IN_CLI = $FILER_C;
                }
                if ( $Original_State =~ /TAKEN_OVER_WAITING_ONTAP/i ) {
                    logcomment("DEBUG : $FILER_C is in $Original_State");
                    $FILER_IN_GB  = $FILER_C;
                    $manual_gvbck = 1;
                }
            }
            logcomment("DEBUG CHECK : CLI : $FILER_IN_CLI | GB:  $FILER_IN_GB ");
            logcomment("DEBUG CHECKD : CLI: $API_NODE_IN_CLI | Manual GB :$manual_gvbck");
            if ( $manual_gvbck == 1 ) {
                logcomment("Manually issue giveback from node : $FILER_IN_CLI");
                logcomment("FILER : $FILER_IN_CLI is in CLI, and $FILER_IN_GB is in Waiting for giveback");
                logcomment("Enter Giveback command from $FILER_IN_CLI");
                logcomment("DEBUG : API CREATED : $API_NODE_IN_CLI");
                logcomment("FILER which is in UP State : $FILER_IN_CLI and FILER WAITING FOR GIVEBACK : $FILER_IN_GB");
                my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
                logcomment("Execute giveback from $FILER_IN_CLI");
                $API_NODE_IN_CLI->execute_raw_command( 'command' => "storage failover giveback -ofnode $FILER_IN_GB -override-vetoes true", 'connectrec-match_table' => $prompts_answers );
                sleep 400;
            }

            foreach my $node (@Nodes) {
                while (1) {
                    $node->refresh_command_interface();
                    my $FILER_C = $node->name();
                    logcomment("Re-check filer State ");
                    my $Transit_obj = NACL::Transit->new( name => $FILER_C );
                    my $new_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
                    logcomment("Filer new state : $new_state");
                    if ( $new_state =~ /USERNAME|PASSWORD/ ) {
                        $Transit_obj->change_state( to => "CLI" );
                        logcomment("$FILER_C Change to CLI");
                        my $Original_State = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
                        logcomment("Filer  $FILER_C  is in  $Original_State\n");
                        last;
                    } elsif ( $new_state =~ /CLI|UP/ ) {
                        logcomment("$FILER_C : FILER STATE is $new_state ");
                        last;
                    } elsif ( $new_state =~ /LOADER/ ) {
                        logcomment("$FILER_C : Filer is in LOADER, Check logs and bootargs");
                        last;
                    } else {
                        logcomment("$FILER_C : FILER STATE CANNOT BE VERFIED, Wait For few more minutes ..");
                        sleep 60;
                    }
                }
                logcomment("$FILER_C : Get Console connetion ");
                my $API = $node->get_7m_or_nodescope_apiset( connid => 'console' );
                $API->execute_raw_command( 'command' => " " );
                $API->execute_raw_command( 'command' => "\013" );
            }

            ####### STEP 6  #########
            tahiti_eye_catcher( Test => $Test, string => "STEP 7 :Check drive path : Loop $loop" );
            logcomment("$FILER_C: Check node state and check drive path");
            my $Transit_obj = NACL::Transit->new( name => $FILER_C );
            my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
            logcomment("Filer $FILER_B is in : $filer_state");
            my $API_aB = $node->get_7m_or_nodescope_apiset( connid => 'console' );
            $API_aB->execute_raw_command( 'command' => " " );
            $API_aB->set_timeout( "connectrec-timeout" => 72000000 );
            $API_aB->execute_raw_command( 'command' => "\013" );
            my $out = $API_aB->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" );
            my $out = $API_aB->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" ) if ( $out eq '' );
            my @failed_drv;

            foreach my $line ( split /\n/, $out ) {
                my ( $drv, $path );
                next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change|\[\S+\]/ );
                if ( ( $line =~ /$FILER_A/ ) && ( $line =~ /$FILER_B/ ) ) {
                    my ( $drv, $path ) = $line =~ /(\S+)\s+(\S+)/;
                    logcomment(" Drive $drv has path $path");
                } else {
                    my ( $drv, $path ) = $line =~ /\S+\:(\S+)\s+(\S+)/;
                    logcomment("$FILER_C :Drive $drv path is incomplete");
                    logcomment("**FATAL** : $FILER_C : Drive $drv path is INCOMPLETE");
                    push( @failed_drv, $drv );
                    $status = 1;
                }
            }
            ## validate check conditions
            logcomment("$FILER_C : Check for scsi check Condition ");
            logcomment("$FILER_C : Log file to read : $READ_LOG_FILE");
            open( FH, "$READ_LOG_FILE" ) || logcomment("Cannot open log file to validate check conditions");
            my @fl_cont = ();
            @fl_cont = <FH>;
            close(FH);
            foreach my $ln (@fl_cont) {

                if ( $ln =~ /\(0x2 - 0x4 (\w+x\w+) \w+x\w+\)|\(0x2 - 0x4 0x0 0x82\)/ ) {
                    logcomment("$FILER_C : scsi check Condition -  (0x2 - 0x4 0x0 0x82) FOUND !!!! ");
                    logcomment("$FILER_C : DEBUG : $ln");
                    $status = 1;
                }
            }
            if ( $status == 1 ) {
                my $Host = host($FILER_C);
                logcomment("$FILER_C : Drive path is incomplete or scsi check Condition -  (0x2 - 0x4 0x0 0x82) FOUND !!!");
                logcomment("$FILER_C : collect memory dumps from the drives");
                $API_aB->execute_raw_command( 'command' => "\013" );
                my $Api_Set_Obj_SS = NACL::APISet->new(
                    hostobj   => $Host,
                    category  => "Node",
                    interface => "CLI",
                    set       => "Systemshell"
                );
                logcomment("System shell API created, getting log pages ");
                my $ss_disk_list = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol devlist" );
                my $drv_result = {};

                foreach my $line ( split( /\n/, $ss_disk_list ) ) {
                    my ( $drv, $ns, $nsz );
                    if ( $line =~ /(\S+)\:\s+(\S+)/ ) {
                        $drv = $1;
                        $drv_result->{$drv}->{'PRDID'} = $2;
                    }
                }
                my @nv_drv = keys(%$drv_result);
                logcomment("System Shell Drive(s) : @nv_drv");
                foreach my $drv (@nv_drv) {
                    logcomment("Dumping memory log infromation for $drv");
                    $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol vu_dump_ss retrieve -o memory $drv" );
                    sleep 20;
                    logcomment("Dump crash logs for drive  $drv");
                    $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol vu_dump_ss retrieve -o crash $drv" );
                    sleep 20;
                    logcomment("Dump debug logs for drive $drv");
                    $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol vu_dump_ss retrieve -o debug $drv" );
                    sleep 20;
                }
                logcomment("$FILER_C : Copy logs under main log directory");
                my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
                my $log_dir = "$actual_log_dir\/DRV_MEMORY_LOGS_\/" . $FILER_C;
                if ( !( -d "$log_dir" ) ) {
                    system("sudo mkdir -p $log_dir");
                }
                my $filer_path = "/mroot/etc/log/peg_nvme*";
                logcomment("Copying logs from $filer_path");
                copy_files_to_client_from_filer(
                    node_present => $node,
                    filer_path   => $filer_path,
                    log_path     => $log_dir,
                );
                $API_aB->execute_raw_command( 'command' => "\013" );
                $API_aB->execute_raw_command( 'command' => "\013" );
                logresult( 'FATAL', "$FILER_A or $FILER_B Drive path is INCOMPLETE, scsi check Condition -  (0x2 - 0x4 0x0 0x82) FOUND !!!" );
            }
        }
        sleep 30;
        ####### STEP 7 #########
        tahiti_eye_catcher( Test => $Test, string => "STEP 8 : Check Storage failover status : Loop $loop" );
        my $API_B = $NODE_B_CON->get_7m_or_nodescope_apiset( connid => 'console' );
        $NODE_B_CON->refresh_command_interface();
        $API_B->set_timeout( "connectrec-timeout" => 72000000 );
        $API_B->execute_raw_command( 'command' => "\013" );
        my $out = $API_B->execute_raw_command( 'command' => "storage fail show" );
        sleep 30;
        $Test->nlog("**********************************************************");
        $Test->nlog("** Loop $loop done, Proceed with next loop if selected ***");
        $Test->nlog("**********************************************************");
        tahiti_eye_catcher( Test => $Test, string => " LOOP $loop COMPLETED" );
        sleep 30;
    }    #End of for loop
    logcomment(" Total test status is : $status ");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

# End of sub_exe
#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode,
        Tahiti          => "yes",
    );

    ###########################################################################################
    ## Delete/Reset test specific options and values
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Host        = host($FILER_C);
        logcomment("disable autogiveback ");
        my $prompts_answers = [ '.*Do you want to disable auto-giveback?' => 'y', ".*command?.*" => "y" , ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback false", 'connectrec-match_table' => $prompts_answers );
        logcomment("Stop filersio instances ");
        my $filersio_stat = $Api_Set_Obj->execute_command( 'command' => "filersio stop" );
        logcomment("Filer sio stopped");
        logcomment("Delete all volume , vserver and aggregate ");

        aggregate_volume_delete( Node => $Node );
        sleep( rand(100) );
        vserver_delete( Node => $Node );
        sleep(15);
        $Api_Set_Obj->execute_raw_command( command => "run local disk zero spares" );
        sleep 20;
        $Api_Set_Obj->execute_raw_command( command => "\013" );
    }

    return $TCD::PASS;
}