#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Compliance.thpl#1 $
#
## @summary Mode
##  Disk Firmware Download in background/foreground mode
#
##
## @description
## Verify that the drive can download disk firmware in foreground and background mode properly
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground
##
## @steps
## The test will execute steps mentioned below:
##
##
## @status  Automated
########################################
### Library functions
##########################################
use strict;
use Data::Dumper;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
  $SFO_LOOP_TIME
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$SFO_LOOP_TIME  = $params->get( 'SFO_LOOP_TIME',  default => 'INFINITE' );
################
# Testcase name
################
my $TC_name;
$TC_name = "333_NADQ02_NVMe_SFO_Test";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home = $ENV{HOME};

my $status    = 0;
my %FILER_API = ();

my $test_wait_time = 12 * 3600;    #12 Hrs test
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( NVMe_SFO => "NVMe Stroage failover test script." );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

     version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );

    my @subtests_m;
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("Filer $FILER_C is in : $filer_state");
        if ( $filer_state !~ /FIRMWARE/ ) {
            push( @subtests_m, subtest( \&nvme_boot_in_req_mode, -runid, "boot_to_load_$FILER_C", -bg, "--", Trans => $Transit_obj, boot_to => "FIRMWARE" ) );
        }
    }
    Subtest::wait_finish( subtest => [@subtests_m] );

    foreach my $Node (@Nodes) {
        my $prompt_fw;
        my $FILER_C        = $Node->name;
        my $host           = host($FILER_C);
        my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ];
        my $prompt_array   = $host->prompt_fw();
        my $prompts        = join( "|", @$prompt_array );
        $prompt_fw = "($prompts)";
        logcomment("DEBUG:This is the prompt regex: $prompt_fw");
        my $Server = new Server( no_cli => 1 );
        $Server->set_execution_server( -Server => $FILER_C );    #set the execution server to the current node
        my @filer_working = $Server->servers();
        logcomment("Number of filers working are @filer_working");
        my $timeout = "7200";                                    # Change time-out in final code.
        logcomment(" Filer - $FILER : Changing state to LOADER to set the option AUTOBOOT to false");
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("Filer $FILER_C is in : $filer_state");
        my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("FILER is in $state state");

        if ( $state =~ /FIRMWARE/ ) {
            logcomment("FILER is in LOADER prompt, setting AUTOBOOT to true");
            $Server->console(
                cmd                        => 'setenv AUTOBOOT true',
                additional_prompts_timeout => 60,
                ignore                     => $ignore_strings,
                additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
            );
        }
        logcomment(" Filer - $FILER : Changing state to boot-menu");
        logcomment("Boot in File system");
    }
    sleep 30;
    my @subtests_C;
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("Filer $FILER_C is in : $filer_state");
        if ( $filer_state !~ /CLI/ ) {
            push( @subtests_C, subtest( \&nvme_boot_in_req_mode, -runid, "boot_to_load_$FILER_C", -bg, "--", Trans => $Transit_obj, boot_to => "CLI" ) );
        }
    }
    Subtest::wait_finish( subtest => [@subtests_C] );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "STEP 1 : Set Privilage level to diag" );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment(" Privilage Level ");
        $Api_Set_Obj->execute_raw_command( 'command' => "set" );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    }
    logcomment("Booted in File system");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000000 );
        my $Host = host($FILER_C);
        logcomment(" Enable autogiveback ");
        my $prompts_answers = [ '.*Do you want to continue.*' => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback true", 'connectrec-match_table' => $prompts_answers );
        my $prompts_answers_1 = [ "Do you want to continue" => 'y' ];
        my $return = $Api_Set_Obj->execute_raw_command( 'command' => 'cluster ha show' );
        $Api_Set_Obj->execute_raw_command( 'command' => 'cluster ha modify -configured true', 'connectrec-match_table' => $prompts_answers_1 );
        logcomment("Check SFO status");
        my $output = $Api_Set_Obj->execute_raw_command( command => "storage fail show" );

    }
    sleep 30;
    return $TCD::PASS;
}

sub NVMe_SFO {
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    logcomment(" User selected : $SFO_LOOP_TIME Loop");
    $SFO_LOOP_TIME = 9999999999 if ( $SFO_LOOP_TIME =~ /INFINITE/ );
    my $loop = 1;
    for ( $loop = 1 ; $loop <= $SFO_LOOP_TIME ; $loop++ ) {
        logcomment("Create API for both nodes");
        @Nodes = shuffle_array( \@Nodes );
        my $NODE_A_CON = $Nodes[0];
        my $NODE_B_CON = $Nodes[1];
        my $FILER_A    = $Nodes[0]->name();
        my $FILER_B    = $Nodes[1]->name();
        logcomment("LOOP $loop : FILER A : $FILER_A");
        logcomment("LOOP $loop : FILER B : $FILER_B");
        ####### STEP 1 #########
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            nvme_eye_catcher( Test => $Test, string => "STEP 1 : Set Privilage level to test" );
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            logcomment(" Privilage Level ");
            $Api_Set_Obj->execute_raw_command( 'command' => "set" );
            my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
            $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0" );
            sleep 30;
        }
        sleep 30;
        ####### STEP 2 #########
        nvme_eye_catcher( Test => $Test, string => "STEP 2  : Print user selected loop count" );
        logcomment("User SELECTED LOOP COUNT : $SFO_LOOP_TIME");

        ####### STEP 3 #########
        nvme_eye_catcher( Test => $Test, string => "STEP 3  : Check Storage failover status" );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            logcomment("$FILER_C : Check storage failover status");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( 'command' => "storage fail show" );
            sleep 30;
        }

        ####### STEP 4 #########
        nvme_eye_catcher( Test => $Test, string => "STEP 4 : Check drives owned be each node" );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            logcomment("$FILER_C : Check storage failover status");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" );
            foreach my $line ( split /\n/, $out ) {
                next if ( $line =~ /disk|^-|displayed/ );
                if ( $line =~ /(\S+)\s+(\S+)/ ) {
                    my $drv  = $1;
                    my $path = $2;
                    if ( ( $path =~ /$FILER_A/ ) && ( $path =~ /$FILER_B/ ) ) {
                        logcomment("Drive $drv has path $path");
                    } else {
                        logcomment("Drive $drv path is incomplete");
                        logcomment("**FATAL** : $FILER_C Drive $drv path is INCOMPLETE");
                        logresult( 'FATAL', "$FILER_C Drive $drv path is INCOMPLETE" );
                        $status = 1;
                    }
                }
            }
            sleep 30;
        }

        ####### STEP 5 #########
        nvme_eye_catcher( Test => $Test, string => "STEP 5 : Issue PANIC on $FILER_A" );
        logcomment("Issue \"PANIC\" Command on $FILER_A");
        logcomment("$FILER_A : Issuing Panic");
        ###
        my $prompt_fw;
        my $host           = host($FILER_A);
        my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ];
        my $prompt_array   = $host->prompt_fw();
        my $prompts        = join( "|", @$prompt_array );
        $prompt_fw = "($prompts)";
        logcomment("DEBUG:This is the prompt regex: $prompt_fw");
        my $Server = new Server( no_cli => 1 );
        $Server->set_execution_server( -Server => $FILER_A );    #set the execution server to the current node
        my @filer_working = $Server->servers();
        logcomment("Number of filers working are @filer_working");
        my $timeout = "7200000";                                 # Change time-out in final code.
        logcomment(" Filer - $FILER_A : Issue PANIC command on filer $FILER_A");
        my $passwd = $host->{conn}->{ssh}->{password};
		logcomment("USER :admin and PASS: $passwd");
        $Server->console(
            cmd                        => 'run local panic',
            additional_prompts_timeout => 6000,
            ignore                     => $ignore_strings,
            additional_prompts         => {
                "run not found"               => "FINAL_PROMPT",
                "$prompt_fw"                  => "FINAL_PROMPT",
                "login"                       => "admin",
                ".*Password.*"                => "$passwd",
				"::>"						  => "FINAL_PROMPT",
				"::*>"						   => "FINAL_PROMPT",
            }
        );
        logcomment("$FILER_A booted up in file system");
        ##
        my $API_A = $NODE_A_CON->get_7m_or_nodescope_apiset( connid => 'console' );
        $API_A->set_timeout( "connectrec-timeout" => 72000000 );
        my $output = $API_A->execute_raw_command( 'command' => "\013" );
		my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $API_A->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        sleep(60);
        my $Transit_obj = NACL::Transit->new( name => $FILER_A );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("Filer $FILER_A is in : $filer_state");

        ####### STEP 6  #########
        nvme_eye_catcher( Test => $Test, string => "STEP 6 :Check node B state and check drive path" );
        logcomment("Check node B state and check drive path");
        my $Transit_obj = NACL::Transit->new( name => $FILER_B );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("Filer $FILER_B is in : $filer_state");
        my $API_B = $NODE_B_CON->get_7m_or_nodescope_apiset( connid => 'console' );
        $API_B->set_timeout( "connectrec-timeout" => 72000000 );
        $API_B->execute_raw_command( 'command' => "\013" );
        my $out = $API_B->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" );

        foreach my $line ( split /\n/, $out ) {
            next if ( $line =~ /disk|^-|displayed/ );
            if ( $line =~ /(\S+)\s+(\S+)/ ) {
                my $drv  = $1;
                my $path = $2;
                if ( ( $path =~ /$FILER_A/ ) && ( $path =~ /$FILER_B/ ) ) {
                    logcomment("Drive $drv has path $path");
                } else {
                    logcomment("Drive $drv path is incomplete");
                    logcomment("**FATAL** : $FILER_A or $FILER_B Drive $drv path is INCOMPLETE");
                    logresult( 'FATAL', "$FILER_A or $FILER_B Drive $drv path is INCOMPLETE" );
                    $status = 1;
                }
            }
        }
        sleep 30;
        ####### STEP 7 #########
        nvme_eye_catcher( Test => $Test, string => "Check Storage failover status " );
        my $API_B = $NODE_B_CON->get_7m_or_nodescope_apiset( connid => 'console' );
        $API_B->set_timeout( "connectrec-timeout" => 72000000 );
        $API_B->execute_raw_command( 'command' => "\013" );
        my $out = $API_B->execute_raw_command( 'command' => "storage fail show" );
        sleep 30;
        ####### STEP 8  #########
        nvme_eye_catcher( Test => $Test, string => "Step 8 : Check disk path on both nodes" );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            my $API = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            logcomment("Checking on filer :$FILER_C  ");
            $API->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" );
            sleep 30;
        }

        logcomment("Loop $loop done, Proceed with next loop if seelcted");
        nvme_eye_catcher( Test => $Test, string => " LOOP $loop COMPLETED" );
        sleep 30;
    }    #End of for loop
    logcomment(" Total test status is : $status ");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

# End of sub_exe
#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );

    ###########################################################################################
    ## Delete/Reset test specific options and values
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Host        = host($FILER_C);
        logcomment("disable autogiveback ");
        my $prompts_answers = [ 'Do you want to disable auto-giveback?' => 'y', 'Do you want to continue' => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback false", 'connectrec-match_table' => $prompts_answers );
    }

    return $TCD::PASS;
}