#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Compliance.thpl#1 $
#
## @summary Mode
##  Disk Firmware Download in background/foreground mode
#
##
## @description
## Verify that the drive can download disk firmware in foreground and background mode properly
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground
##
## @steps
## The test will execute steps mentioned below:
##
##
## @status  Automated
########################################
### Library functions
##########################################
use strict;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
use Server;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
  $USER_LOOPS
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$USER_LOOPS     = $params->get( 'USER_LOOPS',     default => '1' );

################
# Testcase name
################
my $TC_name;
$TC_name = "426_NADQ02_NVMe_Sys_PwrCyc_BMC_Loader";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home = $ENV{HOME};

my $test_wait_time = 12 * 3600;    #12 Hrs test
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( NVMe_System_Pwr_Cycle => "AC system power cycle script." );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    version_test( node_present => $node_ref, tc_name => $TC_name );
    my @Filer_array;
    foreach my $temp_node (@Nodes) {
        my $Filer = $temp_node->name();
        push( @Filer_array, $Filer );
    }

    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
    # Check filer State and change to required state
    $Test->nlog("##################################################################################");
    $Test->nlog("##");
    $Test->nlog("## Begining of PRE_TEST logs for Filer : @Filer_array");
    $Test->nlog("##");
    $Test->nlog("##################################################################################");

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "Check state of filers" );
        my $Transit_obj;
        $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        $Test->nlog("$FILER_C : Filer is in $filer_state ");
    }

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name;
        nvme_eye_catcher( Test => $Test, string => "Change filer state to LOADER and start test" );
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        $Transit_obj->change_state( to => "FIRMWARE" );
        sleep 60;
        logcomment("Set AUTO BOOT to False ");
        my $Host           = host($FILER_C);
        my $prompt_array   = $Host->prompt_fw();
        my $prompts        = join( "|", @$prompt_array );
        my $prompt_fw      = "($prompts)";
        my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ];
        logcomment("DEBUG:This is the prompt regex: $prompt_fw");
        my $Server = new Server( no_cli => 1 );
        $Server->set_execution_server( -Server => $FILER_C );
        logcomment("Filer halted, Set AUTOBOOT to FALSE");
        $Server->console(
            cmd                        => 'setenv AUTOBOOT false;saveenv;printenv',
            additional_prompts_timeout => 60,
            ignore                     => $ignore_strings,
            additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
        );

    }

    $Test->nlog("##################################################################################");
    $Test->nlog("##");
    $Test->nlog("## End of PRE_TEST logs for Filer : @Filer_array");
    $Test->nlog("##");
    $Test->nlog("##################################################################################");

    return $TCD::PASS;
}

sub NVMe_System_Pwr_Cycle {
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    my $drv_summary = {};

    #STEP 1
    nvme_eye_catcher( Test => $Test, string => "STEP 1 of 7 : Check State of System, and Halt System" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Transit_obj;
        $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        $Test->nlog("$FILER_C : Filer is in $filer_state ");
        if ( $filer_state !~ /FIRMWARE|LOADER/ ) {
            logcomment("Change filer state to LOADER and start test");
            $Transit_obj->change_state( to => "FIRMWARE" );
            sleep 20;
        }

    }

    #STEP 2
    nvme_eye_catcher( Test => $Test, string => "STEP 2 of 7 : Set Privilage to test" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Host    = host($FILER_C);

        my $Api_Set_Obj = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Firmware",
        );
        logcomment("$FILER_C : Firmware API created");
        $Api_Set_Obj->execute_raw_command( command => "priv set test" );
        sleep 3;
    }

    # STEP 3
    nvme_eye_catcher( Test => $Test, string => "STEP 3 of 7 : Print user selected loop count " );
    logcomment("User selected : $USER_LOOPS");
    logcomment("Step 4 to 7 will execute $USER_LOOPS loops");
    for ( my $i = 1 ; $i <= $USER_LOOPS ; $i++ ) {
        logcomment("Loop : $i BEGIN");

        #STEP 4
        my $nvme_drive_count_s4 = {};
        nvme_eye_catcher( Test => $Test, string => "STEP 4 of 7 : Check Total NVMe drives present on both nodes - Loop $i" );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            my $Host    = host($FILER_C);

            my $Api_Set_Obj = NACL::APISet->new(
                hostobj   => $Host,
                category  => "Node",
                interface => "CLI",
                set       => "Firmware",
            );
            logcomment("$FILER_C : Checking NVMe Drives ");
            my $output = $Api_Set_Obj->execute_raw_command( command => "show pci" );
            foreach my $line ( split( /\n/, $output ) ) {
                if ( $line =~ /mass storage subclass/ ) {
                    if ( $line =~ /(\S+)\:\s+(.*)\s+\(/ ) {
                        my $slot   = $1;
                        my $vendor = $2;
                        $slot =~ s/\//-/g;
                        $nvme_drive_count_s4->{$FILER_C}->{$slot}->{'Vendor'} = $vendor;
                    }
                }
            }

            my @drives  = keys %{ $nvme_drive_count_s4->{$FILER_C} };
            my $drv_tot = @drives;
            logcomment("$FILER_C : NVMe Drives : @drives");
            $drv_summary->{$FILER_C}->{'drvs_s4'}      = [@drives];
            $drv_summary->{$FILER_C}->{'s4_drv_count'} = $drv_tot;
            logcomment("$FILER_C : Drive capture completed");
        }

        #STEP 5

        foreach my $Node (@Nodes) {
            my $FILER_C       = $Node->name();
            my $host          = host($FILER_C);
            my $SFO_LOOP_TIME = 10;
            nvme_eye_catcher( Test => $Test, string => "STEP 5 : Press CTRL+G  on $FILER_C and login to BMC - Loop $i" );
            my $prompt_fw;
            my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ];
            my $prompt_array   = $host->prompt_fw();
            my $bmc_user       = $host->BMC_user();
            my $bmc_pass       = $host->BMC_password();
            logcomment("BMC USER : $bmc_user");
            logcomment("BMC password : $bmc_pass");

            if ( $bmc_user eq '' ) {
                logcomment("**WARNING** : Default BMC user - admin and Password : netapp1! will be used");
                $bmc_user = "admin";
                $bmc_pass = "netapp1!";
            }
            my $prompts = join( "|", @$prompt_array );
            $prompt_fw = "($prompts)";
            logcomment("DEBUG:This is the prompt regex: $prompt_fw");
            my $Server = new Server( no_cli => 1 );
            $Server->set_execution_server( -Server => $FILER_C );    #set the execution server to the current node
            my $timeout = 7200 * $SFO_LOOP_TIME;                     # Change time-out in final code
            logcomment(" Filer - $FILER_C : Press CTRL+G and login to BMC");
            my $Transit_obj = NACL::Transit->new( name => $FILER_C );
            my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
            logcomment("Filer $FILER_C is in : $filer_state");
            eval {
                $Server->console(
                    cmd                        => "\cG",
                    additional_prompts_timeout => 300,
                    ignore                     => $ignore_strings,
                    additional_prompts         => {
                        "$prompt_fw"   => "FINAL_PROMPT",
                        ".*Login.*"    => "FINAL_PROMPT",
                        ".*Password.*" => "FINAL_PROMPT",
                        "\#"           => "cli",
                        "BMC Login:" => "FINAL_PROMPT",
                        ".*Login:.*" => "FINAL_PROMPT",
                      }

                );
            };

            if ($@) {
                logcomment("Chck DEBUG");
                if ( $@ =~ /Error sending the command|Login incorrect/i ) {
                    $Server->console(
                        cmd                        => "\cG",
                        additional_prompts_timeout => 300,
                        ignore                     => $ignore_strings,
                        additional_prompts         => {
                            "$prompt_fw"   => "FINAL_PROMPT",
                            ".*login.*"    => "admin",
                            ".*Password.*" => "netapp1!",
                            "BMC>"         => "FINAL_PROMPT",
                            ".*>.*"        => "FINAL_PROMPT",
                            "BMC Login:"   => "admin",
                            ".*password.*" => $bmc_pass,
                        }
                    );
                }
            }
            logcomment("Press Enter ");
            $Server->console(
                cmd                        => "\013",
                additional_prompts_timeout => $timeout,
                ignore                     => $ignore_strings,
                additional_prompts         => {
                    "$prompt_fw" => "FINAL_PROMPT",
                    "BMC>"       => "FINAL_PROMPT",
                    ".*>.*"      => "FINAL_PROMPT",
                    ">"          => "FINAL_PROMPT",
                    ".*BMC.*"    => "FINAL_PROMPT",
                    ".*Login:"   => "FINAL_PROMPT",
                    "BMC Login:" => "FINAL_PROMPT",
                }
            );
            logcomment("Enter user name and password");
            $Server->console(
                cmd                        => "\013",
                additional_prompts_timeout => $timeout,
                ignore                     => $ignore_strings,
                additional_prompts         => {
                    "$prompt_fw"  => "FINAL_PROMPT",
                    "BMC Login:"  => $bmc_user,
                    ".*Login:"    => $bmc_user,
                    ".*login.*"   => $bmc_user,
                    ".*:"         => $bmc_user,
                    "Password:"   => $bmc_pass,
                    ".*Password:" => $bmc_pass,
                    ".*>"         => "FINAL_PROMPT",
                    ".*>.*"       => "FINAL_PROMPT",
                    ">"           => "FINAL_PROMPT",
                }
            );
            logcomment("BMC Login Successful");
            logcomment("Filer $FILER_C is in : BMC prompt");
            logcomment("Exeucte \"system reset current \" on filer $FILER_C");
            logcomment("Issuing system reset current");
            $Server->console(
                cmd                        => "system reset current",
                additional_prompts_timeout => $timeout,
                ignore                     => $ignore_strings,
                additional_prompts         => {
                    "$prompt_fw"                                             => "FINAL_PROMPT",
                    ".*:~\$.*"                                               => "FINAL_PROMPT",
                    ".*SP.*"                                                 => "FINAL_PROMPT",
                    ".*This will cause a dirty shutdown of your appliance.*" => "y",
                    ".*This command will trigger a destage of ONTAP.*"       => "y",
                    ".*Continue.*"                                           => "y",
                    ".*]:.*"                                                 => "FINAL_PROMPT",
                    "BMC>"                                                   => "FINAL_PROMPT",
                    ">"                                                      => "FINAL_PROMPT",
                    ".*$FILER_C\>"                                           => "FINAL_PROMPT",
                }
            );
            logcomment("Press Ctrl+D to exit from BMC prompt and wait for 120 sec");
            $Server->console(
                cmd                        => "\cD",
                additional_prompts_timeout => $timeout,
                ignore                     => $ignore_strings,
                additional_prompts         => {
                    "$prompt_fw" => "FINAL_PROMPT",
                    ".*LOADER.*" => "FINAL_PROMPT",
                    ".*login.*"  => "FINAL_PROMPT",
                    "login"      => "FINAL_PROMPT",
                    "login:"     => "FINAL_PROMPT",
                    "::>"        => "FINAL_PROMPT",
                    "::*>"       => "FINAL_PROMPT",
                    "\#"         => "FINAL_PROMPT",
                }
            );
            sleep 120;
            logcomment("$FILER_C is in Prompt - \# ,Issue Ctrl+D again");
            logcomment("Press Ctrl+D to exit from BMC prompt");
            $Server->console(
                cmd                        => "\cD",
                additional_prompts_timeout => $timeout,
                ignore                     => $ignore_strings,
                additional_prompts         => {
                    "$prompt_fw" => "FINAL_PROMPT",
                    ".*LOADER.*" => "FINAL_PROMPT",
                    ".*login.*"  => "FINAL_PROMPT",
                    "login"      => "FINAL_PROMPT",
                    "login:"     => "FINAL_PROMPT",
                    "::>"        => "FINAL_PROMPT",
                    "::*>"       => "FINAL_PROMPT",
                }
            );
            logcomment("Check state of filer $FILER_C after issuing system reset current from BMC prompt");
            my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
            logcomment("$FILER_C is in : $filer_state after system reset current");
        }

        #STEP 6
        my $nvme_drive_count_s6 = {};
        nvme_eye_catcher( Test => $Test, string => "STEP 6 of 7 : Check Total NVMe drives present on both nodes - Loop $i" );
        foreach my $Node (@Nodes) {
            my $FILER_C     = $Node->name();
            my $Host        = host($FILER_C);
            my $Api_Set_Obj = NACL::APISet->new(
                hostobj   => $Host,
                category  => "Node",
                interface => "CLI",
                set       => "Firmware",
            );
            logcomment("$FILER_C : Checking NVMe Drives ");
            my $output = $Api_Set_Obj->execute_raw_command( command => "show pci" );
            foreach my $line ( split( /\n/, $output ) ) {
                if ( $line =~ /mass storage subclass/ ) {
                    if ( $line =~ /(\S+)\:\s+(.*)\s+\(/ ) {
                        my $slot   = $1;
                        my $vendor = $2;
                        $slot =~ s/\//-/g;
                        $nvme_drive_count_s6->{$FILER_C}->{$slot}->{'Vendor'} = $vendor;
                    }
                }
            }

            my @drives  = keys %{ $nvme_drive_count_s6->{$FILER_C} };
            my $drv_tot = @drives;
            logcomment("$FILER_C : NVMe Drives : @drives");
            $drv_summary->{$FILER_C}->{'drvs_s6'}      = [@drives];
            $drv_summary->{$FILER_C}->{'s6_drv_count'} = $drv_tot;
            logcomment("$FILER_C : Drive capture completed");
        }

        #STEP 7
        nvme_eye_catcher( Test => $Test, string => "STEP 7 of 7 : Compare Drive count before and after power cycle - Loop $i" );
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            logcomment("$FILER_C : Check NVMe drive count before and after powercycle");
            my $drv_cnt_s4 = $drv_summary->{$FILER_C}->{'s4_drv_count'};
            my $drv_cnt_s6 = $drv_summary->{$FILER_C}->{'s6_drv_count'};
            my $drv_s4     = $drv_summary->{$FILER_C}->{'drvs_s4'};
            my $drv_s6     = $drv_summary->{$FILER_C}->{'drvs_s6'};
            if ( $drv_cnt_s4 == $drv_cnt_s6 ) {
                logcomment("$FILER_C : NVMe Drive count matched");
                logcomment("$FILER_C : Before Pwr cycle : $drv_cnt_s4 | After Pwr cycle : $drv_cnt_s6");
                logcomment("$FILER_C : Drive/Slots Before : @$drv_s4");
                logcomment("$FILER_C : Drive/Slots After : @$drv_s6");
            } else {
                logcomment("$FILER_C : NVMe Drive count MISMATCHED !!!");
                logcomment("$FILER_C : Before Pwr cycle : $drv_cnt_s4 | After Pwr cycle : $drv_cnt_s6");
                logcomment("$FILER_C : Drive/Slots Before : @$drv_s4");
                logcomment("$FILER_C : Drive/Slots After : @$drv_s6");
                $status = 1;
                logcomment("**FATAL** : $FILER_C : NVMe Drive count MISMATCHED !! Before Pwr cycle : $drv_cnt_s4 | After Pwr cycle : $drv_cnt_s6" );
                logresult( 'FATAL', "$FILER_C  : NVMe Drive count MISMATCHED !! Before Pwr cycle : $drv_cnt_s4 | After Pwr cycle : $drv_cnt_s6" );
            }
        }
        logcomment("Loop : $i COMPLETED");
    }    #end of loop
    logcomment("Total test status is : $status");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }

}

# End of sub_exe
#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;
    my @Filer_array;
    foreach my $temp_node (@Nodes) {
        my $Filer = $temp_node->name();
        push( @Filer_array, $Filer );
    }

    $Test->nlog("##################################################################################");
    $Test->nlog("##");
    $Test->nlog("## Begining of POST_TEST logs for Filer : @Filer_array");
    $Test->nlog("##");
    $Test->nlog("##################################################################################");

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        nvme_eye_catcher( Test => $Test, string => "Check state of filers" );
        my $Transit_obj;
        $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        $Test->nlog("$FILER_C : Filer is in $filer_state ");
    }

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name;
        nvme_eye_catcher( Test => $Test, string => "Change filer state to LOADER" );
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        $Transit_obj->change_state( to => "FIRMWARE" );
        sleep 60;
    }

    $Test->nlog("##################################################################################");
    $Test->nlog("##");
    $Test->nlog("## End of POST_TEST logs for Filer : @Filer_array");
    $Test->nlog("##");
    $Test->nlog("##################################################################################");

    return $TCD::PASS;
}