#!/usr/software/bin/perl
## All rights reserved.
##
##
## @description
## Evaluates Fundamental freset disk latency command
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  There is no dependency
##
## @steps
## The test will execute steps mentioned below:
##		1 : Set Privilage level to test
##		2 : Print user selected interval for freset
##		3 : Check drives owned by each node
##      	4 : Start disk latency command on all spares for 4 hrs
##		5 : Issue F-RESET on all drives and execute disk IO latency on all drives
##		6 : Check  drive path from both nodes
##
## @status  Automated
## @author  mvemuri
## @burt    1221710
##
########################################
### Library functions
##########################################
use strict;
use Data::Dumper;
use Storage::NVMe_Common_Lib;
use Storage::Tahiti_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
#Global parameters
use vars qw(
  $FILER
  $MODE
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
  $TOT_RUNTIME
  $INTERVAL
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$TOT_RUNTIME    = $params->get( 'TOT_RUNTIME',    default => '14400' );                       #4hrs
$INTERVAL       = $params->get( 'INTERVAL',       default => '60' );
################
# Testcase name
################
my $TC_name;
$TC_name = "631_NADQ02_NVMe_Th_Freset_Dsk_Latency";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my %test_details;
my @failed_drv;
my $Home      = $ENV{HOME};
my $status    = 0;
my %FILER_API = ();

#my $test_wait_time = 12 * 3600;    #12 Hrs test
my $test_wait_time = 21600;    ##equal to 6 hrs
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( NVMe_Th_SFO => "Tahiti Freset Dsk Latency test script." );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );
    return $TCD::PASS;
}

########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode,
        Tahiti          => "yes"
    );
    sleep 30;
    return $TCD::PASS;
}

sub NVMe_Th_SFO {
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    logcomment(" User selected interval / sleep time for freset : $INTERVAL");
    logcomment(" Total run time: $TOT_RUNTIME");
    logcomment("Disk IO Latency runs for : 4 hrs");
    my @main_sub;

    foreach my $Node (@Nodes) {
        my $FILER_C       = $Node->name();
        my $READ_LOG_FILE = "$LOGDIR" . "/" . "$TC_name" . "/" . "FRESET_IO_$FILER_C" . "/" . "issue_reset_all_drv_$FILER_C" . ".log";
        logcomment(" LOG : $READ_LOG_FILE");
        push( @main_sub, subtest( \&freset_all_interv, -runid, "FRESET_IO_$FILER_C", -bg, "--", $Node, $INTERVAL, $TOT_RUNTIME, $READ_LOG_FILE ) );
    }

    Subtest::wait_finish( subtest => [@main_sub] );
    my $return = status_return( subtest_ref => [@main_sub] );
    logcomment("Test status is $return");
    if ( $return == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }

}

sub freset_all_interv {
    my ( $Node, $interval, $tot_run_time, $log_2_read ) = @_;
    logcomment("Dwell time / Wait time between reset : $interval");
    logcomment("Total runtime : $tot_run_time");
    logcomment("Log file to read : $log_2_read");
    my $FILER_C = $Node->name();
    my @loc_spare;
    logcomment("$FILER_C : get all spares owned");
    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
    my $sp_out = $Api_Set_Obj->execute_raw_command( 'command' => "run local aggr status -s" );

    foreach my $line ( split( /\n/, $sp_out ) ) {
        next if ( $line =~ /disk|Device|^-/ );
        if ( $line =~ /(\S+)\s+(\S+\.\S+\.\S+\.\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ) {
            my $drv = $2;
            push( @loc_spare, $drv );
        }
    }
    my @disk_info = sort(@loc_spare);
    logcomment("Spares available on filer :$FILER_C : @disk_info");
    logcomment("Test will execute on Node : $FILER_C");
    tahiti_eye_catcher( Test => $Test, string => "FILER-$FILER_C ::: Total execution time :  $tot_run_time seconds" );
    my $FILER_A = $Nodes[0]->name();
    my $FILER_B = $Nodes[1]->name();
    logcomment("FRESET and IO LATENCY will be executed on FILER A : $FILER_A");
    logcomment("FILER NAMEs to CHECK PATH : $FILER_A and $FILER_B");
    ####### STEP 1 #########
    tahiti_eye_catcher( Test => $Test, string => "$FILER_C : STEP 1 : Set Privilage level to test" );
    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
    logcomment(" Privilage Level ");
    $Api_Set_Obj->execute_raw_command( 'command' => "set" );
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0" );
    sleep 30;
    ####### STEP 2 #########
    tahiti_eye_catcher( Test => $Test, string => "$FILER_C : STEP 2  : Print user selected interval for freset" );
    logcomment("User SELECTED LOOP COUNT : $INTERVAL");

    ####### STEP 3 #########
    tahiti_eye_catcher( Test => $Test, string => "$FILER_C : STEP 3 : Check drives owned by each node" );
    logcomment("$FILER_C : Check storage failover status");
    my $status = check_drive_path($Node);
    if ( ( $status == 1 ) && (@failed_drv) ) {
        logcomment("**FATAL** : $FILER_C : @failed_drv drives path is INCOMPLETE");
        $status = 1;
        return logresult( "INFO", msg => $status );
    }
    sleep 30;

    ####### STEP 4 #########
    tahiti_eye_catcher( Test => $Test, string => "$FILER_C : STEP 4 : Start disk latency command on all spares for 4 hrs" );
    my @m_subtest;
    my $SSH_FILER_A = $Node->get_7m_or_nodescope_apiset( connid => "ssh", conntype => "ssh-persist" );
    push( @m_subtest, subtest( \&exe_disk_io_lat, -runid, "disk_io_latency_$FILER_C", -bg, "--", $SSH_FILER_A, \@disk_info, $tot_run_time ) );
    logcomment("Started disk IO latency on $FILER_C");
    Subtest::wait_finish( subtest => [@m_subtest] );

    ####### STEP 5 #########
    tahiti_eye_catcher( Test => $Test, string => "$FILER_C : STEP 5 : Issue F-RESET on all drives and execute disk IO latency on all drives" );
    logcomment("Execute IO latency on all drives");
    my $API_NO_A = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
    push( @m_subtest, subtest( \&exe_multiple_reset, -runid, "issue_reset_all_drv_$FILER_A", -bg, "--", $API_NO_A, \@disk_info, $interval, $tot_run_time ) );
    logcomment("Executing disk io latency and freset on all drives in parallel ");
    Subtest::wait_finish( subtest => [@m_subtest] );

    logcomment("Proceed with next freset loop");
    ####### STEP 6  #########
    tahiti_eye_catcher( Test => $Test, string => "$FILER_C : STEP 6 :Check  drive path from both nodes" );
    logcomment("$FILER_C : Check storage failover status");
    check_drive_path($Node);
    ## validate check conditions
    logcomment("Check for scsi check Condition ");
    logcomment("Log file to read : $log_2_read");
    open( FH, "$log_2_read" ) || logcomment("Cannot open log file to validate check conditions");
    my @fl_cont = ();
    @fl_cont = <FH>;
    close(FH);
    foreach my $ln (@fl_cont) {

        if ( $ln =~ /\(0x2 - 0x4 (\w+x\w+) \w+x\w+\)|\(0x2 - 0x4 0x0 0x82\)/ ) {
            logcomment("scsi check Condition -  (0x2 - 0x4 0x0 0x82) FOUND !!!! ");
            logcomment("DEBUG : $ln");
            $status = 1;
        }
    }
    if ( $status == 1 ) {
        my $Host = host($FILER_C);
        logcomment("Drive path is incomplete or scsi check Condition -  (0x2 - 0x4 0x0 0x82) FOUND !!!");
        logcomment("collect memory dumps from the drives");
        my $Api_Set_Obj_SS = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Systemshell"
        );
        logcomment("System shell API created, getting log pages ");
        my $ss_disk_list = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol devlist" );
        my $drv_result = {};

        foreach my $line ( split( /\n/, $ss_disk_list ) ) {
            my ( $drv, $ns, $nsz );
            if ( $line =~ /(\S+)\:\s+(\S+)/ ) {
                $drv = $1;
                $drv_result->{$drv}->{'PRDID'} = $2;
            }
        }
        my @nv_drv = keys(%$drv_result);
        logcomment("System Shell Drive(s) : @nv_drv");
        foreach my $drv (@nv_drv) {
            logcomment("Dumping memory log infromation for $drv");
            $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol vu_dump_ss retrieve -o memory $drv" );
            sleep 20;
            logcomment("Dump crash logs for drive  $drv");
            $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol vu_dump_ss retrieve -o crash $drv" );
            sleep 20;
            logcomment("Dump debug logs for drive $drv");
            $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol vu_dump_ss retrieve -o debug $drv" );
            sleep 20;
        }
        logcomment("Copy logs under main log directory");
        my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/;
        my $log_dir = "$actual_log_dir\/DRV_MEMORY_LOGS_\/" . $FILER_C;
        if ( !( -d "$log_dir" ) ) {
            system("sudo mkdir -p $log_dir");
        }
        my $filer_path = "/mroot/etc/log/peg_nvme*";
        logcomment("Copying logs from $filer_path");
        copy_files_to_client_from_filer(
            node_present => $Node,
            filer_path   => $filer_path,
            log_path     => $log_dir,
        );
        return logresult( "INFO", msg => "$FILER_A or $FILER_B Drive path is INCOMPLETE, scsi check Condition -  (0x2 - 0x4 0x0 0x82) FOUND !!!" );
    }
    sleep 30;
}

sub exe_disk_io_lat {
    my ( $conn, $drvs, $tot_run_time ) = @_;
    logcomment("Interval : $tot_run_time");
    logcomment("IO latency will execute for $tot_run_time seconds");
    my @drvs = @$drvs;
    logcomment("Executing disk io latency on @drvs");
    $conn->set_timeout( "connectrec-timeout" => 72000000 );
    $conn->execute_command( 'command' => "disk_latency -p rand_seq_write -t $tot_run_time -q 32 -l 50000 -s 266240 -f @drvs" );
    logcomment("Completed disk io latency");
}

sub check_drive_path {
    my $Node    = @_[0];
    my $FILER_A = $Nodes[0]->name();
    my $FILER_B = $Nodes[1]->name();
    logcomment("Node is: $Node");
    my $FILER_C = $Node->name();

    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" );
    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" ) if ( $out eq '' );

    foreach my $line ( split /\n/, $out ) {
        my ( $drv, $path );
        next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change|\[\S+\]/ );
        if ( ( $line =~ /$FILER_A/ ) && ( $line =~ /$FILER_B/ ) ) {
            my ( $drv, $path ) = $line =~ /(\S+)\s+(\S+)/;
            logcomment("Drive $drv has path $path");
        } else {
            my ( $drv, $path ) = $line =~ /\S+\:(\S+)\s+(\S+)/;
            logcomment("Drive $drv path is incomplete");
            logcomment("**FATAL** : $FILER_C Drive $drv path is INCOMPLETE");
            push( @failed_drv, $drv );
            return $status = 1;
        }
    }
    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
    sleep 30;
}

sub exe_multiple_reset {
    my ( $API_A, $drvs, $interval, $total_run_time ) = @_;
    logcomment("Test over all time is $total_run_time");
    logcomment("Interval : $interval");
    logcomment("Issuing freset on all drives");
    $API_A->set_timeout( "connectrec-timeout" => 72000000 );
    my @drvs = @$drvs;
    my $shelf;
    my $tport;
    my $drv_count = @drvs;

    my $kill_time     = $total_run_time;
    my $time_out_temp = 0;
    $API_A->set_timeout( "connectrec-timeout" => ( $kill_time + 60000000 ) );
    my $reset_count = 1;
    while ( $kill_time > $time_out_temp ) {
        my ( $s,  $m,  $h )  = gmtime($kill_time);
        my ( $s1, $m1, $h1 ) = gmtime($time_out_temp);
        logcomment("DEBUG:Total Wait time: $h HRS $m MINS $s SECS and  Elapsed Time: $h1 HRS $m1 MINS $s1 SECS");
        my $output = $API_A->execute_raw_command( 'command' => "\013" );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        logcomment("Issuing freset on @drvs");
        logcomment("Issuing reset on $drv_count drives");

        foreach my $drv (@drvs) {
            if ( $drv =~ /\S+\.(\S+)\.\S+\.(\S+)/ ) {
                $shelf = $1;
                $tport = $2 + 6;
                $API_A->execute_raw_command( 'command' => "\013" );
                my $psmadmin_out = $API_A->execute_raw_command( 'command' => "run local psmadmin inband_cli 0x.$shelf 'psmmgr_tp 4 $tport 3000'" );
                $API_A->execute_raw_command( 'command' => "\013" );
                if ( $psmadmin_out =~ /(Executed Fundamental Reset|Fundamental |Executed |xecuted Fundamental |Reset )/i ) {
                    logcomment("Executed fundamental reset command successfully on drive :$drv");
                } else {
                    logcomment("Waiting for 100 sec for output.");
                    sleep(100);
                    $API_A->execute_raw_command( 'command' => "\013" );
                    if ( $psmadmin_out =~ /(Executed Fundamental Reset|Fundamental |Executed |xecuted Fundamental |Reset )/i ) {
                        logcomment("Executed fundamental reset command successfully on drive :$drv");
                    } else {
                        logcomment("**FATAL**:Command 'psmadmin' NOT executed on drive $drv.");
                        $status = 1;
                        return logresult( "INFO", msg => $status );
                    }
                }

            }
        }

        logcomment("Reset Count : $reset_count");
        $reset_count++;
        logcomment("INTERVAL time specified by user : $interval");
        sleep($interval);
        $time_out_temp += $interval;
        logcomment("check for disk path");
        logcomment("Check storage failover status");
        my $FILER_A     = $Nodes[0]->name();
        my $FILER_B     = $Nodes[1]->name();
        my $Api_Set_Obj = $API_A;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" );
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" ) if ( $out eq '' );
        my @failed_drv;

        foreach my $line ( split /\n/, $out ) {
            next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change|\[\S+\]/ );
            if ( ( $line =~ /$FILER_A/ ) && ( $line =~ /$FILER_B/ ) ) {
                if ( $line =~ /(\S+)\s+(\S+)/ ) {
                    logcomment("Drive $1 has path $2");
                }
            } else {
                if ( $line =~ /\S+\:(\S+)\s+(\S+)/ ) {
                    logcomment("Drive $1 path is incomplete");
                    logcomment("**FATAL** : Drive $1 path is INCOMPLETE");
                    push( @failed_drv, $1 );
                    $status = 1;
                }
            }
        }
        if ( ( $status == 1 ) && (@failed_drv) ) {
            logcomment("**FATAL** : @failed_drv drives path is INCOMPLETE");
            $status = 1;
            return logresult( "INFO", msg => $status );
        } else {
            logcomment("Proceed with next freset");
        }
    }
}

# End of sub_exe
#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode,
        Tahiti          => "yes"
    );
    ###########################################################################################
    ## Delete/Reset test specific options and values
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Host        = host($FILER_C);
    }

    return $TCD::PASS;
}