#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Th_Fsio_Adp_Rst.thpl
#
## @summary Mode
##  Evaluated reset command on all tahiti drives.
#
##
## @description
## Verifying the filersio running status on Tahiti drives and evaluates reset command on all tahiti drives.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground
##
## @steps
## The test will execute steps mentioned below:
##		1. List all drives
##		2. Check non-root aggregate and create one non-root aggregate
##		3. Get drives where IO is executing
##		4. Verify that filersio is running.
##		5. Verify IOs to the drives
##		6. Execute IO for specified time along with contoller reset evry 60 seconds
##		7. Abort filersio
##
##
## @status  Automated
## @author  mvemuri
## @burt    1241141
##
########################################
### Library functions
##########################################
use strict;
use Storage::Tahiti_Common_Lib;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $TEST_CONFIG
  $TEST_SETUP
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
  $POW_SLP_TIME
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$POW_SLP_TIME   = $params->get( 'POW_SLP_TIME',   default => '30' );
################
# Testcase name
################
my $TC_name;
$TC_name = "322_NADQ02_NVMe_Th_Fsio_Ctrl_Rst";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home = $ENV{HOME};

my $test_wait_time = 21600;    #6hrs

if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}
#########################################
# Test case available for execution
#########################################

my @Testcases = ( Tahiti_Fsio_Ctrl_Rst => "Tahiti FSIO Ctrl reset command" );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode,
        Tahiti          => "yes"
    );

    return $TCD::PASS;
}

sub Tahiti_Fsio_Ctrl_Rst {
    my @subtests;
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    my $partition = 0;
    logcomment("Checking whether the system is PARTITIONED or UNPARTITIONED");
    my $FILER_C = $Nodes[0]->name();
    my $disk_hash = disk_show_v( Node => $Nodes[0] );

    foreach ( keys %{$disk_hash} ) {
        if ( $_ =~ /P\d+$/ ) {
            $partition = 1;
            last;
        }
    }

    logcomment("DEBUG : NODE PRESENT : @Nodes");
    my $status;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Test Executing on $FILER_C");
        push( @subtests, subtest( \&exe_fsio_rest_sub, -runid, "Exe_Fsio_Rst_$FILER_C", -bg, "--", $Node ) );
        Subtest::wait_finish( subtest => [@subtests] );
        $status = status_return( subtest_ref => [@subtests] );
        logcomment("Total test status is : $status");
    }
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub exe_fsio_rest_sub {
    $Test->description("Execute FSIO and Reset command ");
    my ( @Nodes, %filer_nvol );
    my $Home = $ENV{HOME};
    my @NON_RT_AGG;
    my @NON_RT_VOLS;
    my %filer_nvol           = ();
    my $create_agg_n_vol     = 0;
    my $aggr_vol_found       = 0;
    my @list_of_disk_2_pwrcy = ();

    push( @Nodes, shift(@_) );
    logcomment("Total Nodes  - @Nodes");
    logcomment( "Filer passed to the subtest :: " . $Nodes[0]->name() );

    #STEP 1
    tahiti_eye_catcher( Test => $Test, string => "STEP 1 of 7 : List all drives " );
    my $disk_list = tahiti_get_all_disk( node_present => [ $Nodes[0] ] );

    #STEP 2
    tahiti_eye_catcher( Test => $Test, string => "STEP 2 of 7 : Check non-root aggregate and create one non-root aggregate with filersio" );
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000000 );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $Host = host($FILER_C);
        logcomment("Check for non-root aggregate, If already present, Proceed with test without creating new aggregate");
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );

        if (@non_rt_agg) {
            logcomment("Total non-root aggregate is $non_rt_status and Non -root aggregate are @non_rt_agg");
            logcomment("Check for non-root volume existance in available non root aggregate");
            vserver_delete( Node => $Node );    #Vserver deletion
            aggregate_volume_delete( Node => $Node );
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( command => "run local disk zero spares" );
            sleep 20;
            $Api_Set_Obj->execute_raw_command( command => "\013" );
        }
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        logcomment("Non-root aggregate cannot be found") if ( $non_rt_status == 0 );
        my $FILER_SIO = 20;
        sleep 30;
        my $spare_list  = tahiti_sto_agg_show_spare_dsk( Node => $Node );
        my @spare_drvs  = keys %$spare_list;
        my $spare_count = @spare_drvs;
        logcomment("$FILER_C : Total spare are : $spare_count and Spare Drive are : @spare_drvs");
        logcomment("Create new vol");
        logcomment("DEBUG :FILERSIO THREADS : $FILER_SIO ");
        sleep 30;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $return_filersio_thrds = nvme_multi_aggr_with_rd_wr_vol_n_fsio(
            API_con  => $Api_Set_Obj,
            handle   => "custom",
            filer    => $FILER_C,
            fsio     => $FILER_SIO,
            loc_drvs => \@spare_drvs,
            Node     => $Node,
        );

    }

    #STEP 3 :
    tahiti_eye_catcher( Test => $Test, string => "STEP 3 of 7 : Get drives where IO is executing" );
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $API_obj = $Node->get_7m_or_nodescope_apiset();
        $API_obj->set_timeout( 'connectrec-timeout' => 7200 );
        my $FILER_C = $Node->name();
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        logcomment("Aggregate(s) found : @non_rt_agg");
        logcomment("Get Drives of all non-root aggregates : @non_rt_agg");
        foreach my $agr (@non_rt_agg) {
            logcomment("Get drives of $agr");
            $API_obj->set_timeout( 'connectrec-timeout' => 7200 );
            my $after = $API_obj->execute_command( 'command' => "aggr status -r $agr" );
            foreach ( split( /\n/, $after ) ) {

                #Do not match the uninitialised
                next if ( $_ =~ /parity uninit/ );
                if ( $_ =~ /(data|dparity|parity)\s*(\S+\.\S+\.\S+\.\S+)/i ) {
                    my $disk = $2;
                    $disk =~ s/(P|L)\d+$//g;
                    push( @list_of_disk_2_pwrcy, $disk );
                }
            }
        }
        logcomment("Drives : @list_of_disk_2_pwrcy will be powercycled , where IO are running");
    }

    #STEP 4
    tahiti_eye_catcher( Test => $Test, string => "STEP 4 of 7 : Verify that filersio is running." );
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $out         = $Api_Set_Obj->execute_raw_command( command => "run local filersio status" );
        if ( $out =~ /no instances currently running/ ) {
            logcomment("**FATAL** : $FILER_C : NO IO Instances are currently running");
        } else {
            logcomment("Filersio isnstances are executing on $FILER_C");
        }
        sleep(10);
    }

    #STEP 5
    tahiti_eye_catcher( Test => $Test, string => "STEP 5 of 7 : Verify IOs to the drives" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Verify IOs to the drives");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->execute_raw_command( command => "run local sysstat -c 15 1" );
    }

    #STEP 6
    tahiti_eye_catcher( Test => $Test, string => "STEP 6 of 7 : Execute IO for specified time along with contoller reset every 60 seconds" );
    foreach my $Node (@Nodes) {
        my $FILER_C         = $Node->name();
        my $Api_Set_Obj     = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" );
        logcomment("IO on these drives : @list_of_disk_2_pwrcy");
        my $timeout = $test_wait_time;
        my ( $lsec, $lmin, $lhrs ) = gmtime($timeout);
        logcomment("Test executin time : $lhrs HOURS $lmin MINUTES and $lsec SECONDS");
        my $curtime = time;
        my $endtime = $curtime + $timeout;
        my $loop    = 0;
        my $shelf;
        my $id;
        my $status;

        while ( $curtime < $endtime ) {
            logcomment("Loop $loop Started");
            logcomment("Current time : $curtime and Endtime :$endtime");
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            logcomment("Check IO Status");
            my $out = $Api_Set_Obj->execute_raw_command( command => "run local filersio status" );
            if ( $out =~ /no instances currently running/ ) {
                logcomment("**FATAL** : $FILER_C : NO IO Instances are currently running");
            } else {
                logcomment("Filersio isnstances are executing on $FILER_C");
            }
            foreach my $drv (@list_of_disk_2_pwrcy) {
                if ( $drv =~ /\S+\.(\S+)\.\S+\.(\S+)/ ) {
                    logcomment("Issuing controller reset to drive : $drv ");
                    $shelf = $1;
                    if   ( $2 >= 0 && $2 <= 9 ) { $id = "0" . $2; }
                    else                        { $id = $2; }
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    my $psmadmin_out = $Api_Set_Obj->execute_raw_command( 'command' => "run local psmadmin inband_cli 0x.$shelf 'bridge nvme reset controller nvme$id'" );
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    if ( $psmadmin_out =~ /(NVMe Reset Controller Successful|Reset Controller Successful|Successful)/i ) {
                        logcomment("Executed reset command successfully on drive :$drv");
                    } elsif ( ( $psmadmin_out =~ /(Invalid name for NVMe Controller|Invalid name|Invalid)/i ) ) {
                        logcomment("**FATAL**:Command 'psmadmin' NOT executed on drive $drv.");
                        $status = 1;
                        return logresult( "INFO", msg => $status );
                    } else {
                        logcomment("Waiting for 100 sec for output.");
                        sleep(100);
                        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                        if ( $psmadmin_out =~ /(NVMe Reset Controller Successful|Reset Controller Successful|Successful)/i ) {
                            logcomment("Executed reset command successfully on drive :$drv");
                        } else {
                            logcomment("**FATAL**:Command 'psmadmin' NOT executed on drive $drv.");
                            $status = 1;
                            return logresult( "INFO", msg => $status );
                        }
                    }

                }
            }
            logcomment("Verify IOs to the drives");
            $Api_Set_Obj->execute_raw_command( command => "run local sysstat -c 15 1" );
            logcomment("Verify IO status");
            my $out = $Api_Set_Obj->execute_raw_command( command => "run local filersio status" );
            if ( $out =~ /no instances currently running/ ) {
                logcomment("**FATAL** : $FILER_C : NO IO Instances are currently running");
            } else {
                logcomment("Filersio isnstances are executing on $FILER_C");
            }
            logcomment("End of loop $loop");
            $curtime = time;
            my $time_left = $endtime - $curtime;
            my ( $lsec, $lmin, $lhrs ) = gmtime($time_left);
            if ( $time_left > 0 ) {
                logcomment("Remaining time for looping is $lhrs HOURS $lmin MINUTES and $lsec SECONDS");
            }
            $loop++;
        }
    }

    #STEP 7 :
    tahiti_eye_catcher( Test => $Test, string => "STEP 7 of 7 : abort filersio" );
    logcomment("Aborting IO ");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->execute_raw_command( command => "run local filersio stop" );
        logcomment("IO thread stopped ");
    }
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        if (@non_rt_agg) {
            logcomment("Total non-root aggregate is $non_rt_status and Non -root aggregate are @non_rt_agg");
            logcomment("Check for non-root volume existance in available non root aggregate");
            vserver_delete( Node => $Node );    #Vserver deletion
            aggregate_volume_delete( Node => $Node );
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( command => "run local disk zero spares" );
            sleep 20;
            $Api_Set_Obj->execute_raw_command( command => "\013" );
        }
    }

    logresult( "INFO", msg => $test_status );
}

#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode,
        Tahiti          => "yes"
    );

    return $TCD::PASS;
}