#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Th_PwrCyc.thpl#1 $
##
##
## @description
## To execute psmadmin command on the drive where IO is running
## using the method available in the NetApp system.
##
##
## @steps
## The test will execute steps mentioned below:
## 1.   List all drives
## 2.   Check non-root aggregate and create one non-root aggregate with filersio
## 3.   Get drives where IO is executing
## 4.   Verify that filersio is running.
## 5.   Verify IOs to the drives
## 6.   Execute IO for specified time along with power_cyc command evry 60 seconds
## 7.   Stop filersio
##
##
## @status  Automated
## @burt    1241139
## @author  sags@netapp.com
########################################
### Library functions
##########################################
use strict;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use Storage::Tahiti_Common_Lib;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $LOGDIR
  $EOE
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $POW_SLP_TIME
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER        = $params->get( 'FILER',        default => 'Filer' );
$LOGDIR       = $params->get( 'LOGDIR',       default => undef );
$EOE          = $params->get( 'EOE',          default => 1 );
$EMAIL        = $params->get( 'EMAIL',        default => 'y' );
$MAIL_TO      = $params->get( 'MAIL_TO',      default => 'Email to' );
$MAIL_FROM    = $params->get( 'MAIL_FROM',    default => 'Email from' );
$POW_SLP_TIME = $params->get( 'POW_SLP_TIME', default => '30' );

################
# Testcase name
################
my $TC_name = "327_NADQ02_NVMe_Th_PwrCyc";

#Common variable declaration
my ( $email_subject, $email_body );
my $node_ref;
my $Api_Set_Obj;
my $FILER_C;
my $filer_names;
my $Mode;
my $test_status    = 0;
my $test_wait_time = 21600;    #6hrs
my %nodes_filer;
my @Nodes;

#########################################
# Test case available for execution
#########################################

my @Testcases = ( NVMe_Th_Pwr_Cyc => "psmadmin_resetdrv" );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description("Initialising all required variables and filer connections");
    $filer_names = $Test->get_param("FILER");
    my @temp_nodes = NACL::C::Node->find();
    foreach my $Node (@temp_nodes) {
        $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node object used for test execution.
    grep { logcomment( "Filers passed to the ntest are " . $_->name() ) } @Nodes;
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 6 hours : " . scalar( localtime() ) );
    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    $Test->nlog("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        filer_mode      => $Mode,
        change_state_to => "CLI",
        Tahiti          => "yes"

    );

    return $TCD::PASS;
}

sub NVMe_Th_Pwr_Cyc {
    my @subtests;
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    foreach my $Node (@Nodes) {
        $FILER_C = $Node->name();
        push( @subtests, subtest( \&Tahiti_exe_pwr_cyc, -runid, "Tahiti_Exe_Pwr_Cyc_$FILER_C", -bg, "--", $Node ) );
        Subtest::wait_finish( subtest => [@subtests] );
        $status = status_return( subtest_ref => [@subtests] );
        logcomment("Total test status is : $status");
    }

    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub Tahiti_exe_pwr_cyc {
    my $FILER_SIO = 20;
    my $bay;
    my $shelf;
    my $non_rt_status;
    my $module;
    my @non_rt_agg;
    my @Nodes;
    my %filer_nvol           = ();
    my @list_of_disk_2_pwrcy = ();
    push( @Nodes, shift(@_) );
    logcomment("Test will be executing on - @Nodes");

    foreach my $Node (@Nodes) {
        $FILER_C = $Node->name();
        $Node->refresh_command_interface();
        $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();

        #STEP 1
        tahiti_eye_catcher( Test => $Test, string => "STEP 1 of 7 : $FILER_C : List all drives" );
        my $disk_list = tahiti_list_only_tahiti_drv( node_present => [$Node] );

        #STEP 2
        tahiti_eye_catcher( Test => $Test, string => "STEP 2 of 7 : $FILER_C : Check non-root aggregate and create non-root aggregate(s) with filersio" );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000000 );
        logcomment("Filer: $FILER_C : Check for non-root aggregate, if already present delete and create new aggregate");
        ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        if (@non_rt_agg) {
            logcomment( "Filer: $FILER_C : Total non-root aggregate(s) found is" . scalar @non_rt_agg . " and Non-root aggregate(s) are: @non_rt_agg" );
            logcomment("Filer: $FILER_C : Deleting Aggregate(s)");
            vserver_delete( Node => $Node );    #Vserver deletion
            aggregate_volume_delete( Node => $Node );
            $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->execute_raw_command( command => "run local disk zero spares" );
            sleep 20;
            $Api_Set_Obj->execute_raw_command( command => "\013" );
        }
        sleep 5;
        my $spare_list = tahiti_sto_agg_show_spare_dsk( Node => $Node );
        my @spare_drvs = keys %$spare_list;
        my $dsks       = _print_array( \@spare_drvs );
        logcomment( "Filer : $FILER_C : Total spare count is : " . scalar @spare_drvs . " and Spare Drive are :\n" . $dsks );
        sleep 10;
        my $return_filersio_thrds = nvme_multi_aggr_with_rd_wr_vol_n_fsio(
            API_con  => $Api_Set_Obj,
            filer    => $FILER_C,
            fsio     => $FILER_SIO,
            loc_drvs => \@spare_drvs,
            Node     => $Node,
        );
        sleep 10;

        #STEP 3 :
        tahiti_eye_catcher( Test => $Test, string => "STEP 3 of 7 : $FILER_C : Get drives on which IO will be executed" );
        $Api_Set_Obj->set_timeout( 'connectrec-timeout' => 7200 );
        ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        logcomment("Filer : $FILER_C : Aggregate(s) found : @non_rt_agg");
        logcomment("Filer : $FILER_C : Get Drives of all non-root aggregates : @non_rt_agg");
        foreach my $agr (@non_rt_agg) {
            logcomment("Filer : $FILER_C : Get all drives of aggreagte: $agr");
            $Api_Set_Obj->set_timeout( 'connectrec-timeout' => 7200 );
            my $after = $Api_Set_Obj->execute_command( 'command' => "aggr status -r $agr" );
            foreach ( split( /\n/, $after ) ) {

                #Do not match the uninitialised
                next if ( $_ =~ /parity uninit/ );
                if ( $_ =~ /(data|dparity|parity)\s*(\S+)/ ) {
                    my $disk = $2;
                    $disk =~ s/(P|L)\d+$//g;
                    push( @list_of_disk_2_pwrcy, $disk );
                }
            }
        }

        #STEP 4
        tahiti_eye_catcher( Test => $Test, string => "STEP 4 of 7 : $FILER_C : Verify that filersio is running" );
        $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $out = $Api_Set_Obj->execute_raw_command( command => "run local filersio status" );
        if ( $out =~ /no instances currently running/ ) {
            logcomment("**FATAL** : $FILER_C : NO IO Instances are currently running");
            return logresult( "INFO", msg => 1 );
        } else {
            logcomment("Filersio instances are executing on $FILER_C");
        }
        sleep(10);

        #STEP 5
        tahiti_eye_catcher( Test => $Test, string => "STEP 5 of 7 : $FILER_C : Verify IOS to the drives" );
        $Api_Set_Obj->execute_raw_command( command => "run local sysstat -c 15 1" );

        #STEP 6
        tahiti_eye_catcher( Test => $Test, string => "STEP 6 of 7 : $FILER_C : Execute IO for specified time along with power_cyc command for every 60 seconds" );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        my $timeout = $test_wait_time;
        my ( $lsec, $lmin, $lhrs ) = gmtime($timeout);
        logcomment("Filer : $FILER_C : Test execution time : $lhrs Hours $lmin Minutes and $lsec Seconds");
        my $curtime = time;
        my $endtime = $curtime + $timeout;
        my $loop    = 0;
        @list_of_disk_2_pwrcy = unique(@list_of_disk_2_pwrcy);

        if ( scalar(@list_of_disk_2_pwrcy) == 0 ) {
            logcomment("**FATAL**: $FILER_C Test case will not be proceed because NON ROOT VOLUMES NOT EXIST - STEP : 3");
            return logresult( "INFO", msg => 1 );
        }
        my $dsks = _print_array( \@list_of_disk_2_pwrcy );
        logcomment( "The drives mentioned below will be powercycled , where IO are running:\n" . $dsks );

        #Executing command to get module
        my $module_out = $Api_Set_Obj->execute_raw_command( command => "storage shelf port show -fields connector-state, module-id" );
        chomp( my @sysconf_lines = split( /\r?\n/, $module_out ) );
        foreach my $line (@sysconf_lines) {
            next if ( $line =~ /^---------/ );
            if ( $line =~ /^\S*\s*\S*\s*(A)\s*connected/gm ) {
                $module = $1;
                last;
            }
            if ( $line =~ /^\S*\s*\S*\s*(B)\s*connected/gm ) {
                $module = $1;
            }
        }
        if ( not defined $module ) {
            logcomment("**FATAL** : Filer : $FILER_C : The module is NOT FOUND");
            return logresult( "INFO", msg => 1 );
        }
        while ( $curtime < $endtime ) {
            logcomment("Filer : $FILER_C : Loop $loop Started");
            logcomment("Filer : $FILER_C : Current time : $curtime and Endtime :$endtime");
            logcomment("Filer : $FILER_C : Check IO Status");
            my $out = $Api_Set_Obj->execute_raw_command( command => "run local filersio status" );
            if ( $out =~ /no instances currently running/ ) {
                logcomment("**FATAL** : $FILER_C : NO IO Instances are currently running");
                return logresult( "INFO", msg => 1 );
            } else {
                logcomment("Filersio instances are executing on $FILER_C");
            }

            foreach my $sel_drive (@list_of_disk_2_pwrcy) {
                next if ( $sel_drive =~ /P1|P2|P3/ );
                if ( $sel_drive =~ /\S+\.\S+\.\S+\.(\S+)/ ) {
                    $bay = $1;
                }
                if ( $sel_drive =~ /(\S+\.(\S+)\.\S+\.\S+)/ ) {
                    $shelf = $2;
                }
                logcomment("Filer : $FILER_C : The bay of the drive : $sel_drive is : $bay");
                logcomment("Filer : $FILER_C : The module of the drive : $sel_drive is : $module");
                logcomment("Filer : $FILER_C : The shelf of the drive :  $sel_drive is : $shelf");
                $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
                sleep 2;
                logcomment("Filer : $FILER_C : Issuing psmadmin_resetdrive command on drive $sel_drive ");

                #Here 5000------>Duration
                my $psmadmin_out = $Api_Set_Obj->execute_command( 'command' => "psmadmin inband_cli 0x.$shelf.$module 'resetdrive 5000 $bay'", 'timeout' => 600000 );
                logcomment("Filer : $FILER_C : Waiting for $POW_SLP_TIME sec");
                sleep $POW_SLP_TIME;
                logcomment("Filer : $FILER_C : psmadmin_resetdrive command completed on drive $sel_drive ");
            }
            logcomment("Filer : $FILER_C : Verify IOs to the drives");
            $Api_Set_Obj->execute_raw_command( command => "run local sysstat -c 15 1" );
            logcomment("Verify IO status");
            my $out = $Api_Set_Obj->execute_raw_command( command => "run local filersio status" );
            if ( $out =~ /no instances currently running/ ) {
                logcomment("**FATAL** : $FILER_C : NO IO Instances are currently running");
                return logresult( "INFO", msg => 1 );
            } else {
                logcomment("Filersio instances are executing on $FILER_C");
            }
            logcomment("End of loop $loop");
            $curtime = time;
            my $time_left = $endtime - $curtime;
            my ( $lsec, $lmin, $lhrs ) = gmtime($time_left);
            if ( $time_left > 0 ) {
                logcomment("Filer : $FILER_C : Remaining time for looping is $lhrs Hours $lmin Minutes and $lsec Seconds");
            }
            $loop++;
        }

        #STEP 7 :
        tahiti_eye_catcher( Test => $Test, string => "STEP 7 of 7 : $FILER_C : Stop filersio" );
        $Api_Set_Obj->execute_raw_command( command => "run local filersio stop" );
        logcomment("IO thread stopped ");
        logcomment("Filer : $FILER_C : Deleting existing aggregate(s) and volume(s)");
        foreach my $Node (@Nodes) {
            my $FILER_C = $Node->name();
            ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
            if (@non_rt_agg) {
                vserver_delete( Node => $Node );    #Vserver deletion
                aggregate_volume_delete( Node => $Node );
                $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
                $Api_Set_Obj->execute_raw_command( command => "run local disk zero spares" );
                sleep 20;
                $Api_Set_Obj->execute_raw_command( command => "\013" );
            }
        }

    }
    logresult( "INFO", msg => $test_status );
}
#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        filer_mode      => $Mode,
        change_state_to => "CLI",
        Tahiti          => "yes"

    );
    return $TCD::PASS;
}