#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Th_Filer_SIO_Test.thpl#1 $
#
## @summary Mode
##  Disk Firmware Download in background/foreground mode
#
##
## @description
## Verify that the drive can download disk firmware in foreground and background mode properly
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground
##
## @steps
## The test will execute steps mentioned below:
##
##
## @status  Automated
## @change 1213033
########################################
### Library functions
##########################################
use strict;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use Storage::Tahiti_Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
use Data::Dumper;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
  $RUNTIME
  $USER_SEL_MOD
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
$RUNTIME        = $params->get( 'RUNTIME',        default => '1' );
$USER_SEL_MOD   = $params->get( 'USER_SEL_MOD',   default => 'prd' );
################
# Testcase name
################
my $TC_name;
$TC_name = "NADQ02_NVMe_Th_Filer_SIO_Test";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home           = $ENV{HOME};
my $FILER_DRV_DETS = {};

my $test_wait_time = 12 * 3600;    #12 Hrs test
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( Tahiti_Filer_IO => "Tahiti Filer sio test" );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################

    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode,
        Tahiti          => "yes",
    );

    return $TCD::PASS;
}

sub Tahiti_Filer_IO {
    my @subtests;
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&filerio_cmd_exe, -runid, "th_filerio_cmd_exe_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $status = status_return( subtest_ref => [@subtests] );
    logcomment("Total test status is : $status");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub filerio_cmd_exe {
    $Test->description("Executing Command Compliance Steps");
    my ( @Nodes, %filer_nvol );
    my $Home = $ENV{HOME};
    my @NON_RT_AGG;
    my @NON_RT_VOLS;
    my $Api_Set_Obj_SS;
    my $drv_result = {};
    my @nv_drv;
    my $Node             = shift(@_);
    my $FILER_C          = $Node->name();
    my $Host             = host($FILER_C);
    my $create_agg_n_vol = 0;
    logcomment("Total Nodes  - $Node");
    logcomment( "Filer passed to the subtest :: " . $Node->name() );

    #STEP 1
    tahiti_eye_catcher( Test => $Test, string => "STEP 1 of 13: Set Privilage level to diag" );
    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
    logcomment(" Privilage Level ");
    $Api_Set_Obj->execute_raw_command( 'command' => "set" );
    logcomment("Set privilage level to diag");
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command( 'command' => "set diag", 'connectrec-match_table' => $prompts_answers );
    logcomment("Privilage level diag set");
    $Api_Set_Obj->execute_raw_command( 'command' => "set" );
    $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "system timeout modify 0" );

    #STEP 2
    tahiti_eye_catcher( Test => $Test, string => "STEP 2 of 13 : Get filer State" );
    logcomment("Get filer state ");
    my $Transit_obj = NACL::Transit->new( name => $FILER_C );
    my $FILER_O_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
    logcomment("FILER- $FILER_C - Is in $FILER_O_STATE state");
    if ( $FILER_O_STATE =~ /CLI|UP/ ) {
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        $FILER_DRV_DETS->{$FILER_C}->{'FILER_NAME'} = $FILER_C;
        $FILER_DRV_DETS->{$FILER_C}->{'FILER_API'}  = $Api_Set_Obj;
    } else {
        logcomment("Filer has to be in CLI/FileSystem mode, Boot filer in file system mode and re-start test");
        $test_status = 1;
        logresult( "INFO", msg => $test_status );
        logresult( "FAIL", "FAILED TO INSTANTIATE TEST OBJECT." );
        return $TCD::FAIL;
    }

    #STEP 3
    tahiti_eye_catcher( Test => $Test, string => "STEP 3 of 13 : Get All Drives" );
    logcomment("Get all drives");
    my @normal_drv;
    my @tahiti_drvs;
    my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk_list" );
    foreach my $line ( split( /\n/, $out ) ) {
        if ( $line =~ /(0n\.\d+)/ ) {
            push( @normal_drv, $1 );
        } elsif ( $line =~ /(\S+\.\S+\.\S+\.\S+)/ ) {
            push( @tahiti_drvs, $1 );
        }
    }
    logcomment("Apollo Drives : @normal_drv");
    logcomment("Tahiti Attached Drives : @tahiti_drvs");
    $FILER_DRV_DETS->{$FILER_C}->{'NOR_DRVS'} = [@normal_drv];
    $FILER_DRV_DETS->{$FILER_C}->{'THT_DRVS'} = [@tahiti_drvs];

    #STEP 4
    tahiti_eye_catcher( Test => $Test, string => "STEP 4 of 13 : Check non-root aggregate and create one non-root aggregate" );

    # Check non root volumes
    logcomment("Check for non-root aggregate ");
    my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
    if (@non_rt_agg) {
        logcomment("Total non-root aggregate is $non_rt_status and Non -root aggregate are @non_rt_agg");
        logcomment("Check for tahiti drives in aggregate");
        foreach my $aggr (@non_rt_agg) {
            logcomment("Check for Tahit connected drives in aggregate - $aggr");
            my $out = $Api_Set_Obj->execute_raw_command( 'command' => "run local aggr status -r" );
            if ( $out =~ /\S+\.\S+\.\S+\.\S+/ ) {    #e4a.00.0.8P2
                logcomment("Tahiti shelf attached drive found in aggregate $aggr");
            } else {
                logcomment("Tahiti attached shelf with ddrives not found, Delete Aggregate");
                $create_agg_n_vol = 1;
                vserver_delete( Node => $Node );     #Vserver deletion
                aggregate_volume_delete( Node => $Node );
            }
        }
    } else {
        logcomment("Non-root aggregate cannot be found");
        $create_agg_n_vol = 1;
    }

    #STEP 5
    tahiti_eye_catcher( Test => $Test, string => "STEP 5 of 13 : Create Aggregate using drives attached to tahiti shelf" );
    logcomment("Get all spare drivespresent in Tahiti shelf");
    my @tahiti_sparedrvs;
    my $shelf;
    my $dr_out = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk_list" );
    my @user_sele_drv;
    my @drv_list_tahi;

    foreach my $line ( split( /\n/, $dr_out ) ) {
        if ( $line =~ /(\S+\.(\S+)\.\S+\.\S+)/ ) {
            $shelf = $2;
            push( @drv_list_tahi, $1 );
        }
    }
    $shelf = int($shelf);
    logcomment("Drives list : @drv_list_tahi");
    logcomment("Tahiti shelf ID : $shelf");
    $shelf =~ s/0//g if ( $shelf =~ /^0/ );
    logcomment("DEBUG : Tahiti shelf ID : $shelf");
    my $sp_out = $Api_Set_Obj->execute_raw_command( command => "storage aggregate show-spare-disks -original-owner $FILER_C" );
    foreach my $line ( split( /\n/, $sp_out ) ) {

        if ( $line =~ /(\S+\.$shelf.\S+)/ ) {
            push( @tahiti_sparedrvs, $1 );
        }
    }

    #burt:1213033

=head
    foreach my $line ( split( /\n/, $dr_out ) ) {
        if ( $line =~ /$USER_SEL_MOD/ ) {
            if ( $line =~ /(\S+\.\S+\.\S+\.(\S+))/ ) {
                my $drv = $1;
                my $id  = $2;
                logcomment("User selected Model Drive : $drv");
                foreach my $spt (@tahiti_sparedrvs) {
                    if ( $spt =~ /\S+\.\S+\.(\S+)/ ) {
                        if ( $1 =~ /^$id$/ ) {
                            push( @user_sele_drv, $spt );
                        }
                    }
                }
            }
        }
    }
=cut

    logcomment("Spare drives present in tahiti shelf are : @tahiti_sparedrvs");
    $FILER_DRV_DETS->{$FILER_C}->{'T_SPARE_DRVS'}    = [@tahiti_sparedrvs];
    $FILER_DRV_DETS->{$FILER_C}->{'T_USR_SEL_SPARE'} = [@user_sele_drv];

    #STEP 6
    tahiti_eye_catcher( Test => $Test, string => "STEP 6 of 13 : Create Multiple aggregate and start filersio on read and write volume" );
    my $FILER_SIO = 20;

    my $spare_drvs = $FILER_DRV_DETS->{$FILER_C}->{'T_SPARE_DRVS'};
    my @spare_drvs = @$spare_drvs;
    logcomment("CHECK 2 : Spares Selected - @spare_drvs");
    logcomment("$FILER_C :FILERSIO THREADS : $FILER_SIO ");
    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
    my $aggregate_name;
    my $return_filersio_thrds = nvme_multi_aggr_with_rd_wr_vol_n_fsio(
        API_con  => $Api_Set_Obj,
        handle   => "custom",
        filer    => $FILER_C,
        fsio     => $FILER_SIO,
        loc_drvs => \@spare_drvs,
        Node     => $Node,
    );
    my @loc_all_threads;
    my @non_roo;

    foreach my $fil ( keys %$return_filersio_thrds ) {
        my $sio_th = $return_filersio_thrds->{$fil}->{'SIO_CMD'};
        @loc_all_threads = @$sio_th;
        if ( $fil =~ /$FILER_C/ ) {
            $aggregate_name = $return_filersio_thrds->{$fil}->{'NON_ROOT_AGGR'};
            @non_roo        = @$aggregate_name;
            logcomment("$FILER_C : Non Root Aggregate is : @non_roo");
        }
    }
    logcomment("Filer sio Started ");
    logcomment("$FILER_C : Basic Command Compliance command Completed ");
    logcomment("Total runtime user selected is $RUNTIME hrs");
    logcomment("Every one hour , statit and disk statistics will be printed");
    my $starttime = time;
    $RUNTIME = $RUNTIME * 3600;
    my $endtime = $starttime + $RUNTIME;
    logcomment("DEBUG : Start : $starttime and End :$endtime");

    while ( $starttime < $endtime ) {
        ###########################################################
        ## STEP 7 - Start the statistics for the disks selected in step#3
        ## Pass/Fail criteria: N/A
        ###########################################################
        tahiti_eye_catcher(
            Test   => $Test,
            string => "STEP 7 of 13 : $FILER_C : Start statisics command for the disks that have been selected",
        );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 360000000 );
        my $sample_id;
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        my $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "statistics start -object disk",
            'connectrec-match_table' => $prompts_answers
        );
        if ( $output =~ /Statistics collection is being started for sample-id:\s+(\S+)/ ) {
            logcomment("$FILER_C : Statistics command for the selected disks as been started");
            $sample_id = $1;
            logcomment("DEBUG: Captured Sample -id $sample_id to Stop the Statistics");
        } else {
            logcomment("**FAIL** : $FILER_C - Statistics command for the selected disks COULD NOT be started");
            return logresult( "INFO", msg => 1 );
        }
        logcomment("$FILER_C : Waiting for Statistics object to start on disk");
        tahiti_eye_catcher(
            Test   => $Test,
            string => "STEP 8  of 13: $FILER_C : Collect the statit info",
        );
        $Api_Set_Obj->execute_command( 'command' => " statit -b " );

        #sleep(60);
        logcomment("Wait for 1 hr and then execute sattis -e");

        sleep(3600);
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 360000000 );
        logcomment("Execute IO for 1 hr,  execute sattis -e");
        logcomment("Execute statit -e");
        my $output = $Api_Set_Obj->execute_command( 'command' => "statit -e" );

        ###########################################################
        ## STEP 9 - Display the statistics for the disks selected in step#3
        ## Pass/Fail criteria: N/A
        ###########################################################
        tahiti_eye_catcher(
            Test   => $Test,
            string => "STEP 9 of 13 : $FILER_C : Displaying the statisics for the disks that have been selected",
        );
        $prompts_answers = [ ".*to page down, .*" => ' ' ];
        sleep 60;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );
        my $disks;
        my $disk_l = $Api_Set_Obj->execute_raw_command( 'command' => "aggr show $non_roo[0] -fields disklist" );
        my @drives_use;
        sleep 20;

        foreach my $line ( split( /\n/, $disk_l ) ) {

            if ( ( $line =~ /\,/ ) && ( $line =~ /(\S+)\s+(\S+)/ ) ) {
                my $drs = $2;
                $drs =~ s/$FILER_C\://g;
                @drives_use = split( /\,/, $drs );
            }
        }
        logcomment("DEBUG : Aggregate drives are : @drives_use");
        foreach my $dsk (@drives_use) {
            $disks = $disks . $dsk . "|";
        }
        $disks =~ s/\|$//;
        logcomment("Get Statistics for drives : $disks ");
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        my $output = $Api_Set_Obj->execute_raw_command(
            'command'                => "statistics show -object disk -instance $disks",
            'connectrec-match_table' => $prompts_answers
        );
        if ( $output =~ /show failed/ ) {
            logcomment("**WARNING** : FAILED TO SHOW THE STATISTICS FOR DISKS $disks");
        }

        ###########################################################
        ## STEP 10 - Stop the statistics for the disks selected in step#3
        ## Pass/Fail criteria: N/A
        ###########################################################

        tahiti_eye_catcher(
            Test   => $Test,
            string => "STEP 10 of 13 : $FILER_C : Stop the statisics for the disks that have been selected",
        );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        my $output = $Api_Set_Obj->execute_raw_command( 'command' => "statistics stop -sample-id $sample_id", 'connectrec-match_table' => $prompts_answers );
        if ( $output =~ /(stopped\s*for\s*sample-id.*$sample_id)|("$sample_id"\s*already\s*stopped)/ig ) {
            logcomment("Statistics collection is being stopped for sample-id: $sample_id");
        } else {
            logcomment("**FAIL** : FAILED TO STOP THE STATSTICS  FOR SAMPLE-ID $sample_id");
        }

        ###########################################################
        ## STEP 11 - Stop the statit
        ## Pass/Fail criteria: N/A
        ###########################################################

        tahiti_eye_catcher(
            Test   => $Test,
            string => "STEP 11 of 13 : $FILER_C : Stop the statit",
        );
        my $output = $Api_Set_Obj->execute_command( 'command' => "filersio status" );
        if ( $output =~ /(asyncio_active running|running)/ig ) {
            logcomment("filersio threads are running");
        }
        $starttime = time;
        $Api_Set_Obj->execute_command( 'command' => "\013" );
        logcomment("DEBUG : 1hr IO completed , Loop Again ");
        logcomment("DEBUG : Current time is : $starttime, End time is $endtime");
        my $time_left = $endtime - $starttime;
        my ( $lsec, $lmin, $lhrs ) = gmtime($time_left) if ( $time_left > 0 );
        logcomment("Remaining time is $lhrs HOURS $lmin MINUTES and $lsec SECONDS") if ( $time_left > 0 );

    }    #End of while
    ###########################################################
    ## STEP 12 - Stop the filersio instances running
    ## Pass/Fail criteria: N/A
    ###########################################################

    tahiti_eye_catcher(
        Test   => $Test,
        string => "STEP 12 of 13 : $FILER_C : Stop the filersio instances",
    );
    $Api_Set_Obj->set_timeout( "connectrec-timeout" => 3600 );
    my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio stop\"" );
    logcomment("Waiting for 5 minutes");
    sleep(300);
    my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio status\"" );

    if ( $output =~ /(filersio:\s*aborting\s*instance\s*)|/ ) {
        logcomment("Filersio threads are stopped Successfully");
    } else {
        logcomment("**WARNING** : Filersio threads are not  stopped ");
    }
    ###########################################################
    ## STEP 13 - Delete the aggregate, volume and vserver created for this test
    ## Pass/Fail criteria: N/A
    ###########################################################

    tahiti_eye_catcher(
        Test   => $Test,
        string => "STEP 13 of 13 : $FILER_C : Delete the volumes, vserver and aggregate created",
    );
    logcomment("Deleting the non root aggregate created for this test");
    aggregate_volume_delete( Node => $Node );
    vserver_delete( Node => $Node );
    return logresult( "INFO", msg => 0 );

}

# End of sub_exe
#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;

    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode,
        Tahiti          => "yes",
    );

    return $TCD::PASS;
}