#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/CMODE/NADQ02_Disk_Test_FS_Mode#1 $
# Copyright (c) 2005 Network Appliance, Inc.
# All rights reserved
#
## @summary MODE
##
##	  Compatible - CMODE
##
## @description
##       This test will be conducted on 50% of the spare drives in every attached shelf.
##       This test is HW configuration independent and can be run in any configuration
##
## @Test MODE
##  File System mode
##
## @Test bed setup
##  SAS HDD : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  none
##
## @steps
## The test will execute steps mentioned below in File System mode:
##   1.  Set Privilege level to test.
##   2.  Record Verify Results.
##   3.  Record Sequential Read Results.
##   4.  Record Sequential Write Results.
##   5.  Record Sequential Write/Read Results.
##
## @status  Automated
## @auther  arunak
## @burt    1151596
## @change  YY/MM/DD from auther: Description of the change
##
#############################################################################
#########################################
### Library functions
##########################################
# Compiler directives.
use strict;

##### CMode Modules #####
use TCD;
use NACL::APISet;
use NACL::C::Node;
use HTML::Mail;
use San;
use Storage::Common_Lib;
use Storage::NVMe_Common_Lib;
use NATE::BaseException qw(:try);
#####
# Global parameters
use vars qw(
  $FILER
  $TEST_SETUP
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
);
param( 'LOGDIR',         '-default', '' );
param( 'EOE',            '-default', 'default' );
param( 'EMAIL',          '-default', 'y' );
param( 'MAIL_TO',        '-mesg',    'Email to' );
param( 'MAIL_FROM',      '-mesg',    'Email from' );
param( 'FILER',          '-mesg',    'Filer' );
param( 'TEST_SETUP',     '-default', 'FC' );
param( 'TEST_WAIT_TIME', '-default', '3' );
param( 'BOOT_TYPE',      '-default', 'A' );
param( 'SSD',            '-default', 'no' );

my ( $email_subject, $email_body );
my $TC_name;
$TC_name = "409_NADQ02_NVMe_Disk_Test_FS_Mode";

#########################################
### Initialization/declaration
#########################################
######################## Global variables###############################
my @Nodes;
my $filer_names;
my $Mode;

# End log parameters
my $log_comments = "$LOGDIR/$TC_name" . "\.log";
my $log_console  = "$LOGDIR/$TC_name" . ".END_FILER_CONFIG" . ".log";

#Set test fail parameter to 0
my $test_status = 0;
my $Home        = $ENV{HOME};
#########################################
# Testcase available for execution
#########################################
## Pre-test processes
#########################################

my @Testcases = ( Disk_Test_FS_Mode => "Disk test File System Mode - Verify Results ," . " Verify Sequential Read Results" );

&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status   = 1;
        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main

########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    my %nodes_filer;
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node objects used for test execution.
    grep { logcomment( "Filers used in test " . $_->name() ) } @Nodes;
    $Mode = $Nodes[0]->mode();
    version_test( node_present => \@Nodes, tc_name => $TC_name );
    logcomment( "Filer- $filer_names : $TC_name : started, expected max completion time 1 hr : " . scalar( localtime() ) );
    logcomment( "Filer- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}

sub setup() {
    $Test->description("Preforming Pre test process - Check filer state, Check disk, check root volume,halt hammer, halt scrub, filer config");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;

    #####################################################################
    # Pre test proces : call for execute_pre_test process
    #####################################################################
    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );

    return $TCD::PASS;
}

########## TEST 1 ###################################################
#NADQ02_SAS_Drive_Head_Health_Test
#####################################################################
sub Disk_Test_FS_Mode {
    my @subtests;
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&Disk_Test_FS_Mode_sub, -runid, "Disk_Test_FS_Mode_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $return = status_return( subtest_ref => [@subtests] );
    if ( $return == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub Disk_Test_FS_Mode_sub {

    my @Nodes;
    push( @Nodes, shift(@_) );

###################
    #Test Variables
###################

    $Test->description(" Disk_Test_FS_Mode ");
    logcomment("Starting execution of test steps. Total steps to be executed as part of the script is: 3");

    #------------------------------------------------------------
    ## Step 1 - Set Privilege level to test
    ## Pass: N/A
    ## Fail criteria: N/A
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 1 of 5 : $FILER_C - Set privilege to test on the filer" );
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->execute_command( 'command' => "priv set -q test" );
    }

    #------------------------------------------------------------
    ## Step 2 - Record Verify Results
    ## Pass: Test runs without error, no check conditions reported by any drive
    ## Fail criteria  : If SAS confidence factor is less than 1.00
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 2 of 5 : $FILER_C - Record Verify Results" );
        my $start_str       = "STEP 2 of 5";
        my $end_str         = "Capture the SAS Confidence Factor value of disktest -V";
        my $prompts_answers = [ 'Do you wish to continue' => 'y' ];
        my $Api_Set_Obj     = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        try {
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 600 );

            $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
            $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );
            $Api_Set_Obj->execute_raw_command( 'command' => "disktest -V", 'connectrec-match_table' => $prompts_answers );
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 60 );
        }
        catch NATE::BaseException with {
            logcomment("Issue control+C at Step 2 for disktest -V");
            $Api_Set_Obj->execute_raw_command( 'command' => "\003" );
            $Api_Set_Obj->execute_raw_command( 'command' => "" );
            sleep 20;
        };
        logcomment("Capture the SAS Confidence Factor value of disktest -V");
        my @disktest = disktest_log( $Node, $start_str, $end_str );

        my @disk_selected = ();
        foreach my $line (@disktest) {
            if ( $line =~ s/Disks selected: //g ) {
                foreach my $disk ( split( / /, $line ) ) {
                    push( @disk_selected, $disk );
                }
            }
        }
        if (@disk_selected) {
            logcomment("Selected disk are : @disk_selected");
        }

        my @Cumulative_data = ();
        foreach my $line (@disktest) {
            $line =~ s/\\r\\r\\n/ /g;
            if ( ( $line =~ /\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\d+\s+(\d+\.\d+)/ ) && ( $line !~ /\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\d+\s+\d+\s+\S+/ ) ) {
                foreach my $data ( split( /\n/, $line ) ) {
                    push( @Cumulative_data, "$data\n" );

                }
            }
        }
        my $low_conf = 0;
        my @conf_fac = ();
        my @conf_val = ();
        foreach my $SCF (@Cumulative_data) {
            $SCF =~ s/^\s+|\s+$//g;
            my @line_arr = split( /\s+/, $SCF );
            if (   ( $line_arr[-1] !~ /^$/ )
                && ( $line_arr[-1] =~ /(\d+)\.(\d+)/ )
                && ( $line_arr[0] !~ /$FILER_C/ ) )
            {
                logcomment("Adapter ID : $line_arr[0] and SAS Confidence Factor : $line_arr[-1]");
                if ( $line_arr[-1] < 1.00 ) {
                    push( @conf_fac, $line_arr[0] );
                    push( @conf_val, $line_arr[-1] );
                    $low_conf = 1;
                }
            }
        }
        if ( $low_conf == 1 ) {
            $test_status = 1;
            logcomment("**FAIL** : Cumulative RCA errors on : @conf_fac for SAS confidence factor on DISKTEST -V @conf_val which is less than 1.00.");
            logresult( "INFO", msg => $test_status );    #Burt1025676 nmamatha
        }
        ##exit to come out  form d-blade to n-blade
        $Api_Set_Obj->execute_raw_command( 'command' => "exit" );
    }
    sleep 60;

    #-----------------------------------------------------------
    ## Step 3 - Record Sequential Read Results
    ## Pass: Test runs without error, no check conditions reported by any drive
    ## Fail criteria  : If SAS confidence factor is less than 1.00
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 3 of 5 : $FILER_C - Record Sequential Read Results" );
        my $start_str       = "STEP 3 of 5";
        my $end_str         = "Capture the SAS Confidence Factor value of disktest -R";
        my $prompts_answers = [ 'Do you wish to continue' => 'y' ];
        my $Api_Set_Obj     = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        try {
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 600 );
            sleep 60;
            $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
            $Api_Set_Obj->execute_raw_command( 'command' => " " );
            $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );
            sleep 10;
            $Api_Set_Obj->execute_raw_command( 'command' => " " );
            $Api_Set_Obj->execute_raw_command( 'command' => "disktest -R", 'connectrec-match_table' => $prompts_answers );
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 600 );
        }
        catch NATE::BaseException with {
            logcomment("Issue control+C at Step 3 for disktest -R");
            $Api_Set_Obj->execute_raw_command( 'command' => "\003" );
        };
        logcomment("Capture the SAS Confidence Factor value of disktest -R");
        my @disktest = disktest_log( $Node, $start_str, $end_str );

        my @disk_selected = ();
        foreach my $line (@disktest) {
            if ( $line =~ s/Disks selected: //g ) {
                foreach my $disk ( split( / /, $line ) ) {
                    push( @disk_selected, $disk );
                }
            }
        }
        if (@disk_selected) {
            logcomment("Selected disk are : @disk_selected");
        }

        my @Cumulative_data = ();
        foreach my $line (@disktest) {
            $line =~ s/\\r\\r\\n/ /g;
            if ( ( $line =~ /\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\d+\s+(\d+\.\d+)/ ) && ( $line !~ /\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\d+\s+\d+\s+\S+/ ) ) {
                foreach my $data ( split( /\n/, $line ) ) {
                    push( @Cumulative_data, "$data\n" );
                }
            }
        }

        my $low_conf = 0;
        my @conf_fac = ();
        foreach my $SCF (@Cumulative_data) {
            $SCF =~ s/^\s+|\s+$//g;
            my @line_arr = split( /\s+/, $SCF );
            if (   ( $line_arr[-1] !~ /^$/ )
                && ( $line_arr[-1] =~ /(\d+)\.(\d+)/ )
                && ( $line_arr[0] !~ /$FILER_C/ ) )
            {
                logcomment("Adapter ID : $line_arr[0] and SAS Confidence Factor : $line_arr[-1]");
                if ( $line_arr[-1] < 1.00 ) {
                    push( @conf_fac, $line_arr[0] );
                    $low_conf = 1;
                }
            }
        }

        if ( $low_conf == 1 ) {
            $test_status = 1;
            logcomment("**FAIL** : Cumulative RCA errors on : @conf_fac for SAS confidence factor on disktest -r are less than 1.00.");
            logresult( "INFO", msg => $test_status );    #Burt1025676 nmamatha
        }
        ##exit to come out  form d-blade to n-blade
        $Api_Set_Obj->execute_raw_command( 'command' => "exit" );
    }
    sleep 60;

    #------------------------------------------------------------
    ## Step 4 - Record Sequential Write Results
    ## Pass: Test runs without error, no check conditions reported by any drive
    ## Fail criteria  : If SAS confidence factor is less than 1.00
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 4 of 5 : $FILER_C - Record Sequential Write Results" );
        my $start_str       = "STEP 4 of 5";
        my $end_str         = "Capture the SAS Confidence Factor value of disktest -W";
        my $prompts_answers = [ 'Do you wish to continue' => 'y' ];
        my $Api_Set_Obj     = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        try {
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 600 );

            $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
            $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );
            sleep 10;
            $Api_Set_Obj->execute_raw_command( 'command' => "disktest -W", 'connectrec-match_table' => $prompts_answers );
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 600 );
        }
        catch NATE::BaseException with {
            logcomment("Issue control+C at STEP 4 for disktest -W");
            $Api_Set_Obj->execute_raw_command( 'command' => "\003" );
        };
        logcomment("Capture the SAS Confidence Factor value of disktest -W");
        my @disktest = disktest_log( $Node, $start_str, $end_str );

        my @disk_selected = ();
        foreach my $line (@disktest) {
            if ( $line =~ s/Disks selected: //g ) {
                foreach my $disk ( split( / /, $line ) ) {
                    push( @disk_selected, $disk );
                }
            }
        }
        if (@disk_selected) {
            logcomment("Selected disk are : @disk_selected");
        }

        my @Cumulative_data = ();
        foreach my $line (@disktest) {
            $line =~ s/\\r\\r\\n/ /g;
            if ( ( $line =~ /\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\d+\s+(\d+\.\d+)/ ) && ( $line !~ /\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\d+\s+\d+\s+\S+/ ) ) {
                foreach my $data ( split( /\n/, $line ) ) {
                    push( @Cumulative_data, "$data\n" );
                }
            }
        }

        my $low_conf = 0;
        my @conf_fac = ();
        foreach my $SCF (@Cumulative_data) {
            $SCF =~ s/^\s+|\s+$//g;
            my @line_arr = split( /\s+/, $SCF );
            if (   ( $line_arr[-1] !~ /^$/ )
                && ( $line_arr[-1] =~ /(\d+)\.(\d+)/ )
                && ( $line_arr[0] !~ /$FILER_C/ ) )
            {
                logcomment("Adapter ID : $line_arr[0] and SAS Confidence Factor : $line_arr[-1]");
                if ( $line_arr[-1] < 1.00 ) {
                    push( @conf_fac, $line_arr[0] );
                    $low_conf = 1;
                }
            }
        }

        if ( $low_conf == 1 ) {
            $test_status = 1;
            logcomment("**FAIL** :Cumulative RCA errors on : @conf_fac for SAS confidence factor on disktest -W are less than 1.00.");
            logresult( "INFO", msg => $test_status );    #Burt1025676 nmamatha
        }
        ##exit to come out  form d-blade to n-blade
        $Api_Set_Obj->execute_raw_command( 'command' => "exit" );
    }
    sleep 60;

    #------------------------------------------------------------
    ## Step 5 - Record Sequential Write/Read Results
    ## Pass: Test runs without error, no check conditions reported by any drive
    ## Fail criteria  : If SAS confidence factor is less than 1.00
    #------------------------------------------------------------
    foreach my $Node (@Nodes) {
        $Node->refresh_command_interface();
        my $FILER_C = $Node->name();
        eye_catcher( Test => $Test, string => "STEP 5 of 5 : $FILER_C - Record Sequential Write/Read Results" );
        my $start_str       = "STEP 5 of 5";
        my $end_str         = "Capture the SAS Confidence Factor value of disktest -B";
        my $prompts_answers = [ 'Do you wish to continue' => 'y' ];
        my @disk_list       = nvme_get_disk_owned( Node => $Node );

        #my $dsk_result      = diskshow_filer( Node => $Node );                            #Burt1025676 nmamatha
        #my @disk_list       = keys %{$dsk_result};
        my @container_disk;
        grep {
            if ( $_ !~ /p\d+/i ) { push( @container_disk, $_ ) }
        } @disk_list;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        try {
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 600 );

            $Api_Set_Obj->execute_raw_command( 'command' => "run local" );
            $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test" );
            $Api_Set_Obj->execute_raw_command( 'command' => "disktest -d @container_disk", 'connectrec-match_table' => $prompts_answers );    #Burt1025676 nmamatha
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 600 );
        }
        catch NATE::BaseException with {
            logcomment("Issue control+C at STEP 5 for disktest -B");
            $Api_Set_Obj->execute_raw_command( 'command' => "\003" );
        };

        logcomment("Capture the SAS Confidence Factor value of disktest -B");
        my @disktest = disktest_log( $Node, $start_str, $end_str );

        my @disk_selected = ();
        foreach my $line (@disktest) {
            if ( $line =~ s/Disks selected: //g ) {
                foreach my $disk ( split( / /, $line ) ) {
                    push( @disk_selected, $disk );
                }
            }
        }
        if (@disk_selected) {
            logcomment("Selected disk are : @disk_selected");
        }

        my @Cumulative_data = ();
        foreach my $line (@disktest) {
            $line =~ s/\\r\\r\\n/ /g;
            if ( ( $line =~ /\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\d+\s+(\d+\.\d+)/ ) && ( $line !~ /\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\d+\s+\d+\s+\S+/ ) ) {
                foreach my $data ( split( /\n/, $line ) ) {
                    push( @Cumulative_data, "$data\n" );

                }
            }
        }

        my $low_conf = 0;
        my @conf_fac = ();
        foreach my $SCF (@Cumulative_data) {
            $SCF =~ s/^\s+|\s+$//g;
            my @line_arr = split( /\s+/, $SCF );
            if (   ( $line_arr[-1] !~ /^$/ )
                && ( $line_arr[-1] =~ /(\d+)\.(\d+)/ )
                && ( $line_arr[0] !~ /$FILER_C/ ) )
            {
                logcomment("Adapter ID : $line_arr[0] and SAS Confidence Factor : $line_arr[-1]");
                if ( $line_arr[-1] < 1.00 ) {
                    push( @conf_fac, $line_arr[0] );
                    $low_conf = 1;
                }
            }
        }

        if ( $low_conf == 1 ) {
            $test_status = 1;
            logcomment("**FAIL** : Cumulative RCA errors on : @conf_fac for SAS confidence factor on disktest -B are less than 1.00.");
            logresult( "INFO", msg => $test_status );    #Burt1025676 nmamatha
        }

        ##exit to come out  form d-blade to n-blade
        $Api_Set_Obj->execute_raw_command( 'command' => "exit" );
    }

###########################################################################################
## Test log/results
###########################################################################################

    logresult( "INFO", msg => $test_status );
}

#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    logcomment(" $filer_names - Clean up and post test process");
    logcomment("Nodes present - @Nodes");
    my $node_ref = \@Nodes;

    ###########################################################################################
    ## Post Test process - Category : "execute_post_test"
    ###########################################################################################
    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );
    return $TCD::PASS;
}

sub disktest_log {
    my ( $Node, $start_str, $end_str ) = @_;
    my $start_fnd = 0;
    my $end_fnd   = 0;
    my @Step_log;
    my $FILER_C = $Node->name();
    my $file    = "Disk_Test_FS_Mode_$FILER_C.log";
    open( STEP_LOG, "<$LOGDIR/$file" ) || die "Can not open file $LOGDIR/$file";
    my @total_data = <STEP_LOG>;
    close STEP_LOG;

    foreach my $line (@total_data) {
        my $flag;
        while (1) {
            $flag = chomp($line);
            last if ( $flag == 0 );
        }
        if ( $line =~ /$start_str/ ) {
            $start_fnd = 1;
        }
        if ( $line =~ /$end_str/ ) {
            $end_fnd   = 1;
            $start_fnd = 0;
        }
        if ( $start_fnd == 1 ) {
            if ( $line ne '' ) {
                $line =~ s/\[HDD\S+\]//g;
                $line =~ s/X;X/ /g;
                $line =~ s/\d+\s+\d+\s+\S+\|//g;
                if ( $line !~ /Warning: Usage of execute_raw_command is not recommended/ ) {
                    push( @Step_log, $line, "\n" );
                }
            }
        } elsif ( $end_fnd == 1 ) {
            last;
        }
    }
    return @Step_log;
}