#!/usr/software/bin/perl
# $Id: //depot/prod/test/Rsierranevada/storage/hdd/NADQ_SEA/NDATE/SUPPORT_SCRIPTS/send_result_to_console.thpl#2 $
# Copyright (c) 2005 Network Appliance, Inc.
# All rights reserved
#
## @summary Mode
##  Filer test result on the console

## @description
##  Send the test result to the main console

## @param FILER             Required: Hostid of filer

## @example ntest FILER=FILERA check_version

## @status public
## @author maji@netapp.com, rreeser@netapp.com

use Cwd qw(abs_path);
use HTML::Mail;
use strict;
use boot;
use Storage::Common_Lib;
use San;
use TCD;

sub kill_shell_process();
sub check_shell_process_terminal();
sub terminatezomprocess();
sub trim($);

########################################
### Initialization/declaration
#########################################
use vars qw(
  $FILER
  $STEST_DEBUG
  $STEST_FILE
  $LOGLVL
  $LOGDIR
  $MAIL_TO
  $MAIL_FROM
  $MAIL_MSG
  $EMAIL
  $TEST_SUITE_NAME
  $TEST_CONFIG
  $GROUP_NO
  $DRIVE_TYPE
  $CMPRSN
);

param( 'FILER',           '-mesg',    'HostID of filer' );
param( 'STEST_DEBUG',     '-default', '1' );
param( 'STEST_FILE',      '-mesg',    'Runlist file' );
param( 'LOGDIR',          '-default', '' );
param( 'LOGLVL',          '-default', '2' );
param( 'MAIL_TO',         '-default', 'default' );
param( 'MAIL_FROM',       '-default', 'default' );
param( 'MAIL_MSG',        '-default', 'n' );
param( 'EMAIL',           '-default', 'n' );
param( 'TEST_SUITE_NAME', '-mesg',    'Test_Suite_Name' );
param( 'TEST_CONFIG',     '-mesg',    'Test config' );
param( 'GROUP_NO',        '-default', '100' );
param( 'DRIVE_TYPE',      '-default', '' );
param( 'CMPRSN',          '-default', '' );

my @Testcases = ( enable_partition_gp5 => "enable_partition_gp5" );
my $Test = new TCD( -testcases => [@Testcases] );
my $MNS_BODY;
unsetting_affp() if ( $LOGDIR !~ /NVMe|UPDATE_APOLLO_AFU/ );
unset_compression_var() if ( $CMPRSN == 1 );
enable_partition_gp5( $GROUP_NO, $Test );
logcomment("Compression Flag : $CMPRSN");

if ( ( $EMAIL =~ /NO/i ) || ( $EMAIL =~ /N/i ) ) {
    $EMAIL = 'Y';
}

my $original_filer_name = $FILER;
if ( $TEST_CONFIG eq 'E' ) {
    $FILER =~ s/\,/\_/;
    logcomment("$LOGDIR and $TEST_SUITE_NAME and $FILER");
    my $nohup_file = "$LOGDIR/../../../../nohup_" . $TEST_SUITE_NAME . "_" . $FILER . "\.txt";
    system("sudo rm -rf $nohup_file");

    my @FILER_ARR = split( /_/, $FILER );
    $FILER = $FILER_ARR[0];

}

my $host = host($FILER);

my $Home = $ENV{HOME};

$Home = trim($Home);

my $filer   = $host->hostname;
my $len     = rindex $STEST_FILE, '.plan';
my $runlist = substr( $STEST_FILE, 0, $len );

my $san_obj = new San( \*ntest::NTEST_STDOUT );
if ( $san_obj->error() ) {
    logresult( 'FAIL', "Failed to instantiate San object" . $san_obj->errmsg() );
}

#del_parsed_dir();

my $rslt_file = "$LOGDIR/../../../HDD_SUMMARY_RESULTS";

if ( -e $rslt_file ) {

    open( H_LOGFILE, "< $rslt_file " )
      or $san_obj->sanlog( -msg => "FILER- $filer : Failed opening $rslt_file" . scalar( localtime() ) );

    my ( $rslt_srt, $statuss ) = ( "", "Status: PASSED" );
    foreach my $line (<H_LOGFILE>) {
        if ( $rslt_srt =~ m/FAILED/ ) {
            $statuss = "Status: INSPECT";
        }
        $rslt_srt = $rslt_srt . $line;

        #  if ($line =~ /(.*):(.*):(.*):(.*):(.*):(.*)/) {
        #		$rslt_srt =  $rslt_srt . "   $2:$3:$4:$5:$6\n";
        #	}
    } ## end foreach my $line (<H_LOGFILE>)

    $san_obj->sanlog( -msg => "\n-----------------------------------------------------------------------\n" . "   TEST RESULT - FILER: $filer  $statuss" . "\n\n-----------------------------------------------------------------------\n$rslt_srt" . "\n-----------------------------------------------------------------------\n" );
} else {
    $san_obj->sanlog( -msg => "\n-----------------------------------------------------------------------\n" . "   TEST RESULT - FILER: $filer  " . "\n\n-----------------------------------------------------------------------\n" . "    No result available\n-----------------------------------------------------------------------\n" );

}

$san_obj->sanlog( -msg => "FILER- $filer : This FILER is free now :" . scalar( localtime() ) );

## Send result to email ids

if ( ( ( $EMAIL eq 'Y' ) || ( $EMAIL eq 'y' ) ) && ( ( $MAIL_FROM ne 'default' ) || ( $MAIL_TO ne 'default' ) ) ) {
    my $file_na = $original_filer_name;
    $file_na =~ s/,|\s+/-/g;
    my $mail_subject = "Final Test Result : $file_na : $STEST_FILE ";
    my $mail_body;

    if ( $MAIL_MSG eq 'y' ) {
        my $defailt_vals = "$Home/NDATE/default_params";
        open( H_LOGFILE, "< $defailt_vals" )
          or die "Failed opening $defailt_vals \n";

        my @defaults;

        foreach my $line (<H_LOGFILE>) {
            if ( $line =~ /^EMAIL_MSG=(.*)/ ) {
                $MAIL_MSG = $1;
            }
        }
    } ## end if ($MAIL_MSG eq 'y')
    if ( $LOGDIR =~ /UPDATE_APOLLO_AFU/ ) {
        my $main_log_dir;
        if ( $LOGDIR =~ /(\S+)\/HDD\/(\S+)/ ) {
            $main_log_dir = $1;
            logcomment("Logdir is $main_log_dir");
        }
        opendir( SDIR_LOG, $LOGDIR );
        my @files_DIR_SLOG = readdir(SDIR_LOG);
        print "\n-------------------------------------------------------\n";
        print "Files present in logpath:@files_DIR_SLOG\n";
        print "\n-------------------------------------------------------\n";
        foreach my $file (@files_DIR_SLOG) {
            if ( $file =~ /^summary_NDATE|UPDATE_APOLLO_AFU_v/ ) {
                logcomment("File found !! copy o main : $file");
                my $fl = "$LOGDIR" . "/" . "$file";
                logcomment("copy file : $fl");
                system("sudo cp -rf $fl $main_log_dir");
            }
        }
        logcomment("Files copied");
    } elsif ( $LOGDIR =~ /NVMe_MNS_Setup/ ) {
        my $main_log_dir;
        if ( $LOGDIR =~ /(\S+)\/HDD\/(\S+)/ ) {
            $main_log_dir = $1;
            logcomment("Logdir is $main_log_dir");
        }
        opendir( SDIR_LOG, $LOGDIR );
        my @files_DIR_SLOG = readdir(SDIR_LOG);
        print "\n-------------------------------------------------------\n";
        print "Files present in logpath:@files_DIR_SLOG\n";
        print "\n-------------------------------------------------------\n";
        my ( $filers, $version, $build, $drv_cap, $ns1, $ns2, $ns3, $ns4, $filerA, $filerB, $result, $col );
        foreach my $file (@files_DIR_SLOG) {

            if ( $file =~ /^NVMe_MNS_setup/ ) {
                logcomment("NVMe- Log file MNS found");
                my $fl = "$LOGDIR" . "/" . "$file";
                open( FH, "$fl" ) || logresult( 'FATAL', " Cannot open the file $fl" );
                my @arr1 = <FH>;
                close(FH);
                foreach my $line (@arr1) {
                    if ( $line =~ /Filers present are\s+(\S+)/ ) {
                        print "$1\n";
                        $filers = $1;
                    }
                    if ( $line =~ /NDATE Build Version used\s+\:\s+(\S+)/ ) {
                        $version = $1;
                    }
                    if ( $line =~ /NDATE Build Date\s+\:\s+(\S+)/ ) {
                        $build = $1;
                    }
                    if ( $line =~ /Drive Capacity\s+\:\s+(\S+)/ ) {
                        $drv_cap = $1;
                    }
                    if ( $line =~ /Name Space Size\s+1\-(\S+)\s+2\-(\S+)\s+3\-(\S+)\s+4\-(\S+)\s+/ ) {
                        $ns1 = $1;
                        $ns2 = $2;
                        $ns3 = $3;
                        $ns4 = $4;
                    }
                    if ( $line =~ /FILER A Container\s+\:\s+(.*)/ ) {
                        $filerA = $1;
                    }
                    if ( $line =~ /FILER B Container\s+\:\s+(.*)/ ) {
                        $filerB = $1;
                    }
                    if ( $line =~ /Pass:\s+(\S+)/ ) {
                        print "$line" if ( $1 == 1 );
                        $result = "PASSED" if ( $1 == 1 );
                        $col = "#ffff00";
                    }
                    if ( $line =~ /Fail:\s+(\S+)/ ) {
                        print "$line" if ( $1 == 1 );
                        $result = "FAILED" if ( $1 == 1 );
                        $col = "#FF6347";
                    }
                    if ( $line =~ /Script:\s+(\S+)/ ) {
                        print "$line\n" if ( $1 == 1 );
                        $result = "SCRIPT_ERROR" if ( $1 == 1 );
                        $col = "#FF6347";
                    }
                }
                logcomment("Captured all values");
                $MNS_BODY = "<table><tbody><tr><td><h1><span style = \"text-decoration: underline\;\">Multi Namespace Setup Results</span></h1> </td></tr></tbody></table>
<table><tbody><tr>
<td>NDATE Version:</td><td>$version</td></tr>
<tr><td>Build Date:</td><td>$build</td></tr>
</tbody></table>
<p>&nbsp;</p><table border=\"1\"><tbody><tr>
<td><strong>Test Log:</strong></td><td>$main_log_dir</td></tr>
<tr><td><strong>Test Result:</strong></td><td style=\"background-color:$col;\"><strong>$result</strong></td></tr></tbody></table>
<hr /><table border=\"1\"><tbody><tr><td><strong>Filer :</strong></td><td>$filers</td></tr>
<tr><td><strong>Drive Capacity:</strong></td><td>$drv_cap</td></tr>
<tr><td><strong>Namespace - N1:</strong></td><td>$ns1</td></tr>
<tr><td><strong>Namespace - N2:</strong></td><td>$ns2</td></tr>
<tr><td><strong>Namespace - N3:</strong></td><td>$ns3</td></tr>
<tr><td><strong>Namespace - N4:</strong></td><td>$ns4</td></tr></tbody></table>
<p>&nbsp;</p><table border=\"1\"><tbody>
<tr><td>FILER_A Drives</td><td>$filerA</td></tr>
<tr><td>FILER_B Drives</td><td>$filerB</td></tr></tbody></table>";
                logcomment("copy file : $fl");
                my $time = time();
                logcomment("Time MNS : $time");
                system("sudo cp -rf $fl $main_log_dir");
            }
        }
        logcomment("MNS Files copied");
    } else {
        my $main_log_dir;
        if ( $LOGDIR =~ /(\S+)\/HDD\/(\S+)/ ) {
            $main_log_dir = $1;
            logcomment("Logdir is $main_log_dir");
        }
        my $SUMMARY;
        if ( -f "$main_log_dir/TEST_SUMMARY_DETAILS" ) {
            $SUMMARY = "$main_log_dir/TEST_SUMMARY_DETAILS";
        } else {
            $SUMMARY = "$main_log_dir/HDD_SUMMARY_RESULTS";
        }
        logcomment("Opening file :: $SUMMARY");
        open( RESULT_FILE, "$SUMMARY" ) || die "Can not open file $SUMMARY";
        my %TEST_DETALS;
        my @TEST_DETALS;
        my $SCRIPT_NAME;
        my $T_EXE_TIME;
        my $Total_time_taken = 0;

        foreach my $line (<RESULT_FILE>) {
            $mail_body = $mail_body . $line;
            if ( $line =~ /SCRIPT\s+NAME\s*:\s*(.*)/ ) {
                $SCRIPT_NAME = $1;
                next;
            }
            if ( $line =~ /SCRIPT\s*RESULT\s*:\s*(.*)/ ) {
                $TEST_DETALS{$SCRIPT_NAME} = $1;
                push( @TEST_DETALS, $SCRIPT_NAME );
                next;
            }
            if ( $line =~ /TOTAL\s+TIME\s+TAKEN\s*:\s*(.*)Hours\,\s*(.*)Minutes\,\s*(.*)Seconds/ ) {
                my $hrsec  = $1 * 3600;
                my $minsec = $2 * 60;
                my $timsec = $hrsec + $minsec + $3;
                $Total_time_taken = $Total_time_taken + $timsec;
                print "Time is sec - $Total_time_taken\n";
            }
        }
        my %color = ( 'PASSED' => "#46ED30", 'FAILED' => "#E60303", 'FATAL' => "#E60303" );
        my $table;
        my @arr    = keys %TEST_DETALS;
        my @values = values %TEST_DETALS;
        logcomment( "Testcases :  @arr :" . scalar @arr );
        logcomment( "Status :  @values :" . scalar @values );
        my $sec = $Total_time_taken;
        my $DAY = int( $sec / ( 24 * 60 * 60 ) );
        my $Hr  = ( $sec / ( 60 * 60 ) ) % 24;
        my $Min = ( $sec / 60 ) % 60;
        my $SEC = $sec % 60;

        if ( $DAY > '0' ) {
            print "Total execution time - $DAY DAY(s) $Hr HOUR(s) $Min MINUTE(s) $SEC SECOND(s)\n";
            $T_EXE_TIME = "Total execution time - $DAY DAY(s) $Hr HOUR(s) $Min MINUTE(s) $SEC SECOND(s)\n";
        } else {
            print "Total execution time - $Hr HOUR(s) $Min MINUTE(s) $SEC SECOND(s)\n";
            $T_EXE_TIME = "Total execution time - $Hr HOUR(s) $Min MINUTE(s) $SEC SECOND(s)\n";
        }

        foreach my $key (@TEST_DETALS) {
            logcomment("Getting the Status of script : $key");
            next if ( $key !~ /\S/ );
            $table = $table . '<tr bgcolor=' . $color{ $TEST_DETALS{$key} } . '>
			<td>' . "$key" . '</td><td>' . "$TEST_DETALS{$key}" . '</td>
		</tr>';
        }
        $table = "CONSOLIDATED SCRIPT RESULT OVERVIEW :\n\n" . '<table border="1"><tr bgcolor="#21CECB"><th>SCRIPT NAME</th><th>FINAL RESULT</th></tr>' . "$table" . '</table>' . "\n\n<b>TOTAL EXECUTION TIME : $T_EXE_TIME </b>\n\nINDIVIDUAL SCRIPT RESULT SUMMARY :";
        logcomment("table : $table");
        {
            my %replace   = ( 'NO_RUN' => "\'background-color: #E60303;\'>NO_RUN", 'FAILED' => "\'background-color: #E60303;\'>FAILED", 'PASSED' => "\'background-color: #46ED30;\'>PASSED", 'FATAL' => "\'background-color: #E60303;\'>FATAL" );
            my $replace   = 'SCRIPT RESULT : <span style=';
            my $replace_1 = '</span>';
            $mail_body =~ s/SCRIPT RESULT\s*:\s*(\S+)/$replace$replace{$1}$replace_1/g;

        }
        $mail_body = $table . "\n" . $mail_body;
        if ( $EMAIL !~ /d/i ) {
            my %mail_params = (
                HTML    => $mail_body,
                Text    => $mail_body,
                From    => $MAIL_FROM,
                To      => $MAIL_TO,
                Subject => $mail_subject,
            );

            $mail_params{HTML} = "<HTML><pre>$mail_params{Text}<\\HTML>";
            my $mail = HTML::Mail->new(%mail_params);
            $mail->send()
              or logresult( "CONFIG", "trouble sending mail" );
        } else {
            logcomment("EMAIL NOT SENT BECAUSE EMAIL SET TO DEBUG BY USER.");
        }

    } ## end if ((($EMAIL eq 'Y') ||...
}
## Send result to email ids end

nvme_afu_summary_mail() if ( $LOGDIR =~ /UPDATE_APOLLO_AFU/ );
nvme_mns_send_mail()    if ( $LOGDIR =~ /NVMe_MNS_Setup/ );

my $logdirt = "$Home/NDATE/TEST_LOGS";

my $new_log_dir = `cd $LOGDIR/../../.. ; pwd`;
$san_obj->sanlog( -msg => "----------------------------------------------------------------------" );
$san_obj->sanlog( -msg => "FILER- $filer : Log location $new_log_dir" . scalar( localtime() ) );
$san_obj->sanlog( -msg => "----------------------------------------------------------------------" );

kill_shell_process();

$san_obj->sanlog( -msg => "FILER- $filer : Stopping background shell process" );
$san_obj->sanlog( -msg => "----------------------------------------------------------------------" );

if ( ( $LOGLVL <= 3 ) ) {
    $san_obj->sanlog( -msg => "Cleaning Directories" );

    #   system("rm -f $LOGDIR/*");
    #system("rm -f $LOGDIR/../hdd-*.log");
    #system("rm -f $LOGDIR/../../*.log");
    #system("rm -f $LOGDIR/../../../00_*");
    #system("rm -f $LOGDIR/../../../HDD.log");
    #system("rmdir -fr $LOGDIR/../../../HDD");
} ## end if (($LOGLVL <= 3))

#if (($LOGLVL <= 3) && (!($LOGDIR=~ /$logdirt/))) {

#         logcomment("deleting directory : $LOGDIR");
#      system("rm -fr $LOGDIR");

#} elsif (($LOGLVL <= 3) && (($LOGDIR=~ /$logdirt/))) {
#	logcomment("deleting parser logs ");
#   system("rm $LOGDIR/pars*.log");
#}
my $user;

sub nvme_mns_send_mail() {
    print "DEBUG NVME CHECK MAIL : $LOGDIR\n";
    my $mail_subject = "MNS TOOL SUMMARY : $STEST_FILE \($original_filer_name\)";
    my $mail_body;
    my $SUMMARY;
    my @mail_cont;
    $mail_body = $MNS_BODY;
    my %mail_params = (
        HTML    => $mail_body,
        Text    => $mail_body,
        From    => $MAIL_FROM,
        To      => $MAIL_TO,
        Subject => $mail_subject,
    );
    $mail_params{HTML} = "<HTML><pre>$mail_params{Text}<\\HTML>";
    my $mail = HTML::Mail->new(%mail_params);
    $mail->send() or logresult( "CONFIG", "trouble sending mail" );
}

sub nvme_afu_summary_mail () {
    print "DEBUG NVME CHECK MAIL : $LOGDIR\n";
    my $mail_subject = "AFU TOOL SUMMARY : $STEST_FILE \($original_filer_name\)";
    my $mail_body;
    my $SUMMARY;
    my @mail_cont;

    #summary_NDATEa
    opendir( DIR_LOG, $LOGDIR );
    my @files_DIR_LOG = readdir(DIR_LOG);
    print "\n-------------------------------------------------------\n";
    print "Files present in logpath:@files_DIR_LOG\n";
    print "\n-------------------------------------------------------\n";
    foreach my $file (@files_DIR_LOG) {
        if ( $file =~ /^summary_NDATE/ ) {
            $SUMMARY = $file;
        }
    }
    $SUMMARY = "$LOGDIR" . "/" . "$SUMMARY";
    logcomment("Opening file :: $SUMMARY");
    open( AFU_SU, "<$SUMMARY" ) || print "Could not open $SUMMARY...";
    while (<AFU_SU>) {

        #17:06:07
        if ( $_ =~ /(00|[0-9]|1[0-9]|2[0-3]):([0-9]|[0-5][0-9]):([0-9]|[0-5][0-9])\s+\-\s+|LOG\s+\:\s+http|^$/ ) {
            print "$_";
            next;
        } else {
            push( @mail_cont, $_ );
            $mail_body = $mail_body . $_;
        }
    }
    my %mail_params = (
        HTML    => $mail_body,
        Text    => $mail_body,
        From    => $MAIL_FROM,
        To      => $MAIL_TO,
        Subject => $mail_subject,
    );

    $mail_params{HTML} = "<HTML><pre>$mail_params{Text}<\\HTML>";
    my $mail = HTML::Mail->new(%mail_params);
    $mail->send() or logresult( "CONFIG", "trouble sending mail" );

}

sub kill_shell_process() {
    ##kill the shell process associated with the current ntest process
    # Check 1 :
    my $runlistprocess = `ps -ef | grep check | grep $runlist`;
    if ( $runlistprocess =~ /sh/g ) {
        foreach $_ ( split( /\n/, $runlistprocess ) ) {
            if ( $_ =~ /\s+(\d+)\s+(.*)\/check.sh/ ) {
                system("kill -9 $1 2>/dev/null");
            }
        }
    }
    $san_obj->sanlog( -msg => "----------------------------------------------------------------------" );
    $san_obj->sanlog( -msg => "FILER- $filer : Killing the shell process associated with this ntest" );
    $san_obj->sanlog( -msg => "----------------------------------------------------------------------" );

    #Check 2:
    open( FH, "<$Home/NDATE/TEST_LISTS/$STEST_FILE" );
    while (<FH>) {
        if ( $_ =~ /PID\s*=\s*(\d+)/ ) {
            system("kill -9 $1 2>/dev/null");
        }
    }
    close(FH);

    my ( $tty, $ps );
    $tty = `tty`;
    $tty =~ s/^ //;
    $tty =~ s/\s$//;
    $tty =~ s/\/dev\///;
    $user = `whoami`;
    $ps   = `ps -ef | grep $tty | grep /NDATE/SUPPORT_SCRIPTS/check.sh | grep $user`;
    if ( $ps =~ /sh/g ) {
        foreach $_ ( split( /\n/, $ps ) ) {
            if ( $_ =~ /\s+(\d+)\s+(.*)\/check.sh/ ) {
                system("kill -9 $1 2>/dev/null");
            }
        }
    }

    #check for shell process without ntest
    check_shell_process_terminal();
}

sub check_shell_process_terminal() {
    my ( $ps, $tty_own, $ps1, $process_kill );

    $ps = `ps -ef | grep /NDATE/SUPPORT_SCRIPTS/check.sh | grep pts | grep $user`;

    my @array_ps = ();

    foreach my $process_IDS ( split( /\n/, $ps ) ) {
        if ( $process_IDS =~ /\s+(pts\/\d+)\s+/ ) {
            my $ps_IDS = $1;
            push( @array_ps, $ps_IDS );
        }
    }

    #my @array_ps = split(/\n/, $ps);
    #Removing duplicate terminal ids

    my %test_hash;
    my @temp_arr;

    foreach (@array_ps) {
        if ( $test_hash{$_} ) {
            next;
        } else {
            $test_hash{$_} = 1;
            push( @temp_arr, $_ );
        }
    }

    $tty_own = `tty`;
    $tty_own =~ s/^ //;
    $tty_own =~ s/\s$//;
    $tty_own =~ s/\/dev\///;

    foreach my $ty (@temp_arr) {
        if ( $ty =~ /$tty_own/ ) {
            next;
        }
        $ps1 = `ps -ef | grep $ty | grep ntest | grep $user`;
        foreach my $out ( split( /\n/, $ps1 ) ) {
            if ( $out =~ /NDATE\/NATE\/bin\/ntest/ ) {

                # print "This terminal id $ty has a ntest process running\n";
                last;
            } else {
                $process_kill = `ps -ef | grep $ty | grep $user`;
                if ( $process_kill =~ /\/NDATE\/SUPPORT_SCRIPTS\/check.sh/ ) {
                    foreach $_ ( split( /\n/, $process_kill ) ) {
                        if ( $_ =~ /\s+(\d+)\s+(.*)\/check.sh/ ) {
                            system("kill -9 $1 2>/dev/null");

                            # print "DEBUG3: Process ids to be killed are $1\n";
                            last;
                        }
                    }
                }
            }
        }
    }
    terminatezomprocess();
}

# Sub to terminate zombie process of check.sh.
sub terminatezomprocess() {
    my $cmd      = "ps -ef | grep check | grep \"?\" | grep $user";
    my $check_ps = `$cmd`;

    my $cmd_ntest = "ps -ef | grep ntest | grep $user";
    my $ntest_ps  = `$cmd_ntest`;

    #Capturing the individual ntest processes
    my @ntest_arr = split( "\n", $ntest_ps );
    my @up_ntest_arr;
    foreach my $pro (@ntest_arr) {
        if ( $pro !~ /grep ntest/ ) {
            push( @up_ntest_arr, $pro );
        }
    }
    if ( !(@up_ntest_arr) ) {
        my @check_arr = split( "\n", $check_ps );
        foreach my $pro_ch (@check_arr) {
            if ( $pro_ch !~ /grep check/ ) {
                $pro_ch =~ /\s+(\d+)\s+/;
                system("kill -9 $1 2>/dev/null");
            }
        }
    }
}

# Delete subtestlog dir
sub del_parsed_dir() {
    my ( $dummy_file, $line_index, $logpath, $parse_id, $subtestdir, $result );
    my @Parse_log_arr = ();

    $dummy_file = "$Home/NDATE/SUPPORT_SCRIPTS/PARSED_OUTPUT_$TEST_SUITE_NAME.txt";
    open( PARSE, "$dummy_file" );
    @Parse_log_arr = <PARSE>;
    $line_index    = 1;
    foreach my $line (@Parse_log_arr) {
        $line =~ s/^ //;
        $line =~ s/\s$//;
        if ( $line_index == 1 ) {
            $logpath = $line;
        } elsif ( $line_index == 2 ) {
            $parse_id = $line;
        }
        $line_index++;
    }
    my $san_obj = new San( \*ntest::NTEST_STDOUT );
    if ( $san_obj->error() ) {
        logresult( 'FAIL', "Failed to instantiate San object" . $san_obj->errmsg() );
    }
    $san_obj->sanlog( -msg => "----------------------------------------------------------------------" );
    $subtestdir = $logpath . $parse_id;
    $san_obj->sanlog( -msg => "Deleting $subtestdir\n" );
    $san_obj->sanlog( -msg => "----------------------------------------------------------------------" );

    # Delete Files
    $result = system("rm -rf $subtestdir");

    #		system("rm -rf $dummy_file");
    $san_obj->sanlog( -msg => "Deleted $subtestdir\n" );
    $san_obj->sanlog( -msg => "----------------------------------------------------------------------" );
}

sub trim($) {
    my $string = shift;
    $string =~ s/^\s+|\s+$//gm;
    s/^\"(.*)\"$/$1/ || s/^\'(.*)\'$/$1/;
    return $string;
}

sub enable_partition_gp5($$) {
    my $gp_id = shift;
    my $Test  = shift;
    logcomment("Checking for PARTITIONED file");
    logcomment("Log - $LOGDIR");
    my $dir        = "$LOGDIR/../../../";
    my @main_dir   = `ls $dir`;
    my $file_found = 0;
    foreach (@main_dir) {

        if ( $_ =~ /^PARTITION_CREATED/ ) {
            logcomment("Partition was disabled on this system. Checking for group number");
            $file_found = 1;
            last;
        } else {
            logcomment("Partition was not disabled, Proceeding without enabling partition");
        }
    }
    logcomment("Selected Group - $gp_id");
    if ( ( $gp_id =~ /5|7|6|NADQ02_SSD_Durability|NADQ02_ERT/ ) && ( $file_found == 1 ) ) {
        logcomment("Group $gp_id has been selected, Partition was enabled before starting test exectuion, Enabling partition");
        my @Nodes = NACL::C::Node->find();
        my @re_subtest;
        foreach my $Node (@Nodes) {
            $Node->refresh_command_interface();
            my $filer_name = $Node->name();
            logcomment( "Calling change_state " . scalar( localtime() ) );
            push(
                @re_subtest,
                subtest(
                    \&boot_filer_in_req_state, -runid, "shutdown_node_" . "$filer_name", -bg, "--",
                    filer_name      => $filer_name,
                    change_state_to => 'FIRMWARE'
                )
            );
        }
        Subtest::wait_finish( subtest => [@re_subtest] );
        logcomment("Booted Filers in FIRMWARE");
        foreach my $Node (@Nodes) {
            $Node->refresh_command_interface();
            my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ];
            my $FILER          = $Node->name();
            my $host           = host($FILER);
            my $prompt_array   = $host->prompt_fw();
            my $prompts        = join( "|", @$prompt_array );
            my $prompt_fw      = "($prompts)";
            my $Transit_obj    = NACL::Transit->new( name => $FILER );

            #my $state = $Transit_obj->get_state('timeout' => 180);
            my $state = $Transit_obj->get_state( 'timeout' => 180, 'get_state_timeout' => 7200 );
            if ( $state =~ /FIRMWARE/ ) {
                logcomment("FILER is in LOADER prompt, disabling auto partition env variables");
                my $Server = new Server( no_cli => 1 );
                $Server->set_execution_server( -Server => $FILER );
                $Server->console(
                    cmd                        => 'unsetenv root-uses-shared-disks?',
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                $Server->console(
                    cmd                        => 'unsetenv allow-ssd-partitions?',
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                $Server->console(
                    cmd                        => 'unsetenv root-uses-shared-ssds?',
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                $Server->console(
                    cmd                        => 'saveenv',
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                logcomment("Saved all required environment variables to enable autopartition");
            }
        }
        logcomment("Set-up Filer, Re-initilize filer");
        setup_cmode_filers(
            node_present => \@Nodes,
            Test         => $Test
        );
        logcomment("Partition functionality has been enabled");
        my $dir2rd   = "$LOGDIR/../../../";
        my @dir_file = `ls $dir2rd`;
        foreach (@dir_file) {
            if ( $_ =~ /^PARTITION_CREATED/ ) {
                my $handle;
                open( $handle, '>>', "$LOGDIR/../../../PARTITION_CREATED" ) or die("Cant open $LOGDIR/../../../PARTITION_CREATED");
                print $handle "PARTITION_ENABLED=yes";
                close($handle) or die("$LOGDIR/../../../PARTITION_CREATED");
                last;

=head
    	        	logcomment("Deleting PARTITION_CREATED file...");
				system ("sudo rm -rf $_");
				logcomment("Deleted PARTITION_CREATED file");
=cut

            } else {
                logcomment("PARTITION_CREATED file could not be found");
            }
        }
    } else {
        logcomment("Execution didn't disabled partition during start of the test");
    }
}

sub unsetting_affp() {
    my @Nodes = NACL::C::Node->find();
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Host    = host($FILER_C);
        my $model_name_aff;
        my $sysconfig_out;
        my $Api_Set_Obj;
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $execution_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("DEBUG:Filer state is : $execution_state");

        if ( $execution_state eq 'CLI' ) {
            $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        } else {
            $Api_Set_Obj = NACL::APISet->new(
                hostobj   => $Host,
                category  => "Node",
                interface => "CLI",
                set       => "Maintenance"
            );
        }
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 );
        if ( $execution_state =~ /CLI|UP/i ) {
            logcomment("Executing sysconfig -v for checking platform");
            $sysconfig_out = $Api_Set_Obj->execute_command( 'command' => 'sysconfig -v', 'timeout' => 600 );
        } elsif ( $execution_state =~ /MAINT/ ) {
            logcomment("Executing sysconfig -v for checking platform");
            $sysconfig_out = $Api_Set_Obj->execute_raw_command( 'command' => 'sysconfig -v' );
        }

        chomp( my @sysconf_lines = split( /\r?\n/, $sysconfig_out ) );
        my $shelf_id;
        foreach my $line (@sysconf_lines) {
            if ( $line =~ /Model Name:\s*(\S+)/ig ) {
                $model_name_aff = $1 unless ( defined $model_name_aff );
            }
        }
        if ( ( $DRIVE_TYPE =~ /SSD$/i ) && ( $model_name_aff !~ /^AFF-A700s/ ) ) {
            my $Transit_obj = NACL::Transit->new( name => $FILER_C );
            my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
            my $prompt_fw;
            my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ];
            my $prompt_array   = $Host->prompt_fw();
            my $prompts        = join( "|", @$prompt_array );
            $prompt_fw = "($prompts)";
            logcomment("This is the prompt regex: $prompt_fw");
            my $Server = new Server( no_cli => 1 );
            $Server->set_execution_server( -Server => $FILER_C );    #set the execution server to the current node
            logcomment("Filer state is : $filer_state");

            if ( $filer_state !~ /LOADER|FIRMWARE/ ) {
                logcomment("Changing the state to FIRMWARE");
                $Transit_obj->change_state( to => "FIRMWARE" );
            }
            my $filer_state1 = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
            if ( $filer_state1 =~ /LOADER|FIRMWARE/ ) {
                logcomment("$FILER_C : Unsetting AFFP bootarg.init.flash_optimized");
                my $addr;
                my $mask;
                my $gw;
                my $loadermod;
                my $Home         = $ENV{HOME};
                my $filer        = "$FILER_C" . "_C";
                my $host_file    = "$Home/NDATE/FILER_INFO/$filer";
                my $defailt_vals = "$Home/NDATE/FILER_INFO/$filer";
                open( H_LOGFILE, "< $defailt_vals" )
                  or die "Failed opening $defailt_vals \n";

                foreach my $data (<H_LOGFILE>) {

                    #                my $filer_config = `cat $host_file`;
                    #               foreach my $data ( split /\n/, $filer_config )
                    if ( $data =~ /^\#/ ) {
                        next;
                    } elsif ( $data =~ /^MGMT_IP=(.*)/ ) {
                        $addr = $1;
                    } elsif ( $data =~ /^MGMT_MASK=(.*)/ ) {
                        $mask = $1;
                    } elsif ( $data =~ /^MGMT_GATEWAY=(.*)/ ) {
                        $gw = $1;
                    } elsif ( $data =~ /^loaderModefi_Nightly=(.*)\s*/ ) {
                        $loadermod = $1;
                    }
                }
                $Server->console(
                    cmd                        => "priv set test",
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                $Server->console(
                    cmd                        => "ifconfig e0M -addr=$addr -mask=$mask -gw=$gw -dns=172.19.2.30",
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                $Server->console(
                    cmd                        => "ifconfig",
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                $Server->console(
                    cmd                        => "mod execute $loadermod",
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { ".*Starting AUTOBOOT press Ctrl-C.*" => "\cC" },
                    ".*LOADER.*>"              => "FINAL_PROMPT",
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", },
                );
                $Server->console(
                    cmd                        => "unsetenv -ro bootarg.init.flash_optimized",
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                $Server->console(
                    cmd                        => "saveenv",
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                logcomment("As filer $FILER_C was in $filer_state before setting the environment variable, changing back to $filer_state state");
                $Transit_obj->change_state( to => "$filer_state" );

                my $filr_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
                logcomment("Filer state is : $filr_state");

            }
        }
    }
}

sub unset_compression_var() {
    my @Nodes = NACL::C::Node->find();
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        my $Host        = host($FILER_C);
        my $prompt_fw;
        my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ];
        my $prompt_array   = $Host->prompt_fw();
        my $prompts        = join( "|", @$prompt_array );
        $prompt_fw = "($prompts)";
        logcomment("This is the prompt regex: $prompt_fw");
        my $Server = new Server( no_cli => 1 );
        $Server->set_execution_server( -Server => $FILER_C );    #set the execution server to the current node
        logcomment("Filer state is : $filer_state");

        if ( $CMPRSN == 1 ) {
            logcomment("COMPRESSION : $FILER_C : As user selected option -c=1, Unsetting the environment variable wafl-disable-packing?");
            if ( $filer_state !~ /LOADER|FIRMWARE/ ) {
                logcomment("Changing the state to FIRMWARE");
                $Transit_obj->change_state( to => "FIRMWARE" );
            }
            my $filer_state1 = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
            if ( $filer_state1 =~ /LOADER|FIRMWARE/ ) {
                $Server->console(
                    cmd                        => "unsetenv wafl-disable-packing?",
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                $Server->console(
                    cmd                        => "saveenv",
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
                $Server->console(
                    cmd                        => "printenv wafl-disable-packing?",
                    additional_prompts_timeout => 60,
                    ignore                     => $ignore_strings,
                    additional_prompts         => { "$prompt_fw" => "FINAL_PROMPT", }
                );
            }
        }
        logcomment("As filer $FILER_C was in $filer_state before setting the environment variable, changing back to $filer_state state");
        $Transit_obj->change_state( to => "$filer_state" );
        my $filr_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("Filer state is : $filr_state");
    }
}