#!/usr/software/bin/perl

use strict;
use Subtest;

## Module imports
use TCD;
use NACL::APISet;
use NACL::C::Node;
use NACL::Transit;
use Storage::Common_Lib;
use NATE::ParamSet;
use Storage::Ems qw(ems_log_get_mark ems_log_dump_by_type ems_log_search ems_log_dump);
use Storage::Disk qw(disk_fw_update);
use Storage::SASUtils qw(ems_error_monitor start_filerhammer);
use RaidInfo;
use NACL::STask::Node;
use filer;
use misc;
use Utils;
use mount;
use Sys::Hostname;
use NACL::STask::Aggregate;
use Storage::DiskUtils qw(vol_stop_scrub wait_for_scrub get_filerdisk_model get_filer_fwfile check_scrub_error compare_disks check_fwfile_exist create_vol);
use Storage::Common_Lib;

# Specified the parameter
# Global parameters
use vars qw(
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $ARMADILLO
);

param ('FILER', -mesg, "hostspec of the filer");
param ('MAIL_TO', -mesg, "Email To");
param ('MAIL_FROM', -mesg, "Email from");

my $params = NATE::ParamSet->new( global => 1 );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'D' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 'default' );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$FILER_PROMPT   = $params->get( 'FILER_PROMPT',   default => '\*>' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );

my $TC_name;
$TC_name = "401_NADQ02_Disk_FW_DL";

########################################
### Initialization/declaration
#########################################
my $test_status = 0;
my %nodes_filer = {};
my @Nodes;
my $filer_names;
my $Mode;
my ( $email_subject, $email_body );
$EOE = 1;
my $Home;
my @drive_list    = ();
my %firmware_hash = ();
my @firmware_list = ();
my $PWR_SLEEP_TIME = 0;
my $var  = 1;
my @non_root_aggr;
my $test_wait_time = (43200/2);    ##equal to 6 hrs
logcomment("Default wait time is 6 hrs");
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 10% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 1");
}
elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 2");
}
elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( $test_wait_time * .95 );
    my ( $sec, $min, $hour ) = gmtime($test_wait_time);
    logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 4");
}

#########################################
# Testcase available for execution
#########################################
## Pre-test processes
#########################################

my @Testcases = ( Disk_FW_FSys => "Disk_FW_FSys" );

&main();


sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }


    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# init automatically called before executing tests
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and FILER connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = [$Node];
    }
    @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer);    # Contains Node objects used for test execution.
    grep { logcomment( "Filers passed to the ntest are " . $_->name() ) } @Nodes;
    $Mode = $Nodes[0]->mode();                                      # gets mode(7mode/cmode) of the nodes used in test execution.
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 12 hours : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );
    return $TCD::PASS;
}
########## SETUP  ###################################################
# Setup automatically called before executing tests
#####################################################################
sub setup() {
    $Test->description("Setup the environment for the test exectuion ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
	my @subtests;
	#---------------------------------------------------
	# boot the filer in CLI state
	#--------------------------------------------------
    foreach my $Node (@Nodes) {
	    my $FILER_C = $Node->name();
		my $Transit_obj    = NACL::Transit->new( name => $FILER_C );
        my $Original_State = $Transit_obj->get_state();
		if($Original_State !~ /CLI/){
        	push( @subtests, 
				subtest(\&boot_filer_in_req_state,-runid,  "boot_to_CLI_" . "$FILER_C", -bg, "--", filer_name => $FILER_C,  Test  => $Test, change_state_to => "CLI"
                    )  );
		}else {
			logcomment("$FILER_C: Filer is in CLI state");
		}
	}
	Subtest::wait_finish( subtest => [@subtests] );
		
    ###########################################################################################
    ## Set test specific options and values
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        my $Host        = host($FILER_C);
        sleep(60);
        $Node->refresh_command_interface();
        $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset();
        my $prompts_answers = [ 'Do you want to disable auto-giveback?' => 'y', 'Do you want to continue' => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback false", 'connectrec-match_table' => $prompts_answers );
        enable_firmware_console_message( Node => $Node );
        logcomment("$FILER_C : Turn off the raid options");
        NACL::C::Options->option(
            command_interface => $Node,
            'option-name'     => 'raid.media_scrub.enable',
            'option-value'    => 'off'
        );
        NACL::C::Options->option(
            command_interface => $Node,
            'option-name'     => 'raid.media_scrub.spares.enable',
            'option-value'    => 'off'
        );
        NACL::C::Options->option(
            command_interface => $Node,
            'option-name'     => 'raid.scrub.enable',
            'option-value'    => 'off'
        );
    }
    $Home = $ENV{HOME};
    chdir("$Home/NDATE/FIRMWARE/DISKFW/HIGH");
    my @disk_fw = <*.LOD>;
    if ( @disk_fw < 1 ) {
        $test_status = 1;
        logcomment("**FATAL**: No disk firmware file detected in $Home/NDATE/FIRMWARE/DISKFW/HIGH, exiting from rest of the test cases.");
        logresult( 'FATAL', "No disk firmware file detected in $Home/NDATE/FIRMWARE/DISKFW/HIGH, exiting from rest of the test cases." );
    }
	logcomment("check the non-root volume, if not present  creating the non-root volume");
	my @non_root_aggr_obj;
	my $no_create;
	foreach my $Node(@Nodes){
		my $FILER_C   =   $Node->name();
		my @non_root_volumes = check_non_root_volumes(Node =>$Node);
		if(!(scalar @non_root_volumes)){
			logcomment("$FILER_C : Non-root volumes are not found in the filer ");
			my @spares;
	        my %part_spares;
    	    my $API_obj = $Node->get_7m_or_nodescope_apiset(connid => "console");
        ###################################################################################
        ## check for 4 for agrr create (if non-root aggr is not present)
        ###################################################################################
	        my $required_spare = 4;
    	    @spares = spare_count_cmode( Node => $Node, waiting => "wait",required_spare => $required_spare );
	        my @spares_aggr;
    	    if( scalar @spares >= $required_spare){
        	    @spares_aggr = splice(@spares,2,);
            	logcomment("$FILER_C :Available spare drives ". scalar @spares." are @spares @spares_aggr");
	        }else {
    	        logcomment("check for additional aggregate");
            ###################################################################################
            ## non-root aggregate check
            ###################################################################################
        	    try {
            	        my @aggregates=NACL::C::Aggregate->find( command_interface => $Node,
                                              filter => {nodes => [$FILER_C]},
                                               canned_filters    => [ qw(not-root) ]
                                              );
                	    foreach my $aggr (@aggregates) {
                    	    my $aggr_name = $aggr->get_one_state_attribute('aggregate');
                        	my $disk_count = $aggr->get_one_state_attribute('diskcount');
	                        logcomment("**DEBUG** : aggregate with $disk_count disks is $aggr_name");
    	                    my $vol = $aggr->get_one_state_attribute('volcount');
        	                push(@non_root_aggr,$aggr_name);
            	            push(@non_root_aggr_obj,$aggr);
                	        logcomment("$FILER_C : Non root aggregates $disk_count disks are @non_root_aggr");
                    	    $no_create = 1;
                        	last;
                    	}
	            } catch NACL::Exceptions::NoElementsFound with {
                      logcomment("**FATAL**:$FILER_C : Non root aggregates are NOT found, Not Enough spare drive to create non-root aggregate. ");
                      logresult( 'FATAL',"Filer :$FILER_C : Non root aggregates are NOT found,Not Enough spare drive to create non-root aggregate " );
    	        };
        	}
		 	if ($no_create==1) {
                aggregate_volume_delete(Node =>$Node);
            ## checking the Spare drives
                @spares = spare_count_cmode( Node => $Node, waiting => "wait",required_spare => $required_spare );
                if(scalar @spares < $required_spare){
                    logcomment("**FATAL**: Test requires the minimum 4 spare for creating non-root aggreagate\n$FILER_C The required spare drives are NOT Present");
                    logresult( 'FATAL'," Test requires the minimum 4 spare for creating non-root aggreagate\n$FILER_C The required spare drives are NOT Present");
                }
            ## reserve 4 spares and create additional aggregate with rest of drives
                @spares_aggr = splice(@spares,2,);
        	}
	        create_flex_vol(Node => $Node, disk_list => [@spares_aggr]);
		}
	}
	foreach my $Node(@Nodes){
            my $cmd     = "priv set -q test";
            my $FILER_C = $Node->name();
            my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
            my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
            $Api_Set_Obj->execute_raw_command( 'command' => "run local");
            $Api_Set_Obj->execute_raw_command( 'command' => "priv set -q test");
            $Api_Set_Obj->execute_raw_command( 'command' => "dbg level diskfw=1");
            my $vers = $Api_Set_Obj->execute_raw_command( 'command' => "version");
            if ($vers =~ /<1d>/i){
                 logcomment("DEBUG kernel is present on filer");
            }else{
                logcomment("**FATAL** : NON-DEBUG kernel is present on filer, disk_fw_update -F is not supported on");
                logresult('FATAL',"NON-DEBUG kernel is present on filer, disk_fw_update -F is not supported on");
            }
			$Api_Set_Obj->execute_raw_command( 'command' => "exit" );
    }
	###########################################################################################
    #Creating separate list of related firmware file
    my $disk_dir      = "$Home/NDATE/FIRMWARE/DISKFW/HIGH";
    my $firmware_list = cm_disk_firmware(
        Node             => $Nodes[0],
        disk_fw_location => $disk_dir,
    );
    %firmware_hash = %$firmware_list;
    my %hash;
    my ( $key, $value );
    while ( ( $key, $value ) = each %firmware_hash ) {
        push( @{ $hash{$value} }, $key );
    }
    @drive_list = grep { (/\S/) } keys(%firmware_hash);
    logcomment("The list of drives on which firmware will download : @drive_list");
    @firmware_list = unique( values(%firmware_hash) );
    logcomment("The list of firmware files present : @firmware_list");
    if ( !@firmware_list ) {
        $test_status = 1;
        logcomment("**FATAL**: NO firmware Files are available for Filer ");
        return logresult( "INFO", msg => $test_status );
    }
    eye_catcher( Test => $Test, string => "$filer_names : Copying the FW files to filer directory" );
    foreach my $Node (@Nodes) {
        my $FILER_C  = $Node->name();
        my @fw_files = copy_FW_files_to_filer_directory(
            Node       => $Node,
            Files_path => $disk_dir,
            Files      => \@firmware_list
        );
        if ( !@fw_files ) {
            $test_status = 1;
            logcomment("**FATAL**: $FILER_C - Disk firmware files are not found in /etc/disk_fw, exiting from the test");
            return logresult( "INFO", msg => $test_status );

            #logresult( 'FATAL', "$FILER_C - Disk firmware files are not found in /etc/disk_fw, exiting from the test" );
        }

    } 
   return $TCD::PASS;
}

########## TEST 1 ###################################################
#NADQ02_Disk_FW_FSys_Download
#####################################################################


sub Disk_FW_FSys {
    my @subtests;
    logcomment( "Number of nodes are " . scalar @Nodes );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&Disk_FW_FSys_sub, -runid, "Disk_FW_FSys_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $return = status_return( subtest_ref => [@subtests] );
    if ( $return == 0 ) {
        return $TCD::PASS;
    }
    else {
        return $TCD::FAIL;
    }

}
#############################################################################
############################ MAIN SUBTEST ###################################
#############################################################################
sub Disk_FW_FSys_sub {
    my $FATAL = 0;
    my @Nodes;
    push( @Nodes, shift(@_) );

##########
## Test variables
##########
    my @actual_drives;
    my @total_drives;
    my $disk_show_step2;
    my $disk_show_step6;

    $Test->description("Disk_FW_FSys");
#####################################################################################################
## Calcualte time out
#####################################################################################################
    my $TIME_OUT_MIN;
    my $TIME_OUT_MAX;
    my $timeout_value;
    my $max_disk_size = get_max_disk_size(
        node_present => \@Nodes
    );
    logcomment("Max disk size : $max_disk_size GB");
    $max_disk_size = ($max_disk_size * 2147483648)/1000;
    if ( $max_disk_size <= 20971520 ) {
        $TIME_OUT_MIN  = 300;
        $TIME_OUT_MAX  = 14400;
        $timeout_value = 43200;
    }elsif ( $max_disk_size <= 2147483648 ) {
        $TIME_OUT_MIN  = 300 * 5;
        $TIME_OUT_MAX  = 14400 * 5;
        $timeout_value = 64800;
    }else {
        $TIME_OUT_MIN  = 300 * 25;
        $TIME_OUT_MAX  = 14400 * 25;
        $timeout_value = 72000;
    }

 ###########################################################################################
 ## TEST STARTS
 ###########################################################################################

##########################################
# Part1: Test Duration - 6hrs
#########################################
	
	foreach my $Node (@Nodes) {
			
			my $FILER_C = $Node->name();
			eye_catcher( Test => $Test, string => "PART $var :Test Duration is 6 hours");
	#------------------------------------------------------------
	## STEP 1 - Run IOs on all volumes using FSIO
	## Pass/Fail criteria: N/A
	#------------------------------------------------------------
		    my $FILER_C = $Node->name();
    		eye_catcher( Test => $Test, string => "PART $var : STEP 1 of 3: $FILER_C : Run IOs on all volumes using FSIO" );
        	my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
			my @NON_ROOT_VOLS = check_non_root_volumes(Node =>$Node);
			foreach my $vol_name(@NON_ROOT_VOLS){
	            my $Api_Set_Obj  = $Node->get_7m_or_nodescope_apiset();
    	        my $output=$Api_Set_Obj->execute_command( 'command' => "df $vol_name");
			}
    	    foreach my $vol_name(@NON_ROOT_VOLS){
        	    for ( my $no_files = 0 ; $no_files <= 19 ; $no_files++ ) {
					my $output = $Api_Set_Obj->execute_command( 'command' => "filersio asyncio_active 80 -r 50 4k 0 12g 48200 10 /vol/$vol_name/file-$no_files -create", 'timeout' => $TIME_OUT_MAX );
				    if($output =~ /failed to start/ig){
				         logcomment ("**FATAL** : $FILER_C : Cannot Set size attribute on file");
				         logresult ('FATAL',"$FILER_C : Cannot Setting size attribute on file ");
    				}
            	}
        	}
	        logcomment("Work load initiated on $FILER_C");
	#-----------------------------------------------------------
	## STEP 2 and 3 - While IOs are running issue firmware download to all drives in parallel with Power Cycle
	## Pass/Fail criteria: N/A
	#------------------------------------------------------------
			my @subtests_3;
	    	push( @subtests_3, subtest( \&firmware_Pwr_Cyc_download, -runid, "$var"."_firmware_Pwr_Cyc_download_$FILER_C", -bg, "--", $Node ) );
			Subtest::wait_finish( subtest => [@subtests_3] );
			logcomment("PART $var : Execution of Running Execution on IOs and Firmware Downloading and Power Cycle on all drives in parallel is finished");
		
	$var = 2;
##########################################
# Part2: Test Duration - 6hrs
#########################################
   		 
            eye_catcher( Test => $Test, string => "PART $var :Test Duration is 6 hours");
    #------------------------------------------------------------
    ## STEP 1 - Run IOs on all volumes using FSIO
    ## Pass/Fail criteria: N/A
    #------------------------------------------------------------
            eye_catcher( Test => $Test, string => "PART $var : STEP 1 of 3: $FILER_C : Run IOs on all volumes using FSIO" );
			my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(connid => "console");
			my $output = $Api_Set_Obj->execute_command( 'command' => "filersio status");
        	if($output !~ / no instances currently running/ig){
            	logcomment("$FILER_C : filersio is still running , Stopping the filersio");
	            my $res = $Api_Set_Obj->execute_command( 'command' => "filersio stop");
    	        logcomment("filersio is stopped")if($res =~ /aborting instance|workload completed/ig);
        	    $Api_Set_Obj->execute_command( 'command' => "filersio stop")if($res !~ /aborting instance|workload completed/ig);
        	}
            my @NON_ROOT_VOLS = check_non_root_volumes(Node =>$Node);
            foreach my $vol_name(@NON_ROOT_VOLS){
                my $Api_Set_Obj  = $Node->get_7m_or_nodescope_apiset();
                my $output=$Api_Set_Obj->execute_command( 'command' => "df $vol_name");
            }
            foreach my $vol_name(@NON_ROOT_VOLS){
                for ( my $no_files = 0 ; $no_files <= 19 ; $no_files++ ) {
					my $output = $Api_Set_Obj->execute_command( 'command' => "filersio asyncio_active 80 -r 50 4k 0 12g 48200 10 /vol/$vol_name/file-$no_files -create", 'timeout' => $TIME_OUT_MAX );
                    if($output =~ /failed to start/ig){
                         logcomment ("**FATAL** : $FILER_C : Cannot Set size attribute on file");
                         logresult ('FATAL',"$FILER_C : Cannot Setting size attribute on file ");
                    }

                }
			}
    		logcomment("Work load initiated on $FILER_C");

    #------------------------------------------------------------
    ## STEP 2 and 3 - While IOs are running issue firmware download to all drives in parallel with Power Cycle.
    ## Pass/Fail criteria: N/A
    #------------------------------------------------------------
            my @subtests_4;
            push( @subtests_4, subtest( \&firmware_Pwr_Cyc_download, -runid, "$var"."_firmware_Pwr_Cyc_download_$FILER_C", -bg, "--", $Node ) );
            Subtest::wait_finish( subtest => [@subtests_4] );
            logcomment("PART $var : Execution of Running Execution on IOs and Firmware Downloading and Power Cycle on all drives in parallel is finished");
		        
    }
	logcomment("Test $TC_name is completed.");
    logresult( "INFO", msg => $test_status );

} #End of main subrutin

####################################################################
#Cleanup
####################################################################
sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    $Test->description("Preforming Post test process - Check filer state, Check disk, check root volume,halt hammer, halt scrub, filer config");
    @Nodes = node_new_obj( node_array => [@Nodes] );
#asyncio_active running
# aborting instance
    foreach my $Node(@Nodes){
        my $FILER_C     = $Node->name;
        my $Host        = host($FILER_C);
        sleep(60);
        $Node->refresh_command_interface();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(connid => "console");
        my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio status\"");
        if($output !~ /no instances currently running/ig){
            logcomment("$FILER_C : filersio is still running , Stopping the filersio");
            my $res = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;filersio stop\"");
			logcomment("filersio is stopped")if($res =~ /aborting instance|workload completed/ig);
        }
        logcomment ("$FILER_C :  Executing the disk_fw_update -S ");
		 my @prompts = ( 'Are you sure you want to' => 'y', );
		$Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test;disk_fw_update -S\"", "connectrec-match_table" => \@prompts );
    }

    return $TCD::PASS;
}
############################################################################################################
# FIRMWARE DOWNLOADED AND POWER-CYCLE 
############################################################################################################

sub firmware_Pwr_Cyc_download {
  my @Nodes;
  push( @Nodes, shift(@_) );   
  foreach my $Node(@Nodes){
    my @aggregates;
	my $API_obj = $Node->get_7m_or_nodescope_apiset(connid => "console");
    my $FILER_C  = $Node->name();
    logcomment("$FILER_C : The list of drives on which firmware will download : @drive_list");
    my $disk_c = @drive_list;
    my $time_out = (($disk_c * 180) + 240 );
	 my @prompts = ( 'Are you sure you want to' => 'y', );
    logcomment("**DEBUG**:Time - $time_out");
    my $loop = 1;
	my $time = time;
    my $start_time_sec = time;
    my $end_time_sec   = $start_time_sec + $test_wait_time;
	while ( $start_time_sec < $end_time_sec ) {
		logcomment( "$FILER_C : Firmware Download and Power-cycle started at " . scalar( localtime() ) );
		eye_catcher( Test => $Test, string => "STEP 2($loop) of 3 : $FILER_C :While IOs are running issue firmware download to all drives" ) ;
		$API_obj->set_timeout( "connectrec-timeout" => $time_out );
		if($var == 1){
		    foreach my $value(@firmware_list){
        			my @arr_t = split( /\./, $value );
            		my $prod_id = $arr_t[0];
		            $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test;disk_fw_update -F $value -t $prod_id\" ", "connectrec-match_table" => \@prompts );
					sleep (60);
					
					eye_catcher( Test => $Test, string => "STEP 3($loop) of 3 : $FILER_C :Issue Power Cycle on all drives in parallel while IOs and FW download is Running" ) ;
					foreach my $dsk (@drive_list){
						logcomment("Executing sasadmin dev power cycle on drive $dsk");
						$API_obj->execute_raw_command( 'command' => "run local \"priv set -q test;sasadmin dev_power_cycle $dsk\" ");
					}
					logcomment("**DEBUG**:Waiting for 30 Sec");
					sleep(30);
					
  			}	
		}elsif($var == 2){
            $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test;disk_fw_update -Q\"", "connectrec-match_table" => \@prompts);
			sleep (60);
			eye_catcher( Test => $Test, string => "STEP 3($loop) of 3 : $FILER_C :Issue Power Cycle on all drives in parallel while IOs and FW download is Running" ) ;
			foreach my $dsk (@drive_list){
                logcomment("Executing sasadmin dev power cycle on drive $dsk");
                $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test;sasadmin dev_power_cycle $dsk\"");
				logcomment("**DEBUG**:Waiting for 35 Sec");
				sleep(35);
			}
		}
		$start_time_sec = time ;
		logcomment( "$FILER_C : Firmware Download and Power-Cycle Finished($loop) at " . scalar( localtime() ) );   
		$loop += 1 ;
	}
  }
}