#!/usr/software/bin/perl
# $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/TEST_SCRIPTS/CMODE/NADQ02_FC_Disk_FW_Download_Background.thpl#1 $
#
## @summary Mode
##  Disk Firmware Download in background/foreground mode
#
##
## @description
## Verify that the drive can download disk firmware in foreground and background mode properly
## using the method available in the NetApp system.
##
## @Test Mode
##  File system mode
##
## @Test bed setup
##  FC   : Cluster Fabric
##  SATA : Cluster
##  SAS  : Cluster
##
## @usage
## The test can be run independently or with other tests as part of STEST.
##
## @dependencies
##  Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground
##
## @steps
## The test will execute steps mentioned below:
##
##
## @status  Automated
########################################
### Library functions
##########################################
use strict;
use Storage::NVMe_Common_Lib;
use Storage::Common_Lib;
use NACL::C::Statistics;
use NACL::MTask::EventLogDetector;
#########################################
#########################################
### Initialization/declaration
#########################################
#####
# Global parameters
use vars qw(
  $FILER
  $MODE
  $FILER_CONN
  $TEST_CONFIG
  $TEST_SETUP
  $FILER_PROMPT
  $LOGDIR
  $EOE
  $TEST_WAIT_TIME
  $BOOT_MODE
  $BOOT_TYPE
  $SSD
  $EMAIL
  $MAIL_TO
  $MAIL_FROM
  $FILER_TYPE
  $FIRMWARE
  $RUNID
  $ARMADILLO
);
my $params = NATE::ParamSet->new( global => 1 );
$FILER          = $params->get( 'FILER',          default => 'Filer' );
$TEST_CONFIG    = $params->get( 'TEST_CONFIG',    default => 'E' );
$TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
$LOGDIR         = $params->get( 'LOGDIR',         default => undef );
$EOE            = $params->get( 'EOE',            default => 1 );
$EMAIL          = $params->get( 'EMAIL',          default => 'y' );
$MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
$MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
$TEST_SETUP     = $params->get( 'TEST_SETUP',     default => 'SAS' );
$BOOT_MODE      = $params->get( 'BOOT_MODE',      default => '1' );
$BOOT_TYPE      = $params->get( 'BOOT_TYPE',      default => 'A' );
$RUNID          = $params->get( 'RUNID',          default => undef );
$SSD            = $params->get( 'SSD',            default => 'no' );
$ARMADILLO      = $params->get( 'ARMADILLO',      default => '2' );
$FILER_TYPE     = $params->get( 'FILER_TYPE',     default => 'BR' );
$FIRMWARE       = $params->get( 'FIRMWARE',       default => 'Firmware file not entered' );
################
# Testcase name
################
my $TC_name;
$TC_name = "407_NADQ02_NVMe_Disk_FW_DL_Bgnd";

#Common variable declaration
my ( $email_subject, $email_body );
my @Nodes;
my $filer_names;
my $Mode;
my $test_status = 0;
my %nodes_filer;
my $firmware_hash = {};
my @firmware_list;
my $cnt = 1;
my $disk_show_v;
my %test_details;
my $Home = $ENV{HOME};

my $test_wait_time = 12 * 3600;    #12 Hrs test
if ( $TEST_WAIT_TIME == 1 ) {
    $test_wait_time = ( $test_wait_time / 10 );
} elsif ( $TEST_WAIT_TIME == 2 ) {
    $test_wait_time = ( $test_wait_time / 2 );
} elsif ( $TEST_WAIT_TIME == 4 ) {
    $test_wait_time = ( ($test_wait_time) * (.95) );
}

#########################################
# Test case available for execution
#########################################

my @Testcases = ( Disk_FW_Download_Background => "Disk_FW_Download_Background." );
&main();

sub main {

    # Debug break point
    $DB::single = 2;

    #  Create Test Case Driver object
    $Test = new TCD( -testcases => [@Testcases] );

    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        $test_status = 1;

        $email_subject = "$TC_name : Test FAILED: FAIL";
        $email_body    = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n";
        send_message(
            mail_subject => $email_subject,
            mail_body    => $email_body,
            MAIL_FROM    => $MAIL_FROM,
            MAIL_TO      => $MAIL_TO,
            EOE          => $EOE,
            EMAIL        => $EMAIL
        );

        return $TCD::FAIL;
    }

    # Performs method callbacks
    $Test->run_test();
    if ( $Test->error ) {
        $Test->log( $Test->errmsg );
        return $TCD::FAIL;
    }

    exit(0);
} ## end sub main
########## INIT  ###################################################
# This init subroutine will initialise the filer.
####################################################################
sub init() {
    $Test->description(" Initialising all required variables and filer connections ");
    $filer_names = $Test->get_param("FILER");    # Capturing Filer names from the param(Test_Suite)
    ##Check for duplicate node object and push unique node object in Nodes array.
    my @temp_nodes = NACL::C::Node->find();      # Find Nodes/Filers used in the test, Based on FILER param.
    foreach my $Node (@temp_nodes) {
        my $FILER_C = $Node->name();
        $nodes_filer{$FILER_C} = $Node;
    }
    @Nodes = values(%nodes_filer);               # Contains Node object used for test execution.
    sort(@Nodes);
    $Mode = $Nodes[0]->mode();
    logcomment("Checking for execution mode");
    logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) );
    logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) );

    return $TCD::PASS;
}
########## SETUP  ###################################################
# setup automatically called before executing tests
####################################################################
sub setup() {
    $Test->description("Setup the environment for the test execution ");
    logcomment("Mode of filer $filer_names : $Mode");
    my $node_ref = \@Nodes;
    version_test( node_present => $node_ref, tc_name => $TC_name );
    #####################################################################
    # Pre test proces : call for pre_n_post test process
    #####################################################################
    nvme_pre_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );

    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Host        = host($FILER_C);
        logcomment("disable autogiveback ");
        my $prompts_answers = [ 'Do you want to disable auto-giveback?' => 'y', 'Do you want to continue' => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback false", 'connectrec-match_table' => $prompts_answers );
        logcomment("Disable background firmwar update option");
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable off" );
        logcomment("Stop firmware update command");
        try {
            my $prompts = [ '.*Are you sure you want to.*' => 'y', '.*stop firmware update qualification.*' => 'y' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk_fw_update -S\"", 'connectrec-match_table' => $prompts );
            logcomment("Firmware Qualification download stopped ");
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            logcomment("FW Download stop test Failed.");
        };
        logcomment("$FILER_C : Turn off the raid options");
        $Api_Set_Obj->execute_raw_command( command => "set -showseparator \" \"" );
        $Api_Set_Obj->execute_raw_command( command => "\013" );
        $Api_Set_Obj->execute_raw_command( command => "set -showallfields false" );
        $Api_Set_Obj->execute_raw_command( command => "\013" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.enable off" );
        $Api_Set_Obj->execute_raw_command( command => "\013" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.spares.enable off" );
        $Api_Set_Obj->execute_raw_command( command => "\013" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.scrub.enable off" );
        $Api_Set_Obj->execute_raw_command( command => "\013" );

    }

    #Checking for firmware files; if not exist, then making the test NORUN - BURT1042571
    my $disk_dir      = "$Home/NDATE/FIRMWARE/DISKFW/HIGH";
    my $firmware_list = nvme_map_fw_dsk_rev_model(
        Node             => $Nodes[0],
        disk_fw_location => $disk_dir,
    );
    my %firmware_hash = %$firmware_list;
    my %hash;
    my ( $key, $value );
    while ( ( $key, $value ) = each %firmware_hash ) {
        push( @{ $hash{$value} }, $key );
    }
    my @drive_list = grep { (/\S/) } keys(%firmware_hash);
    logcomment("The list of drives on which firmware will download : @drive_list ");
    my @firmware_list = unique( values(%firmware_hash) );
    logcomment("The list of firmware files present : @firmware_list ");
    if ( !@firmware_list ) {
        logcomment("**NORUN**: NO FIRMWARE FILES ARE AVAILABALE FOR FILER");
        return $TCD::FAIL;
    }

    return $TCD::PASS;
}

sub Disk_FW_Download_Background {
    my @subtests;
    my $status = 0;
    logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        push( @subtests, subtest( \&Disk_FW_Download_Background_sub, -runid, "Disk_FW_Download_Background_$FILER_C", -bg, "--", $Node ) );
    }
    Subtest::wait_finish( subtest => [@subtests] );
    my $status = status_return( subtest_ref => [@subtests] );
    logcomment("Total test status is : $status");
    if ( $status == 0 ) {
        return $TCD::PASS;
    } else {
        return $TCD::FAIL;
    }
}

sub Disk_FW_Download_Background_sub {
    $Test->description("Disk_FW_Download_Background");
    my ( @Nodes, %filer_nvol );
    my $Home = $ENV{HOME};
    my @NON_RT_AGG;
    my @NON_RT_VOLS;
    my %filer_nvol       = ();
    my $create_agg_n_vol = 0;
    my $aggr_vol_found   = 0;

    push( @Nodes, shift(@_) );
    logcomment("Total Nodes  - @Nodes");
    logcomment( "Filer passed to the subtest :: " . $Nodes[0]->name() );

    #STEP 1
    nvme_eye_catcher( Test => $Test, string => "STEP 1 of 9: List all drives " );
    my $disk_list = nvme_list_only_nvme( node_present => [ $Nodes[0] ] );

    #STEP 2
    nvme_eye_catcher( Test => $Test, string => "STEP 2 of 9: Update to lower disk firmware" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Downgrade firmware for filer :$FILER_C using firmware files from DISKFW/LOW directory");
        my $low_dir = "$Home/NDATE/FIRMWARE/DISKFW/LOW";
        my @fw_dwmld_drvs = nvme_disk_fw_update_low_or_high( Node => $Node, FW_DIR_LOC => $low_dir, FORCE_UPDTE => "yes" );
        logcomment("Filer $FILER_C firmware downgrade compelted");
        logcomment("Filer $FILER_C : Drives @fw_dwmld_drvs , Firmware downgraded to version present in $low_dir directory");
    }

    #STEP 3
    nvme_eye_catcher( Test => $Test, string => "STEP 3 of 9: Check background firmware update option" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Check background firmware update option");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable" );
    }

    #STEP
    nvme_eye_catcher( Test => $Test, string => "STEP 4 of 9: Check non-root aggregate and create one non-root aggregate" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node );
        if (@non_rt_agg) {
            logcomment("Total non-root aggregate is $non_rt_status and Non -root aggregate are @non_rt_agg");
            logcomment("Check for non-root volume existance in available non root aggregate");
            my @vol_created = check_non_root_volumes( Node => $Node );
            if (@vol_created) {
                logcomment("$FILER_C : Volumes which are available in the filer are :: @vol_created");
                $filer_nvol{$FILER_C} = [@vol_created];
                $aggr_vol_found = 1;
            } else {
                $create_agg_n_vol = 1;
                vserver_delete( Node => $Node );    #Vserver deletion
                aggregate_volume_delete( Node => $Node );
            }
        } else {
            logcomment("Non-root aggregate cannot be found");
            $create_agg_n_vol = 1;
        }
        if ( $create_agg_n_vol == 1 ) {
            logcomment("Check for spare drive availability, Minimum 3 spares are required to create aggregate");
            my $spare_list  = nvme_sto_agg_show_spare_dsk( Node => $Node );
            my @spare_drvs  = keys %$spare_list;
            my $spare_count = @spare_drvs;
            logcomment("$FILER_C : Total spare are : $spare_count and Spare Drive are : @spare_drvs");
            if ( $spare_count > 3 ) {
                logcomment("Sufficient spares are available on Filer $FILER_C");
                logcomment("Checking for non-root aggregate and creating aggregate ");
                my $drv_ag = $spare_count - 2;
                my @aggr_lst = nvme_get_agg_dtls_n_create( Node => $Node, disk_count => $drv_ag );
                logcomment("Aggregate created - @aggr_lst");
                if ( !@aggr_lst ) {
                    logcomment("Non-root Aggregate were not found, Cannot create non-root aggregate");
                    logcomment("**FATAL** : $FILER_C  : Non-root Aggregate were not found, Cannot create non-root aggregate");
                    logresult( 'FATAL', "$FILER_C  : Non-root Aggregate were not found, Cannot create non-root aggregate" );
                }
                my $add_aggr = @aggr_lst;
                logcomment("Total Additional aggregate count is $add_aggr and aggregate are @aggr_lst is present");
                @NON_RT_AGG = @aggr_lst;
                my $required_size;
                my $vol_size = 90;
                foreach my $agg (@aggr_lst) {
                    logcomment("Get Usable size of Aggregate $agg");
                    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
                    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
                    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
                    $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" );
                    my $aggr_sz = $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate show -aggregate $agg -node $FILER_C -root false -fields state,availsize" );
                    foreach my $line ( split( /\n/, $aggr_sz ) ) {
                        next if ( $line =~ /---/ );
                        if ( $line =~ /(\S+)\s+(\S+)\s+(online|offline)/ ) {
                            my $agg_sz = $2;
                            my ( $aggs, $unit ) = $agg_sz =~ /(\S+)(KB|MB|GB|TB|PB)/;
                            logcomment("Aggregate $agg size is $aggs");
                            $required_size = int( ( $vol_size / 100 ) * $aggs );
                            $required_size = "$required_size" . "$unit";
                        }
                    }
                }
            } else {
                logcomment("Spare available is not sufficient to create additional aggregate");
                logcomment("**FATAL**:$FILER_C  : Spare available is not sufficient to create additional aggregate");
                logresult( 'FATAL', "$FILER_C  : Spare available is not sufficient to create additional aggregate" );
            }
        } else {
            if ( $aggr_vol_found == 1 ) {
                logcomment("Already aggregate and volume exists on filer :$FILER_C, re-using aggregates and volume");
            } else {
                logcomment("**FATAL**:$FILER_C  :Cannot get volume and aggregate status");
                logresult( 'FATAL', "$FILER_C  : Cannot get volume and aggregate status" );
            }
        }
    }

    #STEP 4
    nvme_eye_catcher( Test => $Test, string => "STEP 5 of 9: Create volume and start a thread of Hammer with the file size 1234" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Create volume and start a thread of Hammer with the file size 1234");
        logcomment("check for non-root volume");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        if ( (@NON_RT_AGG) && ( $aggr_vol_found == 0 ) ) {
            logcomment("NO NON_ROOT VOLUMES FOUND, CREATE VOLUMES ON AGGREGATE AVAILABLE");
            my $Api_Set_Obj  = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
            my $vserver_name = "vser_$FILER_C";
            my $root         = "volvs";
            my $vser_cret    = 0;
            logcomment("Creating vserver $vserver_name");
            my $vser_out = $Api_Set_Obj->execute_raw_command( 'command' => "vserver show -fields vserver,rootvolume" );
            if ( $vser_out =~ /($vserver_name)\s+(\S+)/ ) {
                logcomment("Vserver $vserver_name found on $FILER_C and root volume is $2 ");
                if ( $2 =~ /-/ ) {
                    logcomment("Vserver does not have root volume, Destroy and create new vserver");
                    $Api_Set_Obj->execute_raw_command( 'command' => "vserver delete -vserver $vserver_name -foreground true" );
                    sleep 3;
                    $vser_cret = 1;
                }
            } else {
                logcomment("Cannot Find Vserver on $FILER_C");
                $vser_cret = 1;
            }
            if ( $vser_cret == 1 ) {
                logcomment("Create Vserver $vserver_name with root volume $root on filer $FILER_C");
                my $prompts_answers = [ ".*Do you want to continue??.*" => 'y' ];
                my $vser_o = $Api_Set_Obj->execute_raw_command( 'command' => "vserver create -vserver $vserver_name -rootvolume $root", 'connectrec-match_table' => $prompts_answers );
                logcomment("DEBUGL : $vser_o");
            }
            my $vol_name;
            my $nm = 1;
            foreach my $ag (@NON_RT_AGG) {
                my $vol_size = 90;
                my $agg_sz;
                my $required_size;
                $vol_name = "vol_" . "$FILER_C" . "$nm";
                $vol_name =~ s/-/_/g;
                logcomment("Create Volume on aggregate $ag with 90% of aggregate size");
                my $aggr_sz = $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate show -aggregate $ag -node $FILER_C -root false -fields state,availsize" );
                foreach my $line ( split( /\n/, $aggr_sz ) ) {
                    next if ( $line =~ /---/ );
                    if ( $line =~ /(\S+)\s+(\S+)\s+(online|offline)/ ) {
                        my $agg_sz = $2;
                        my ( $aggs, $unit ) = $agg_sz =~ /(\S+)(KB|MB|GB|TB|PB)/;
                        logcomment("Aggregate $ag size is $aggs");
                        $required_size = int( ( $vol_size / 100 ) * $aggs );
                        if ( ( $required_size > 100 ) && ( $unit =~ /TB/ ) ) {
                            logcomment("Volume size to be created is $required_size $unit, changing to 100TB");
                            $required_size = 100;
                        }
                        $required_size = "$required_size" . "$unit";
                    }
                }
                logcomment("Volume size to be created : $required_size");
                logcomment("Aggregate $ag will be used to create vol $vol_name");
                my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
                my $ot = $Api_Set_Obj->execute_raw_command( 'command' => "vol create -vserver $vserver_name -volume $vol_name -aggregate $ag -size $required_size -state online", 'connectrec-match_table' => $prompts_answers );
                logcomment("$ot");
                if ( $ot =~ /Error|Invalid|Failed/ ) {
                    logcomment("**FATAL** : Volume $vol_name cannot be created");
                    logresult( 'FATAL', "$FILER_C : Volume $vol_name cannot be created" );
                }
                logcomment("Volume $vol_name created");
                push( @NON_RT_VOLS, $vol_name );
                $nm++;
            }
            logcomment("Volumes Created are :@NON_RT_VOLS");
            $filer_nvol{$FILER_C} = [@NON_RT_VOLS];
        }
        logcomment("Volume found on filer $FILER_C, re-using available volumes") if ( $aggr_vol_found == 1 );
        my @n_root_vol = @{ $filer_nvol{$FILER_C} };
        foreach my $vol (@n_root_vol) {
            logcomment("$FILER_C : Start a thread of kernel tool  Hammer with the file size 1234 on volume $vol");
            my $filename = "/vol/$vol/h0";
            my $size     = '1234';
            my $result   = hammer_start(
                Node      => $Node,
                file_name => $filename,
                file_size => $size,
                Test      => $Test,
            );
            if ( $result == 1 ) {
                logcomment("- $FILER_C - Hammer with file $filename with size $size is not executed properly");
                logcomment("- $FILER_C - Thread of kernel tool Hammer with file-size 1234 failed on $FILER_C");
                logcomment("**FAIL** :THREAD OF KERNEL TOOL HAMMER WITH FILE-SIZE 1234 FAILED ON $FILER_C,skipping rest of the test steps : Step 5 ");
                return logresult( 'INFO', msg => 1 );
            } else {
                logcomment("Thread of kernel tool Hammer with file-size 1234 started on $FILER_C");
            }
            logcomment("$FILER_C : Start a thread of kernel tool  Hammer with the file size 123456 on volume $vol");
            my $filename = "/vol/$vol/h1";
            my $size     = '123456';
            my $result   = hammer_start(
                Node      => $Node,
                file_name => $filename,
                file_size => $size,
                Test      => $Test,
            );
            if ( $result == 1 ) {
                logcomment("- $FILER_C - Hammer with file $filename with size $size is not executed properly");
                logcomment("- $FILER_C - Thread of kernel tool Hammer with file-size 123456 failed on $FILER_C");
                logcomment("**FAIL** :THREAD OF KERNEL TOOL HAMMER WITH FILE-SIZE 123456 FAILED ON $FILER_C,skipping rest of the test steps : Step 5 ");
                return logresult( 'INFO', msg => 1 );
            } else {
                logcomment("Thread of kernel tool Hammer with file-size 123456 started on $FILER_C");
            }
        }
    }

    #STEP 5
    nvme_eye_catcher( Test => $Test, string => "STEP 6 of 9: Verify that hammer is running." );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $return = hammer_status( Node => $Node, Test => $Test );
        if ( $return == 0 ) {
            logcomment("$FILER_C: Hammer is running ");
        } else {
            logcomment("Hammer not running on $FILER_C");
        }
        sleep(10);
    }

    #STEP 4
    nvme_eye_catcher( Test => $Test, string => "STEP 7 of 9: Copy firmware from HIGH directory" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        logcomment("Rename firmware files from and copy to filer direcotry");
        my $high_dir = "$Home/NDATE/FIRMWARE/DISKFW/HIGH";
        my ( $ret_val, $fw_file_list ) = nvme_rename_fw_files( Node => $Node, rename_to => "99", disk_fw_location => $high_dir );
        logcomment("RETURN VALUE : $ret_val");
        logcomment("FW : $fw_file_list");
        if ( $ret_val == 1 ) {
            logcomment("$FILER_C : Drives have NT firmware version, NT is not supported ");
            logcomment("**NO_RUN** : $FILER_C : Drives have NT firmware version, NT is not supported ");
            return logresult( 'INFO', msg => 1 );
        }

    }

    #STEP 5
    nvme_eye_catcher( Test => $Test, string => "STEP 8 of 9: Enable background firmware update option" );
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $loop    = 1;

        #my $timeout = 1800;  #30 min
        my $timeout = $test_wait_time;
        logcomment("Test time is $timeout");
        logcomment("Enable Back ground firmeare updaet ");
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=1" );
        my $bk_cmd_out = $Api_Set_Obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable true" );
        logcomment("Background fimrware update enabled");
        logcomment("cmd output : $bk_cmd_out");
        sleep 20;
        my $curtime   = time;
        my $endtime   = $curtime + $timeout;
        my $fw_bg_fld = 0;
        my @fw_passed_drv;

        while ( $curtime < $endtime ) {
            logcomment("Current time : $curtime and Endtime :$endtime");
            logcomment("Check for background firmware messages on filer $FILER_C");
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000000000 );
            my $bck_out = $Api_Set_Obj->execute_raw_command( command => " " );
            logcomment("BAVK : $bck_out");
            logcomment("Waiting for 60 sec");
            sleep 60;
            foreach my $line ( split( /\n/, $bck_out ) ) {

                if ( $line =~ /STARTING/ ) {
                    if ( $line =~ /\s+Disk\s+(\S+)/ ) {
                        logcomment("Starting Firmware download on drive : $1");
                        chomp($1);
                        push( @fw_passed_drv, $1 ) if ( $1 =~ /(\S+\.\S+)/ );
                    }
                }
            }
            if (@fw_passed_drv) {
                logcomment("Background firmware started on drives @fw_passed_drv");
            }
            $curtime = time;
        }
        if (@fw_passed_drv) {
            logcomment("Background firmware started on drives @fw_passed_drv");
        } else {
            $fw_bg_fld = 1;
        }
        if ( $fw_bg_fld == 1 ) {
            logcomment("Background fimrware update failed on filer $FILER_C");
            logcomment("**FAIL** : $FILER_C Background fimrware update failed on filer ");
            return logresult( 'INFO', msg => 1 );
        }
    }

    #STEP 5 :
    nvme_eye_catcher( Test => $Test, string => "STEP 9 of 9: Disable disk fw debug level and abort hammer" );
    logcomment("Reset back disk fw debug level ");
    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=0" );
        my $response = $Api_Set_Obj->hammer_status();
        logcomment("Hammer Status - $response ");
        my $response = $Api_Set_Obj->hammer_abort();
        logcomment("Hammer thread stopped ");
    }
}

#####################################################################
# Cleanup - Post Process
#####################################################################

sub cleanup() {
    $Test->nlog(" $filer_names - Clean up and post test process");
    @Nodes = node_new_obj( node_array => [@Nodes] );
    my $node_ref = \@Nodes;
    nvme_post_test(
        node_present    => $node_ref,
        Test            => $Test,
        change_state_to => "CLI",
        filer_mode      => $Mode
    );

    ###########################################################################################
    ## Delete/Reset test specific options and values
    ###########################################################################################
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name;
        my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        my $Host        = host($FILER_C);
        logcomment("disable autogiveback ");
        my $prompts_answers = [ 'Do you want to disable auto-giveback?' => 'y', 'Do you want to continue' => 'y' ];
        $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback false", 'connectrec-match_table' => $prompts_answers );
        logcomment("Disable background firmwar update option");
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable off" );
        logcomment("Stop firmware update command");
        try {
            my $prompts = [ '.*Are you sure you want to.*' => 'y', '.*stop firmware update qualification.*' => 'y' ];
            $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk_fw_update -S\"", 'connectrec-match_table' => $prompts);
            logcomment("Firmware Qualification download stopped ");
        }
        catch NACL::APISet::Exceptions::ResponseException with {
            logcomment("FW Download stop test Failed.");
        };
        logcomment("$FILER_C : Turn off firmware dwbug messages");
        $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=0" );
        logcomment("$FILER_C : Turn off the raid options");
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.enable off" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.spares.enable off" );
        $Api_Set_Obj->execute_raw_command( command => "run local options raid.scrub.enable off" );
        my $response = $Api_Set_Obj->hammer_status();
        logcomment("Hammer Status - $response ");
        my $response = $Api_Set_Obj->hammer_abort();
        logcomment("Hammer thread stopped ");

    }

    return $TCD::PASS;
}