#!/usr/software/bin/perl # $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/TEST_SCRIPTS/CMODE/NADQ02_FC_Disk_FW_Download_Background.thpl#1 $ # ## @summary Mode ## Disk Firmware Download in background/foreground mode # ## ## @description ## Verify that the drive can download disk firmware in foreground and background mode properly ## using the method available in the NetApp system. ## ## @Test Mode ## File system mode ## ## @Test bed setup ## FC : Cluster Fabric ## SATA : Cluster ## SAS : Cluster ## ## @usage ## The test can be run independently or with other tests as part of STEST. ## ## @dependencies ## Disk firmwares should be copied to ~/NDATE/FIRMWARE/DISKFW/HIGH folder for foreground ## ## @steps ## The test will execute steps mentioned below: ## ## ## @status Automated ######################################## ### Library functions ########################################## use strict; use Storage::NVMe_Common_Lib; use Storage::Common_Lib; use NACL::C::Statistics; use NACL::MTask::EventLogDetector; ######################################### ######################################### ### Initialization/declaration ######################################### ##### # Global parameters use vars qw( $FILER $MODE $FILER_CONN $TEST_CONFIG $TEST_SETUP $FILER_PROMPT $LOGDIR $EOE $TEST_WAIT_TIME $BOOT_MODE $BOOT_TYPE $SSD $EMAIL $MAIL_TO $MAIL_FROM $FILER_TYPE $FIRMWARE $RUNID $ARMADILLO $SATA_SETUP ); my $params = NATE::ParamSet->new( global => 1 ); $FILER = $params->get( 'FILER', default => 'Filer' ); $TEST_CONFIG = $params->get( 'TEST_CONFIG', default => 'E' ); $TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' ); $LOGDIR = $params->get( 'LOGDIR', default => undef ); $EOE = $params->get( 'EOE', default => 1 ); $EMAIL = $params->get( 'EMAIL', default => 'y' ); $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); $TEST_SETUP = $params->get( 'TEST_SETUP', default => 'SAS' ); $BOOT_MODE = $params->get( 'BOOT_MODE', default => '1' ); $BOOT_TYPE = $params->get( 'BOOT_TYPE', default => 'A' ); $RUNID = $params->get( 'RUNID', default => undef ); $SSD = $params->get( 'SSD', default => 'no' ); $ARMADILLO = $params->get( 'ARMADILLO', default => '2' ); $FILER_TYPE = $params->get( 'FILER_TYPE', default => 'BR' ); $FIRMWARE = $params->get( 'FIRMWARE', default => 'Firmware file not entered' ); $SATA_SETUP = $params->get( 'SATA_SETUP', default => 'n' ); ################ # Testcase name ################ my $TC_name; $TC_name = "408_NADQ02_NVMe_Ping_Pong"; #Common variable declaration my ( $email_subject, $email_body ); my @Nodes; my $filer_names; my $Mode; my $test_status = 0; my %nodes_filer; my $firmware_hash = {}; my @firmware_list; my $cnt = 1; my $disk_show_v; my %test_details; my $Home = $ENV{HOME}; my $test_wait_time = 12 * 3600; if ( $TEST_WAIT_TIME == 1 ) { $test_wait_time = ( $test_wait_time / 10 ); } elsif ( $TEST_WAIT_TIME == 2 ) { $test_wait_time = ( $test_wait_time / 2 ); } elsif ( $TEST_WAIT_TIME == 4 ) { $test_wait_time = ( ($test_wait_time) * (.95) ); } ######################################### # Test case available for execution ######################################### my @Testcases = ( Disk_FW_Ping_Pong => "Disk_FW_Ping_Pong." ); &main(); sub main { # Debug break point $DB::single = 2; # Create Test Case Driver object $Test = new TCD( -testcases => [@Testcases] ); if ( $Test->error ) { $Test->log( $Test->errmsg ); $test_status = 1; $email_subject = "$TC_name : Test FAILED: FAIL"; $email_body = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n"; send_message( mail_subject => $email_subject, mail_body => $email_body, MAIL_FROM => $MAIL_FROM, MAIL_TO => $MAIL_TO, EOE => $EOE, EMAIL => $EMAIL ); return $TCD::FAIL; } # Performs method callbacks $Test->run_test(); if ( $Test->error ) { $Test->log( $Test->errmsg ); return $TCD::FAIL; } exit(0); } ## end sub main ########## INIT ################################################### # This init subroutine will initialise the filer. #################################################################### sub init() { $Test->description(" Initialising all required variables and filer connections "); $filer_names = $Test->get_param("FILER"); # Capturing Filer names from the param(Test_Suite) ##Check for duplicate node object and push unique node object in Nodes array. my @temp_nodes = NACL::C::Node->find(); # Find Nodes/Filers used in the test, Based on FILER param. foreach my $Node (@temp_nodes) { my $FILER_C = $Node->name(); $nodes_filer{$FILER_C} = $Node; } @Nodes = values(%nodes_filer); # Contains Node object used for test execution. sort(@Nodes); $Mode = $Nodes[0]->mode(); logcomment("Checking for execution mode"); logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) ); logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) ); return $TCD::PASS; } ########## SETUP ################################################### # setup automatically called before executing tests #################################################################### sub setup() { $Test->description("Setup the environment for the test execution "); logcomment("Mode of filer $filer_names : $Mode"); my $node_ref = \@Nodes; version_test( node_present => \@Nodes, tc_name => $TC_name ); ##################################################################### # Pre test proces : call for pre_n_post test process ##################################################################### nvme_pre_test( node_present => $node_ref, Test => $Test, change_state_to => "CLI", filer_mode => $Mode ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name; my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); my $Host = host($FILER_C); logcomment("disable autogiveback "); my $prompts_answers = [ 'Do you want to disable auto-giveback?' => 'y', 'Do you want to continue' => 'y' ]; $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback false", 'connectrec-match_table' => $prompts_answers ); logcomment("Disable background firmwar update option"); $Api_Set_Obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable off" ); logcomment("Stop firmware update command"); try { my $prompts = [ '.*Are you sure you want to.*' => 'y' , '.*stop firmware update qualification.*' =>'y']; $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk_fw_update -S\"", 'connectrec-match_table' => $prompts); logcomment("Firmware Qualification download stopped "); } catch NACL::APISet::Exceptions::ResponseException with { logcomment("FW Download stop test Failed."); }; logcomment("$FILER_C : Turn off the raid options"); $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.enable off" ); $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.spares.enable off" ); $Api_Set_Obj->execute_raw_command( command => "run local options raid.scrub.enable off" ); } #Checking for firmware files; if not exist, then making the test NORUN - BURT1042571 my $disk_dir = "$Home/NDATE/FIRMWARE/DISKFW/HIGH"; my $firmware_list = nvme_map_fw_dsk_rev_model( Node => $Nodes[0], disk_fw_location => $disk_dir, ); my %firmware_hash = %$firmware_list; my %hash; my ( $key, $value ); while ( ( $key, $value ) = each %firmware_hash ) { push( @{ $hash{$value} }, $key ); } my @drive_list = grep { (/\S/) } keys(%firmware_hash); logcomment("The list of drives on which firmware will download : @drive_list "); my @firmware_list = unique( values(%firmware_hash) ); logcomment("The list of firmware files present : @firmware_list "); if ( !@firmware_list ) { logcomment("**NORUN**: NO FIRMWARE FILES ARE AVAILABALE FOR FILER"); return $TCD::FAIL; } return $TCD::PASS; } sub Disk_FW_Ping_Pong { my @subtests; my $status = 0; logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&Disk_FW_Ping_Pong_sub, -runid, "Disk_FW_Ping_Pong_sub_$FILER_C", -bg, "--", $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); my $status = status_return( subtest_ref => [@subtests] ); logcomment("Total test status is : $status"); if ( $status == 0 ) { return $TCD::PASS; } else { return $TCD::FAIL; } } sub Disk_FW_Ping_Pong_sub { my $test_status = 0; $Test->description("Disk_FW_Download Ping Pong"); my ( @Nodes, %filer_nvol ); my $Home = $ENV{HOME}; my $total_drv_count_s2; my %scsi_result_s3 = {}; my %scsi_result_s11 = {}; push( @Nodes, shift(@_) ); logcomment("Total Nodes - @Nodes"); logcomment( "Filer passed to the subtest :: " . $Nodes[0]->name() ); #Checking for non root volumes foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $vol = nvme_check_non_root_aggr( Node => $Node ); logcomment("Non -root volumes are "); } #STEP 0 - Copy low and Hig firlware files to filer #STEP 1 nvme_eye_catcher( Test => $Test, string => "STEP 1 of 12: List all drives " ); my $disk_list = nvme_list_only_nvme( node_present => [ $Nodes[0] ] ); my @dsk_list = keys %$disk_list; $total_drv_count_s2 = scalar(@dsk_list); #STEP 2 nvme_eye_catcher( Test => $Test, string => "STEP 2 of 12: Check to see if there is any unusual increase in the drive error logs" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $scsi_result_s3{$FILER_C} = scsi_disk_errors( Node => $Node ); logcomment("Record the log sense data from each drive completed : $FILER_C"); } #STEP 3 nvme_eye_catcher( Test => $Test, string => "STEP 3 of 12: Record device information before testing. All device in the system" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Get Drive owned by $FILER_C"); my @drv_own = nvme_get_disk_owned( Node => $Node ); logcomment("Drives @drv_own belongs to $FILER_C"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); foreach my $disk_t (@drv_own) { if ( $disk_t =~ /P/ ) { ( $disk_t = $disk_t ) =~ s/P\d//g; } ( my $mydisk = $disk_t ) =~ s/$FILER_C\://g; logcomment("Excute scsi inquiry on $mydisk"); my $output = $Api_Set_Obj->execute_command( 'command' => "priv set -q test;scsi inquiry $mydisk" ); } } #STEP 4 nvme_eye_catcher( Test => $Test, string => "STEP 4 of 12: Record HDD FRU information for all HDD FRU in the system" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $stsb_status; if ( $SATA_SETUP eq 'Y' ) { logcomment("Checking for stsb/acorn command is available "); $stsb_status = get_filer_check( Node => $Node ); chomp($stsb_status); logcomment("DEBUG :$stsb_status"); my @drv_own = nvme_get_disk_owned( Node => $Node ); logcomment("Drives @drv_own belongs to $FILER_C"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); foreach my $disk_t (@drv_own) { if ( $disk_t =~ /P/ ) { ( $disk_t = $disk_t ) =~ s/P\d//g; } ( $disk_t = $disk_t ) =~ s/$FILER_C\://g; my $cmd = "$stsb_status get_status $disk_t"; my $output = $Api_Set_Obj->execute_command( 'command' => $cmd ); } } else { logcomment("Not a SATA testbed. Does not record device information after firmware download"); } } my ( $first_time, $end_time ); my $time_now = time; my $time_end = $time_now + $test_wait_time; my ( $sec, $min, $hrs ) = gmtime($test_wait_time); my $loop = 1; my $firm_delay_time; if ( $total_drv_count_s2 <= 48 ) { $firm_delay_time = 60; } else { $firm_delay_time = 30; } logcomment("DEBUG : Firmware delay time / sleep time in between ping pong step : $firm_delay_time "); while ( $time_now < $time_end ) { #STEP 5 nvme_eye_catcher( Test => $Test, string => "STEP 5 of 12: Download older (LOW) verion FW to target drives : $loop loop" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Downgrade firmware for filer :$FILER_C using firmware files from DISKFW/LOW directory"); my $low_dir = "$Home/NDATE/FIRMWARE/DISKFW/LOW"; my @drv_dwlded = nvme_disk_fw_update_low_or_high( Node => $Node, FW_DIR_LOC => $low_dir, FORCE_UPDTE => "yes" ); logcomment("Filer $FILER_C firmware downgrade compelted on @drv_dwlded drives"); logcomment("Check drive firmware "); nvme_get_all_disk( node_present => [$Node] ); } # STEP 6 nvme_eye_catcher( Test => $Test, string => "STEP 6 of 12: Record all device information after firmware download" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Get Drive owned by $FILER_C"); my @drv_own = nvme_get_disk_owned( Node => $Node ); logcomment("Drives @drv_own belongs to $FILER_C"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); foreach my $disk_t (@drv_own) { if ( $disk_t =~ /P/ ) { ( $disk_t = $disk_t ) =~ s/P\d//g; } ( my $mydisk = $disk_t ) =~ s/$FILER_C\://g; logcomment("Excute scsi inquiry on $mydisk"); my $output = $Api_Set_Obj->execute_command( 'command' => "priv set -q test;scsi inquiry $mydisk" ); } } #STEP 7 nvme_eye_catcher( Test => $Test, string => "STEP 7 of 12: Download newer (HIGH) FW to target drives and look for any download failures" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Downgrade firmware for filer :$FILER_C using firmware files from DISKFW/HIGHdirectory"); my $low_dir = "$Home/NDATE/FIRMWARE/DISKFW/HIGH"; my @drv_dwlded = nvme_disk_fw_update_low_or_high( Node => $Node, FW_DIR_LOC => $low_dir, FORCE_UPDTE => "yes" ); logcomment("Filer $FILER_C firmware downgrade compelted on @drv_dwlded drives"); logcomment("Check drive firmware "); nvme_get_all_disk( node_present => [$Node] ); } #STEP 8 nvme_eye_catcher( Test => $Test, string => "STEP 8 of 12: Record all device information after firmware download" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Get Drive owned by $FILER_C"); my @drv_own = nvme_get_disk_owned( Node => $Node ); logcomment("Drives @drv_own belongs to $FILER_C"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); foreach my $disk_t (@drv_own) { if ( $disk_t =~ /P/ ) { ( $disk_t = $disk_t ) =~ s/P\d//g; } ( my $mydisk = $disk_t ) =~ s/$FILER_C\://g; logcomment("Excute scsi inquiry on $mydisk"); my $output = $Api_Set_Obj->execute_command( 'command' => "priv set -q test;scsi inquiry $mydisk" ); } } #STEP 9 nvme_eye_catcher( Test => $Test, string => "STEP 9 of 12: Repeat step 5 to 8 for 12 hours" ); logcomment("User selected -w=$TEST_WAIT_TIME so the looping time will be $hrs HOURS $min MINUTES and $sec SECONDS"); logcomment( "Start time for looping is " . scalar( localtime() ) ); $time_now = time; my $time_left = $time_end - $time_now; my ( $lsec, $lmin, $lhrs ) = gmtime($time_left); if ( $time_left > 0 ) { logcomment("Remaining time for looping is $lhrs HOURS $lmin MINUTES and $lsec SECONDS"); } $loop++; sleep($firm_delay_time); logcomment("Waiting time is $firm_delay_time seconds per firmware download"); } #End of While #STEP 10 nvme_eye_catcher( Test => $Test, string => "STEP 10 of 12:Record HDD FRU information for all HDD FRU in the system" ); foreach my $Node (@Nodes) { my $stsb_status; my $FILER_C = $Node->name(); if ( $SATA_SETUP eq 'Y' ) { logcomment("Checking for stsb/acorn command is available "); $stsb_status = get_filer_check( Node => $Node ); chomp($stsb_status); logcomment("DEBUG :$stsb_status"); my @drv_own = nvme_get_disk_owned( Node => $Node ); logcomment("Drives @drv_own belongs to $FILER_C"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); foreach my $disk_t (@drv_own) { if ( $disk_t =~ /P/ ) { ( $disk_t = $disk_t ) =~ s/P\d//g; } ( $disk_t = $disk_t ) =~ s/$FILER_C\://g; my $cmd = "$stsb_status get_status $disk_t"; my $output = $Api_Set_Obj->execute_command( 'command' => $cmd ); } } else { logcomment("Not a SATA testbed. Does not record device information after firmware download"); } } #STEP 11 nvme_eye_catcher( Test => $Test, string => "STEP 11 of 12: Check to see if there is any unusual increase in the drive error logs" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $scsi_result_s11{$FILER_C} = scsi_disk_errors( Node => $Node ); logcomment("Record the log sense data from each drive completed : $FILER_C"); } logcomment("Compare log sense data"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $before = $scsi_result_s3{$FILER_C}; my $after = $scsi_result_s11{$FILER_C}; my $value = compare_scsi_disk( scsi_disk_error_before => $before, scsi_disk_error_after => $after ); if ( $value == 0 ) { logcomment("The output of log sense data is identical to step 3"); } else { $test_status = 1; logcomment("**FAIL**: THE OUTPUT OF LOG SENSE DATA IS DIFFERENT FROM STEP 3 - STEP : 11"); return logresult( "INFO", msg => 1 ); } } #STEP 12 : nvme_eye_catcher( Test => $Test, string => "STEP 12 of 12: Disable disk fw debug level" ); logcomment("Reset back disk fw debug level "); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=0" ); } logresult( "INFO", msg => $test_status ); } ##################################################################### # Cleanup - Post Process ##################################################################### sub cleanup() { $Test->nlog(" $filer_names - Clean up and post test process"); @Nodes = node_new_obj( node_array => [@Nodes] ); my $node_ref = \@Nodes; nvme_post_test( node_present => $node_ref, Test => $Test, change_state_to => "MAINT", filer_mode => $Mode ); ########################################################################################### ## Delete/Reset test specific options and values ########################################################################################### foreach my $Node (@Nodes) { my $FILER_C = $Node->name; my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); my $Host = host($FILER_C); logcomment("disable autogiveback "); my $prompts_answers = [ 'Do you want to disable auto-giveback?' => 'y', 'Do you want to continue' => 'y' ]; $Api_Set_Obj->execute_raw_command( command => "storage failover modify -node * -auto-giveback false", 'connectrec-match_table' => $prompts_answers ); logcomment("Disable background firmwar update option"); $Api_Set_Obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable off" ); logcomment("Stop firmware update command"); try { my $prompts = [ '.*Are you sure you want to.*' => 'y' , '.*stop firmware update qualification.*' =>'y']; $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk_fw_update -S\"", 'connectrec-match_table' => $prompts); logcomment("Firmware Qualification download stopped "); } catch NACL::APISet::Exceptions::ResponseException with { logcomment("FW Download stop test Failed."); }; logcomment("$FILER_C : Turn off the raid options"); $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.enable off" ); $Api_Set_Obj->execute_raw_command( command => "run local options raid.media_scrub.spares.enable off" ); $Api_Set_Obj->execute_raw_command( command => "run local options raid.scrub.enable off" ); } return $TCD::PASS; }