#!/usr/software/bin/perl # $Id: //depot/prod/test/Rsierranevada/storage/hdd/NADQ_SEA/BR/NADQ02_ERT_Fsys.thpl#2 $ # ## @summary Mode ## ERT file system tests # ## ## @description ## The basic process to follow is to maintain the HW configuration of the systems from the Mode 1 testing. ## Reboot the filers and create file systems on the three Configurations as called out in the ## reference tables for systems ERT-1, ERT-2 and ERT-3. Then start a test and let it run overnight. ## Next morning, stop the hammer test and any error injection that is running; and run the verify script. ## Start a new test and continue. The test operator should make sure to add his email address to the ## autosupport.to list so that he can monitor any failures. ## ## @Test Mode ## File System mode ## ## @Test bed setup ## FC :Dual Path, ## SATA :Dual Path ## SAS :Dual Path, ## ## @usage ## The test can be run independently or with other tests as part of an STEST. ## ## @step in days ## The test will execute commands mentioned below: ## ## Day 33 Assign all disks and boot it with option 4a ## Snap reserve 1 sets the reserve to 1% for the vol ## Start disk scrub ## Start hammer (512, 1024,10240, 40960, 65536) ## Execute for 23 hrs ## Stop disk scrub ## Stop hammer ## Take logs ## ## Day 34-36 Start disk scrub ## Start hammer (1024, 20480, 65536) ## Start Device Resets (interval $RESETVALUE seconds) ## Execute for 23 hrs ## Stop disk scrub ## Stop hammer ## Stop error injection ## Take logs # ## Day 37-39 Start disk scrub ## Start hammer (512, 65536) ## Start Adapter Resets (interval $ADAPRESETVALUE seconds) ## Stop disk scrub ## Stop hammer ## Stop error injection ## Take logs # ## ## @param FILER - required for cluster setup ## - optional for Dual path setup Name of filer to be used ## @param FILERB - required for cluster setup ## - optional for Dual path setup Name of the partner node ## @param TEST_SETUP - optional Default value set to 'FC' ## @param FILER_CONN - optional Type of connection, default is set to 'rsh' for this test ## @param MODE - optional Filer mode maint/normal, default set to 'normal' for this test ## @param FILER_PROMPT - optional prompt of the filer ## @param LOGDIR - optional This is required to generate 'END-LOG'. Default set to main log directory. ## @param EOE - optional 'default' is set to default value ## @param FILER_TYPE - Required to set DataONTAP version DataONTAP version is either IC or BR ## @param RESETVALUE - Required for device reset value, default is 120. ## @param ADAPRESETVALUE - Required for adapter reset value, default is 120. ## @param ARMADILLO - This parameter will indicate if an ARMADILLO configuration is under testing (if yes, ARMADILLO=1 else ARMADILLO=2) ## @param BOOT_MODE - optional Boot mode for filer , default set to '1' for this test ## @param BOOT_TYPE - optional Boot type, default set to 'A' (automatic) for this test ## ## @status Automated ## @auther arjunl@netapp.com ## @burt 808727, 808728 ## @change YY/MM/DD from auther: Description of the change ## ## use strict; ############################################################################# ######################################## ### Library functions ########################################## use Storage::Common_Lib; # Global parameters use vars qw( $FILER $TEST_SETUP $MODE $FILER_PROMPT $LOGDIR $EOE $TEST_WAIT_TIME $ERT_VOL $BOOT_MODE $ADAPRESETVALUE $RESETVALUE $BOOT_TYPE $SSD $EMAIL $MAIL_TO $MAIL_FROM $ERT_TEST_ID $FILER_TYPE ); my $params = NATE::ParamSet->new( global => 1 ); $FILER = $params->get( 'FILER', default => '' ); $MODE = $params->get( "MODE", default => 'maint' ); $FILER_PROMPT = $params->get( "FILER_PROMPT", default => '\*>' ); $LOGDIR = $params->get( 'LOGDIR', default => '' ); $TEST_SETUP = $params->get( 'TEST_SETUP', default => 'FC' ); $TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' ); $ERT_TEST_ID = $params->get( "ERT_TEST_ID", default => '' ); $ADAPRESETVALUE = $params->get( "ADAPRESETVALUE", default => '120' ); $RESETVALUE = $params->get( "RESETVALUE", default => '120' ); $ERT_VOL = $params->get( "ERT_VOL", default => '1' ); $EOE = $params->get( 'EOE', default => '3' ); $BOOT_MODE = $params->get( 'BOOT_MODE', default => '5' ); $BOOT_TYPE = $params->get( 'BOOT_TYPE', default => 'A' ); $SSD = $params->get( 'SSD', default => 'no' ); $EMAIL = $params->get( 'EMAIL', default => 'y' ); $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); $FILER_TYPE = $params->get( 'FILER_TYPE', default => 'filer type' ); my $TC_name = "NADQ02_ERT_Fsys"; my @Nodes; my $filer_names; my %Temp; my %non_root_final; my $Mode; my $FILER_STATE; my %total_disks_step_3; my %Fc_link_s_12; my $API_obj; my @disk_list; my %scsi_disk_error_result_s7; my $scsi_disk_error_result_s13; my $vol_flag; my %output_sys_config_2; my $Glist_countfile; my %nodes_filer; my %map_s6; my %map_s10; my %state_s4; my %Fc_link_s_5; my %Fc_link_s_14; my %state_s13; my $email_subject; my $email_body; my %FILER_CHANNEL; #Set test fail parameter to 0 my $test_status = 0; my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/; logcomment("Main log directory is :: $actual_log_dir"); #hex to decimal conversion my %h2d = ( 0 => "0", 1 => "1", 2 => "2", 3 => "3", 4 => "4", 5 => "5", 6 => "6", 7 => "7", 8 => "8", 9 => "9", a => "10", b => "11", c => "12", d => "13", e => "14", f => "15", ); ######################################### ## Pre-test processes ######################################### my @Testcases = ( ERT_Fsys => "ERT_Fsys" ); &main(); sub main { # Debug break point $DB::single = 2; # Create Test Case Driver object $Test = new TCD( -testcases => [@Testcases] ); if ( $Test->error ) { $Test->log( $Test->errmsg ); $test_status = 1; $email_subject = "$TC_name : Test FAILED: FAIL"; $email_body = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n"; send_message( mail_subject => $email_subject, mail_body => $email_body, MAIL_FROM => $MAIL_FROM, MAIL_TO => $MAIL_TO, EOE => $EOE, EMAIL => $EMAIL ); return $TCD::FAIL; } # Performs method callbacks $Test->run_test(); if ( $Test->error ) { $Test->log( $Test->errmsg ); return $TCD::FAIL; } exit(0); } ## end sub main ########## INIT ################################################### # This init subroutine will initialise the filer. #################################################################### sub init() { $Test->description(" Initialising all required variables and FILER connections "); $filer_names = $Test->get_param("FILER"); # Capturing Filer names from the param(Test_Suite) $Test->nlog("Filers present are $filer_names"); ##Check for duplicate node object and push unique node object in Nodes array. my $Home = $ENV{HOME}; my @Filers; if ( $FILER =~ /\,/ ) { @Filers = split( /\,/, $FILER ); } else { push( @Filers, $FILER ); } my @temp_nodes = NACL::C::Node->find(); # Find Nodes/Filers used in the test, Based on FILER param. foreach my $Node (@temp_nodes) { my $FILER_C = $Node->name(); $nodes_filer{$FILER_C} = $Node; } @Nodes = values(%nodes_filer); # Contains Node object used for test execution. logcomment("Checking for execution mode"); $Mode = $Nodes[0]->mode(); # gets mode(7mode/cmode) of the nodes used in test execution. version_test( node_present => \@Nodes, tc_name => $TC_name ); return $TCD::PASS; } ########## SETUP ################################################### # Setup automatically called before executing tests ##################################################################### sub setup() { $Test->description("Setup the environment for the test exectuion "); $Test->nlog("Mode of filer $filer_names : $Mode"); my $node_ref = \@Nodes; ##################################################################### # Pre test proces : call for pre_n_post test process ##################################################################### execute_pre_test( node_present => $node_ref, Test => $Test, change_state_to => "CLI", filer_mode => $Mode, LOGDIR => $LOGDIR, BOOT_TYPE => $BOOT_TYPE, test_setup => $TEST_SETUP, ); return $TCD::PASS; } ########## TEST 1 ################################################### #NADQ02_ERT_Maint ##################################################################### sub ERT_Fsys { my @subtests; logcomment( "Number of nodes are " . scalar @Nodes . " and the filers are $filer_names" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&ERT_Fsys_sub, -runid, "ERT_Fsys_$FILER_C", -bg, "--", $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); my $return = status_return( subtest_ref => [@subtests] ); if ( $return == 0 ) { return $TCD::PASS; } else { return $TCD::FAIL; } } ####################### ##Subtest_create ####################### sub ERT_Fsys_sub { my @Nodes; push( @Nodes, shift(@_) ); $Test->description("Starting ERT Filesystem"); eye_catcher( Test => $Test, string => "Starting execution of test steps. Total steps to be executed as part of the script are: 27" ); # my $required_drives = (($ERT_VOL*3)+4); my $required_drives = ( $ERT_VOL * 3 ); #3 for creating 1 aggr and 2 leaving in spare pool foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); my $disk_filer = disk_show_drive_maint( API_Obj => $Api_Set_Obj, Filer => $FILER_C ); my @disk_filer = @{ $disk_filer->{'driveA'} }; my $driv_cnt = scalar @disk_filer; logcomment( "$FILER_C : Total number of drives present in the system are " . scalar @disk_filer . " and the drives are @disk_filer" ); logcomment("$FILER_C : Required drives for the test are $required_drives"); if ( $required_drives > scalar @disk_filer ) { logcomment("**FATAL**: $FILER_C : Total number of drives present in the system are less than the required drives"); return logresult( "INFO", msg => 1 ); } my @spares = spare_count_cmode( Node => $Node, required_spare => $required_drives ); my $spare_cnt = @spares; # my $spare_cnt = spare_count_cmode(Node=>$Node); logcomment("$FILER_C : Spare count is $spare_cnt"); ############################## #Checking for non root volumes ############################## my @non_root = check_non_root_volumes( Node => $Node ); logcomment( "Total number of Non-root volumes already exist are " . scalar @non_root . " and the volumes are @non_root" ); my $non_root = scalar @non_root; if ( $non_root < $ERT_VOL ) { my $rest = ( $ERT_VOL - $non_root ); logcomment("Required non-root volumes are $rest"); if ( ( $rest * 3 ) > $spare_cnt ) { logcomment( "**FATAL**: $FILER_C : Spare drives are less than the required number i.e " . ( $rest * 3 ) ); return logresult( "INFO", msg => 1 ); } # my $count = ($rest%3); if ( $rest == 1 ) { ## if only one volume required then create volume with all the spares - 2 logcomment("Only one volume is required"); my $drives; # if ( ( $spare_cnt == 5 ) || ( $spare_cnt == 6 ) ) { # $drives = 5; # } else { # $drives = $spare_cnt - 2; # } $drives = $spare_cnt; logcomment("Total number of spare drives passing to create volume are :: $drives"); my @drv_used = splice( @spares, 0, $drives ); create_flex_vol( Node => $Node, #disk_list => \@drv_used, size => '95' ); } else { # for ( my $i = 1 ; $i <= $rest ; $i++ ) { # logcomment("Volume creation count $i"); # my @drv_used = splice( @spares, 0, 3 ); create_flex_vol( Node => $Node, #disk_list => \@drv_used, size => '95' ); logcomment("$FILER_C : Checking the Spare count on the filer"); my @spares1 = spare_count_cmode( Node => $Node ); my $spare_cnt = @spares1; logcomment( "Total Number of spare drives left are " . $spare_cnt ); # } } @non_root = check_non_root_volumes( Node => $Node ); # $non_root_final{$FILER_C} = [@non_root]; logcomment( "Number of volumes created are " . scalar @non_root . " and the volumes are @non_root" ); if ( scalar @non_root < $ERT_VOL ) { logcomment("**FATAL**: $FILER_C : Volumes created are less than the required number of volumes"); return logresult( "INFO", msg => 1 ); } } $non_root_final{$FILER_C} = [@non_root]; logcomment("$FILER_C: Checking for upchannels"); my $sysconfig = sysconfig_v_data( node => $Node ); my @channels_used = @{ $sysconfig->{'upchannels'} }; $FILER_CHANNEL{$FILER_C} = [@channels_used]; } #------------------------------------------------------------ ## Step 1 - Execute ERT command ## Pass/Fail criteria: N/A #------------------------------------------------------------ my @subtests; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); eye_catcher( Test => $Test, string => "STEP 1 of 3 : $FILER_C : Execute ERT command.", ); my @non_r = @{ $non_root_final{$FILER_C} }; my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); foreach (@non_r) { logcomment("Executing snap reserve on volume $_"); $Api_Set_Obj->execute_command( 'command' => "snap reserve $_ 1" ); } push( @subtests, subtest( \&_execute_ert_cmd, -runid, "execute_ert_cmd_" . "$FILER_C", -bg, "--", $Node ) ); # _execute_ert_cmd($Node); } Subtest::wait_finish( subtest => [@subtests] ); ######################################## ##Display the temperature of all drives ##at the end of the test ######################################## foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); my $disk_filer = disk_show_drive_maint( API_Obj => $Api_Set_Obj, Filer => $FILER_C ); my @disk_filer = @{ $disk_filer->{'driveA'} }; logcomment("$FILER_C : Displaying the temperatures of drives"); # my @disk_list = list_of_disk(Node => $Nodes[0]); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); foreach my $disk (@disk_filer) { my $logsense_data = $Api_Set_Obj->execute_command( 'command' => "scsi logsense -p 0xd $disk" ); foreach ( split( /\n/, $logsense_data ) ) { if ( $_ =~ /^0000:/ ) { my @arr = split( /\s+/, $_ ); my $temp1 = substr( $arr[10], 2, 1 ); my $temp2 = substr( $arr[10], 3, 1 ); $temp1 =~ s/(.)/$h2d{lc $1}/g; $temp2 =~ s/(.)/$h2d{lc $1}/g; my $decimal = ( $temp1 * 16 ) + $temp2; $Temp{$disk} = $decimal; last; } } } sleep 10; logcomment(" $FILER_C : The temperature values for all drives is as follows:"); foreach my $dsk ( keys %Temp ) { print "Disk: $dsk Temperature : $Temp{$dsk} deg C\n"; } } sleep(10); ##logcomment("The temperature values for all drives is as follows:"); #------------------------------------------------------------ ## Step 2 - Stop scrub on all the volumes ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); eye_catcher( Test => $Test, string => "STEP 2 of 3 : $FILER_C : Stop scrub on all the volumes.", ); logcomment("FILER : $FILER_C : Stop scrub on all the volumes."); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); logcomment("$FILER_C : Stopping Scrub"); $Api_Set_Obj->execute_command( 'command' => "disk scrub stop" ); logcomment("$FILER_C : Stopping hammer"); ###################################################### ##Aborting the HAMMER ###################################################### my $ret; try { $ret = $Api_Set_Obj->hammer_abort(); #$Test->nlog("Hammer abort". Dumper($ret)); } catch NACL::APISet::Exceptions::ResponseException with { my $exception_object = shift; my $resp = $exception_object->get_response_object(); logcomment( "Caught a " . ref($exception_object) . "!" ); logcomment( "Error is: " . $exception_object->text() ); logcomment( "Raw output was:\n" . $resp->get_raw_output() ); logcomment("Hammer abort has not done properly"); }; if ( ( $ERT_TEST_ID >= 36 ) && ( $ERT_TEST_ID <= 42 ) ) { logcomment("$FILER_C - Stop error injection"); logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Error enjection stopped " . scalar( localtime() ) ); $Api_Set_Obj->execute_command( 'command' => "sasadmin inject disable" ); } sleep 60; if ( $ERT_TEST_ID == 36 ) { logcomment("Collect shm statistics on the drives for day $ERT_TEST_ID"); $Api_Set_Obj->execute_command( 'command' => "disk shm_stats" ); } } #------------------------------------------------------------ ## Step 3 - Creating Glist count file and collect shm statistics on the drives. ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); eye_catcher( Test => $Test, string => "STEP 3 of 3 : $FILER_C : Creating Glist count file and collect shm statistics on the drives.", ); if ( $ERT_TEST_ID == 42 ) { # Calling the subroutine to capture the Glist value $Glist_countfile = "$actual_log_dir/NADQ02_ERT_Fsys_Day_$ERT_TEST_ID" . "_" . "$FILER_C" . ".Glist_countfile"; # $Glist_countfile = "$LOGDIR/../../../NADQ02_ERT_Fsys_Day_$ERT_TEST_ID"."_"."$FILER_C" . ".Glist_countfile"; logcomment("Glist value for day : $ERT_TEST_ID will be captured in : $Glist_countfile"); _Glist_count($Node); } else { logcomment("ERT TEST day is not 42, so not Creating Glist count file"); } } ########################################################################################### ## Test log/results ########################################################################################### logcomment("Test $TC_name is completed."); logresult( "INFO", msg => $test_status ); } ##################################################################### # Cleanup - Post Process ##################################################################### sub cleanup() { $Test->nlog(" $filer_names - Clean up and post test process"); $Test->description("Preforming Post test process - Check filer state, Check disk, check root volume,halt hammer, halt scrub, filer config."); my $node_ref = \@Nodes; ########################################################################################### ## Post Test process - Category : "post_test" ########################################################################################### execute_post_test( filer_mode => $Mode, node_present => [@Nodes], Test => $Test, change_state_to => "CLI", LOGDIR => $LOGDIR, test_setup => $TEST_SETUP, BOOT_TYPE => $BOOT_TYPE, ); return $TCD::PASS; } sub _execute_ert_cmd($) { my $Node = shift; _execute_hammer_scrub($Node); if ( ( $ERT_TEST_ID >= 33 ) && ( $ERT_TEST_ID <= 35 ) ) { _wait_to_complete( $Node, 19 ); } elsif ( ( $ERT_TEST_ID >= 36 ) && ( $ERT_TEST_ID <= 38 ) ) { _dev_reset($Node); _wait_to_complete( $Node, 23 ); } elsif ( ( $ERT_TEST_ID >= 39 ) && ( $ERT_TEST_ID <= 42 ) ) { _adpt_reset($Node); _wait_to_complete( $Node, 23 ); } } sub _execute_hammer_scrub($) { my $Node = shift; my $FILER_C = $Node->name(); my @non_r = @{ $non_root_final{$FILER_C} }; logcomment( "$FILER_C : Non root-volumes present on the filer are " . scalar @non_r . " and the volumes are @non_r" ); logcomment("Executing Hammer Scrub on the volumes"); foreach my $vol (@non_r) { if ( $ERT_VOL == 24 ) { my $return = hammer_start( Node => $Node, file_name => "/vol/$vol/h0", file_size => '65536', Test => $Test ); if ( $return == 0 ) { logcomment("Hammer Started with the file name ' \/vol\/$vol\/h0 ' on the filer $FILER_C"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: Could not Start HAMMER with the file size 65536"); } } elsif ( $ERT_VOL == 12 ) { my %hammer; my @h = qw(h1 h2); my @size = qw(20480 65536); @hammer{@h} = @size; foreach my $key ( keys %hammer ) { my $return = hammer_start( Node => $Node, file_name => "/vol/$vol/$key", file_size => $hammer{$key}, Test => $Test ); if ( $return == 0 ) { logcomment("Hammer with files name ' \/vol\/$vol\/$key ' Started on the filer $FILER_C"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: Could not Start HAMMER with the file size $hammer{$key}"); } } } else { my %hammer; my @h = qw(h1 h2 h3 h4); my @size = qw(512 1024 10240 40960 65536); @hammer{@h} = @size; foreach my $key ( keys %hammer ) { my $return = hammer_start( Node => $Node, file_name => "/vol/$vol/$key", file_size => $hammer{$key}, Test => $Test ); if ( $return == 0 ) { logcomment("Hammer with files name ' \/vol\/$vol\/$key ' Started on the filer $FILER_C"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: Could not Start HAMMER with the file size $hammer{$key}"); } } } } } sub _wait_to_complete($$) { my ( $Node, $time_slot ) = @_; my $test_wait_time = ( $time_slot * 3600 ); logcomment("Default wait time is $time_slot hrs"); if ( $TEST_WAIT_TIME == 1 ) { $test_wait_time = ( $test_wait_time / 10 ); my ( $sec, $min, $hour ) = gmtime($test_wait_time); logcomment("Total wait time is reduced to 10% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 1"); } elsif ( $TEST_WAIT_TIME == 2 ) { $test_wait_time = ( $test_wait_time / 2 ); my ( $sec, $min, $hour ) = gmtime($test_wait_time); logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 2"); } elsif ( $TEST_WAIT_TIME == 4 ) { $test_wait_time = ( $test_wait_time * .95 ); my ( $sec, $min, $hour ) = gmtime($test_wait_time); logcomment("Total wait time is reduced to 95% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 4"); } my $FILER_C = $Node->name(); my ( $SEC, $MIN, $HOUR ) = gmtime($test_wait_time); my $kill_time = time + $test_wait_time; logcomment("Waiting for the hammer to complete, with-in the allocated time"); { my $status; my $sleep_time = ( $test_wait_time / 12 ); my ( $sec, $min, $hour ) = gmtime($sleep_time); # my $FILER_C = $Node->name(); logcomment("Total wait time user has selected is $HOUR HOURS $MIN MINUTES $SEC SECONDS "); while ( $kill_time > time ) { $status = hammer_status( Node => $Node ); if ( $status == 0 ) { logcomment("$FILER_C : Hammer is still running, so waiting for $hour HOURS $min MINUTES $sec SECONDS "); sleep $sleep_time; } else { logcomment("Hammer is completed on the filer $FILER_C"); last; } } } logcomment("WAITING is done, proceeding to next steps"); } sub _dev_reset($) { my $Node = shift; my $FILER_C = $Node->name(); my @channels = @{ $FILER_CHANNEL{$FILER_C} }; my $cmd_status; my $host = host($FILER_C); logcomment("$FILER_C - Inject a device reset on all active port devices @channels every $RESETVALUE seconds"); foreach my $ch (@channels) { $cmd_status = sasadmin_inject_sub( host => $host, adapter_name => $ch, command => "dev_reset", period => $RESETVALUE, filermode => "normal", ); if ( $cmd_status == 1 ) { logcomment( "FILER - $FILER_C : $TC_name for day $ERT_TEST_ID : Inject a device reset on port device $ch every $RESETVALUE seconds started :" . scalar( localtime() ) ); logcomment("Software error injection device_reset successful "); if ( scalar @channels > 1 ) { logcomment("FILER - $FILER_C : Wait for 20 seconds after reset on Adapter to occur before executing the next step"); sleep 20; } } else { _clean_up($Node); logcomment( "FILER - $FILER_C : $TC_name for day $ERT_TEST_ID : Inject a device reset on device every $RESETVALUE seconds on Adapter $ch FAILED, exiting from the test :" . scalar( localtime() ) ); logresult( 'FATAL', "$FILER_C : Inject a device reset on device every $RESETVALUE seconds on Adapter $ch failed " ); } $cmd_status = sasadmin_inject_sub( host => $host, adapter_name => $ch, command => "show", period => $RESETVALUE, filermode => "normal", ); } } sub _adpt_reset($) { my $Node = shift; my $FILER_C = $Node->name(); my @channels = @{ $FILER_CHANNEL{$FILER_C} }; my $cmd_status; my $host = host($FILER_C); logcomment("$FILER_C - Inject a device reset on all active port devices @channels every $RESETVALUE seconds"); foreach my $ch (@channels) { $cmd_status = sasadmin_inject_sub( host => $host, adapter_name => $ch, command => "adapter_reset", period => $ADAPRESETVALUE, filermode => "normal", ); if ( $cmd_status == 1 ) { logcomment( "FILER - $FILER_C : $TC_name for day $ERT_TEST_ID : Inject a device reset on port device $ch every $RESETVALUE seconds started :" . scalar( localtime() ) ); logcomment("Software error injection device_reset successful "); if ( scalar @channels > 1 ) { logcomment("FILER - $FILER_C : Wait for 20 seconds after reset on Adapter to occur before executing the next step"); sleep 20; } } else { _clean_up($Node); logcomment( "FILER - $FILER_C : $TC_name for day $ERT_TEST_ID : Inject a device reset on device every $RESETVALUE seconds on Adapter $ch FAILED, exiting from the test :" . scalar( localtime() ) ); logresult( 'FATAL', "$FILER_C : Inject a device reset on device every $RESETVALUE seconds on Adapter $ch failed " ); } $cmd_status = sasadmin_inject_sub( host => $host, adapter_name => $ch, command => "show", filermode => "normal", period => $ADAPRESETVALUE, ); } } sub _Glist_count($) { my $Node = shift; my $FILER_C = $Node->name(); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); my $disk_filer = disk_show_drive_maint( API_Obj => $Api_Set_Obj, Filer => $FILER_C ); my @disk_list = @{ $disk_filer->{'driveA'} }; # my @disk_list = list_of_disk(Node => $Node); my $FILER_C = $Node->name(); logcomment("$FILER_C : Collect shm statistics on the drives for day $ERT_TEST_ID"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); $Api_Set_Obj->execute_command( 'command' => "disk shm_stats" ); logcomment("$FILER_C : setting the logsense options"); logcomment("$FILER_C : options disk.defect_data_read_all on"); $Api_Set_Obj->execute_command( 'command' => "options disk.log_sense_read_all on" ); logcomment("$FILER_C : options disk.read_defect.interval 5"); $Api_Set_Obj->execute_command( 'command' => "options disk.read_defect.interval 5" ); logcomment("$FILER_C : options disk.log_sense.interval 5"); $Api_Set_Obj->execute_command( 'command' => "options disk.log_sense.interval 5" ); my $stor_show = storage_show_disk_a_data( Node => $Node ); open( GLISTCOUNT, ">$Glist_countfile" ); print GLISTCOUNT "\n\n"; print GLISTCOUNT "\t\t\t\*************** GLIST COUNT for the FILER ***************\n"; print GLISTCOUNT "\n********************************************************\n"; print GLISTCOUNT "FILER: $FILER_C\n"; print GLISTCOUNT "********************************************************\n"; print GLISTCOUNT "\n\t\t\tDISK \tGlist count Value \tPRODUCT ID \t\tSERIAL NUMBER\n"; print GLISTCOUNT "\t\t\t==== \t================= \t========== \t\t=============\n"; foreach (@disk_list) { print GLISTCOUNT "\t\t\t$_ \t$stor_show->{$_}->{'Glist count'} \t\t$stor_show->{$_}->{'Model'} \t$stor_show->{$_}->{'Serial'}\n"; } close(GLISTCOUNT); sleep(600); logcomment("Unsetting the logsense options"); logcomment("options disk.read_defect.interval 30"); $Api_Set_Obj->execute_command( 'command' => "options disk.read_defect.interval 30" ); logcomment("options disk.log_sense.interval 30"); $Api_Set_Obj->execute_command( 'command' => "options disk.log_sense.interval 30" ); } sub _clean_up($) { my $Node = shift; my $FILER_C = $Node->name(); logcomment("Stop any disk scrub process started earlier on filer $FILER_C"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); logcomment("Stopping the disk scrub on the filer $FILER_C "); $Api_Set_Obj->execute_command( 'command' => "disk scrub stop" ); logcomment("Stopping the Hammer on the filer $FILER_C "); $Api_Set_Obj->execute_command( 'command' => "hammer abort" ); logcomment("Stop any error injection if running on $FILER_C"); $Api_Set_Obj->execute_command( 'command' => "sasadmin inject disable" ); } sub _send_message($$) { my ( $mail_subject, $mail_body ) = @_; $mail_body = $mail_body . "\nLog location :$LOGDIR\n"; my %mail_params = ( HTML => $mail_body, Text => $mail_body, From => $MAIL_FROM, To => $MAIL_TO, Subject => $mail_subject, ); $mail_params{HTML} = "
$mail_params{Text}<\><\\HTML>";
    my $mail = HTML::Mail->new(%mail_params);
    $mail->send() or logresult( "CONFIG", "trouble sending mail" );
} ## end sub _send_message($$)