#!/usr/software/bin/perl # $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/BR/NADQ02_ERT_Maint.thpl#17 $ # ## @summary Mode ## ERT Maintenance Mode tests # ## ## @description ## This mode of operation is intended to validate the hardware robustness by using simple ## read/write commands and data patterns for the first 32 days of the total 42 day test. ## No file system is present on the disks and the tool of choice is disk_qual. This script ## execute command for a specific day. ## ## @Test Mode ## Maintenance mode ## ## @Test bed setup ## SATA :Dual Path ## SAS :Dual Path, ## ## @usage ## The test can be run independently or with other tests as part of an STEST. ## ## @step in days ## The test will execute commands mentioned below: ## initial. Set disqual to stop on error ## Day 0 Sequential Verify Test. Print logsense ## Day 1-3 Single block random read, queue depth of 1 - No verify. Run this test for 4 hours. ## Mixed Test, Sector interval [0, end] group of 30 ops, 67% random read (4K) - No verify. ## Will check for uncorrectable errors. ## Day 4-6 Mixed Test, sector interval [0, end] group of 50 ops, 50% random read (4K) - No verify. ## Will check for uncorrectable errors ## Day 7-9 Mixed Test, sector interval [0, end] group of 100 ops, 100% random read (4K)- No verify. ## Will check for uncorrectable errors ## Day 10 Sequential Verify test. Print Logsense ## Day 11-12 Mixed Test, sector interval [0, end] group of 100 ops, 10% random read (4K)- No verify. ## Will check for uncorrectable errors ## Day 13-14 Mixed Test, sector interval [0, OD] group of 200 ops, 10% random read (4K)- No verify. ## Will check for uncorrectable errors ## Day 15-16 Mixed Test, sector interval [0, OD] group of 200 ops, 10% random read (64K) ## Day 17 Sequential Verify test. ## Day 18 Mixed Test, sector interval [Mid, ID] group of 100 ops, 30% random read (256K) ## Day 19-20 Butterfly Wr/Rd, sector interval [MD] noverify (32K) ## Day 21-22 Random Wr/Rd, sector interval [OD] noverify (0K) ## Day 23-24 Ripple Wr/Rd, sector interval [ID] noverify (24K) ## Day 25 Sequential Verify test. ## Day 26-27 Multi-Stream Sequential Read, sector interval [0, OD] test (4K) ## Day 28-29 Multi-Stream Sequential Read, sector interval [0, OD] test (64K) ## Day 30 FC AND SAS ONLY: Seq. Write_And_Verify test, sector interval [0, OD] test (64K) ## SATA ONLY: Seq. Write test, sector interval [0, OD] test (64K) ## Day 31 SATA ONLY: Seq. Verify test, sector interval [0, OD] test (64K) ## Day 32 Write Same test. Compile Write Same completion time average, sigma for entire ## population by capacity and interface type ## ## @param FILER - required for cluster set-up ## - optional for Dual path set-up Name of filer to be used ## @param FILERB - required for cluster set-up ## - optional for Dual path set-up Name of the partner node ## @param TEST_SETUP - optional Default value set to 'FC' ## @param FILER_CONN - optional Type of connection, default is set to 'console' for this test ## @param Mode - optional Filer mode maint/normal, default set to 'maint' for this test ## @param FILER_PROMPT - optional prompt of the filer ## @param LOGDIR - optional This is required to generate 'END-LOG'. Default set to main log directory. ## @param EOE - optional 'default' is set to default value ## @param FILER_TYPE - Required to set DataONTAP version DataONTAP version is either IC or BR ## @param BOOT_MODE - optional Boot mode for filer , default set to '5' for this test ## @param BOOT_TYPE - optional Boot type, default set to 'A' (automatic) for this test ## ## @status Automated ## @auther arjun.lavu@netapp.com ## @burt 770226,808718 ## @change YY/MM/DD from auther: Description of the change ## @change khosur : Modified for Burt ## ## ############################################################################# ### Library functions ########################################## # Compiler directives. use strict; use warnings; # MODULES imports use Storage::Common_Lib; #User defined functions sub _array_diff(\@\@); sub _execute_ert_cmd($); sub _trim($); sub _IOPS_data($$$); sub _update_disk_list($$); sub _send_message($$); # MODULES use Storage::Sasadmin qw(sasadmin_dev_stats sasadmin_expander_map sasadmin_inject); use Storage::SASUtils qw(disable_dbg_msgs); # Global parameters use vars qw( $FILER $TEST_SETUP $MODE $FILER_PROMPT $LOGDIR $EOE $TEST_WAIT_TIME $BOOT_MODE $BOOT_TYPE $SSD $EMAIL $MAIL_TO $MAIL_FROM $ERT_TEST_ID $FILER_TYPE ); my $params = NATE::ParamSet->new( global => 1 ); $FILER = $params->get( 'FILER', default => '' ); $MODE = $params->get( "MODE", default => 'maint' ); $FILER_PROMPT = $params->get( "FILER_PROMPT", default => '\*>' ); $LOGDIR = $params->get( 'LOGDIR', default => '' ); $TEST_SETUP = $params->get( 'TEST_SETUP', default => 'FC' ); $TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' ); $ERT_TEST_ID = $params->get( "ERT_TEST_ID", default => '' ); $EOE = $params->get( 'EOE', default => '3' ); $BOOT_MODE = $params->get( 'BOOT_MODE', default => '5' ); $BOOT_TYPE = $params->get( 'BOOT_TYPE', default => 'A' ); $SSD = $params->get( 'SSD', default => 'no' ); $EMAIL = $params->get( 'EMAIL', default => 'y' ); $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); $FILER_TYPE = $params->get( 'FILER_TYPE', default => 'filer type' ); #Cmode and common variables my $Test; my $Mode; my $Filer_names; my @FILER_ARR; my @Nodes; my @API_ARRAY; my @Nodes_all; my %nodes_filer; my %FILER_API; my %Fil_disk; my %disk_qual_12_s12; my %disk_qual_12_s18; my %disk_qual_12_s7; my %disk_qual_12_s22; my %map_s6; my %sysconfig_v_10; my %Fc_link_s_5; my %state_s4; my %map_s12; my %state_s13; my %Fc_link_s_14; my %disk_qual_12_s5; my @total_disks; my %total_disks_step_3; my @total_disks_step_11; my %sysconfig_v_2; my $disk_list_11; my $disk_max_sup = 40; my %STSB_FOR_EACH_DISKS_PRE = (); my $stsb_pre; my $stsb_post; my $STSB_FOR_EACH_DISKS_POST; my $noips_flag = 0; # my $IOPS_datafile; my $test_status = 0; my ( $email_subject, $email_body ); my $TC_name = "NADQ02_ERT_Maint"; my $log_file_dq = $LOGDIR . "/" . $TC_name . "_" . $ERT_TEST_ID . ".log "; my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/; logcomment("Main log directory is :: $actual_log_dir"); _trim($ERT_TEST_ID); ##Burt 532027 my $test_wait_time = 1800; if ( $TEST_WAIT_TIME == 1 ) { $test_wait_time = ( $test_wait_time / 10 ); } elsif ( $TEST_WAIT_TIME == 2 ) { $test_wait_time = ( $test_wait_time / 2 ); } elsif ( $TEST_WAIT_TIME == 4 ) { $test_wait_time = ( $test_wait_time * 0.95 ); } my $expected_comp_time = ( ( 46 * ($test_wait_time) / 3600 ) ) + 1; my $test_time_step = ( 24 * $test_wait_time ); my $test_time_step_hr = ( $test_time_step / 3600 ); #hex to decimal conversion my %h2d = ( 0 => "0", 1 => "1", 2 => "2", 3 => "3", 4 => "4", 5 => "5", 6 => "6", 7 => "7", 8 => "8", 9 => "9", a => "10", b => "11", c => "12", d => "13", e => "14", f => "15", ); ######################################### # Testcase available for execution ######################################### my @Testcases = ( ERT_Maint => "ERT_Maint" ); &main(); sub main { # Debug break point $DB::single = 2; # Create Test Case Driver object $Test = new TCD( -testcases => [@Testcases] ); if ( $Test->error ) { $Test->nlog( $Test->errmsg ); $test_status = 1; $email_subject = "$TC_name : Test FAILED: FAIL"; $email_body = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n"; send_message( mail_subject => $email_subject, mail_body => $email_body, MAIL_FROM => $MAIL_FROM, MAIL_TO => $MAIL_TO, EOE => $EOE, EMAIL => $EMAIL ); return $TCD::FAIL; } # Performs method callbacks $Test->run_test(); if ( $Test->error ) { $Test->log( $Test->errmsg ); return $TCD::FAIL; } exit(0); } ## end sub main ########## INIT ################################################### # This init subroutine will initialise the filer. #################################################################### sub init() { $Test->description("########## Finding Nodes ######### Initialising the Filer"); $Filer_names = $Test->get_param("FILER"); $Test->nlog("Filers present are $Filer_names"); my $Home = $ENV{HOME}; my @Filers; if ( $FILER =~ /\,/ ) { @Filers = split( /\,/, $FILER ); } else { push( @Filers, $FILER ); } my @Nodes_temp = NACL::C::Node->find(); foreach my $Node (@Nodes_temp) { my $FILER_C = $Node->name(); $nodes_filer{$FILER_C} = $Node; } @Nodes = values(%nodes_filer); @Nodes_all = @Nodes; logcomment("Changing state to maintenance MODE"); my @subtests; foreach my $node (@Nodes) { my $filer_name = $node->name(); push( @subtests, subtest( \&boot_filer_in_req_state, -runid, "boot_filer_in_MAINT_" . "$filer_name", -bg, "--", Test => $Test, filer_name => $filer_name, change_state_to => "MAINT", ) ); } Subtest::wait_finish( subtest => [@subtests] ); logcomment("Checking for execution mode"); $Mode = $Nodes[0]->mode(); version_test( node_present => \@Nodes, tc_name => $TC_name ); return $TCD::PASS; } ## end sub init() ########## SETUP ################################################### # setup automatically called before executing tests #################################################################### sub setup() { $Test->description("Setup the environment for the test exectuion "); logcomment("Mode of filer $Filer_names : $Mode"); my $node_ref = \@Nodes; ##################################################################### # Pre test proces : call for pre_n_post test process ##################################################################### execute_pre_test( node_present => $node_ref, Test => $Test, change_state_to => "MAINT", filer_mode => $Mode, LOGDIR => $LOGDIR, BOOT_TYPE => $BOOT_TYPE, test_setup => $TEST_SETUP, ); ############################################################### #Creating API object and getting disk information ############################################################### foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment(" Creating API object for filer - $FILER_C"); my $Host = host($FILER_C); my $API_obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); push( @API_ARRAY, $API_obj ); $FILER_API{$FILER_C} = $API_obj; my @temp_disk_info = get_disk_information( API_Obj => $API_obj, Test => $Test, Filer => $FILER_C ); @temp_disk_info = sort(@temp_disk_info); $Fil_disk{$FILER_C} = [@temp_disk_info]; logcomment( "Filer - $FILER_C : drives which are attached to filer are" . scalar @temp_disk_info . " and the drives are @temp_disk_info" ); my $disks_file = "$LOGDIR/../../../" . $FILER_C . "_disks_list"; # if (!(-e $disks_file)) { # my $disks_file = "$LOGDIR/../../../" . $FILER_C . "_disks_list"; open( H_LOGFILE, ">$disks_file" ) or die "write:Failed opening $disks_file \n"; foreach my $line (@temp_disk_info) { if ( ( $line =~ /(^\s+$)|(^\.+$)/g ) ) { next; } print( H_LOGFILE "$line \n" ); } close H_LOGFILE; # } ## end if (!(-e $disks_file)) disable_dbg_msgs( host => $Host, filermode => "maint" ); my $disk_status = check_ert_config( Node => $Node, disk_list => [@temp_disk_info] ); if ( $disk_status == 0 ) { my $Home = $ENV{HOME}; $test_status = 1; logcomment("**FATAL**: $FILER_C : Minimum and maximum drives number are not satisfied. \nPlease modify the file $Home/NDATE/ERT_CONFIG if the data is wrong"); logresult( 'FATAL', "$FILER_C : Minimum and maximum drives number are not satisfied. \nPlease modify the file $Home/NDATE/ERT_CONFIG if the data is wrong\n\n" ); } } return $TCD::PASS; } ########## TEST 1 ################################################### #NADQ02_ERT_Maint ##################################################################### sub ERT_Maint { my @subtests; logcomment( "Number of nodes are " . scalar @Nodes . " and the filers are $Filer_names" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&ERT_Maint_sub, -runid, "ERT_Maint_$FILER_C", -bg, "--", $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); my $return = status_return( subtest_ref => [@subtests] ); if ( $return == 0 ) { return $TCD::PASS; } else { return $TCD::FAIL; } } ####################### ##Subtest_create ####################### sub ERT_Maint_sub { my @Nodes; push( @Nodes, shift(@_) ); $Test->description("Starting ERT Maintenance"); eye_catcher( Test => $Test, string => "Starting execution of test steps. Total steps to be executed as part of the script are: 27" ); my $cmd = 'set test'; if ( ( $ERT_TEST_ID == 4 ) || ( $ERT_TEST_ID == 21 ) || ( $ERT_TEST_ID == 22 ) || ( $ERT_TEST_ID == 7 ) || ( $ERT_TEST_ID == 1 ) || ( $ERT_TEST_ID == 12 ) || ( $ERT_TEST_ID == 18 ) ) { $noips_flag = 1; } ###------------------------------------------------------------ ## Step 1 - Execute ERT command ## Pass/Fail criteria: N/A #------------------------------------------------------------ my @subtests; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER : Enable all storage debug messages"); my $Host = host($FILER_C); disable_dbg_msgs( host => $Host, filermode => "$MODE", enable => "1" ); eye_catcher( Test => $Test, string => "STEP 1 of 1 : $FILER_C : Execute ERT command.", ); push( @subtests, subtest( \&_execute_ert_cmd, -runid, "execute_ert_cmd_" . "$FILER_C", -bg, "--", $Node ) ); } logcomment("Subtest @subtests"); Subtest::wait_finish( subtest => [@subtests] ); ######################################## ##Display the temperature of all drives ##before disk_qual stop ######################################## my %Temp; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $API_Obj = $FILER_API{$FILER_C}; my @disks = @{ $Fil_disk{$FILER_C} }; logcomment("Display the temperature of all drives before disk_qual stop"); foreach my $disk (@disks) { my $logsense_data = $API_Obj->execute_raw_command( command => "scsi logsense -p 0xd $disk" ); foreach ( split( "\n", $logsense_data ) ) { if ( $_ =~ /^0000:/ ) { my @arr = split( /\s+/, $_ ); my $temp1 = substr( $arr[10], 2, 1 ); my $temp2 = substr( $arr[10], 3, 1 ); $temp1 =~ s/(.)/$h2d{lc $1}/g; $temp2 =~ s/(.)/$h2d{lc $1}/g; my $decimal = ( $temp1 * 16 ) + $temp2; $Temp{$disk} = $decimal; last; } } } sleep(5); logcomment("$FILER_C : The temperature values for all drives is as follows:"); foreach my $dsk ( keys %Temp ) { logcomment("Disk: $dsk Temperature : $Temp{$dsk} deg C\n"); } logcomment("$FILER_C : Listing of temperature values completed "); } #------------------------------------------------------------ ## STEP 2 - Wait for all disks to report results and stop testing ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); eye_catcher( Test => $Test, string => "STEP 2 of 2 : $FILER_C : Wait for all disks to report results and stop testing.", ); my $timeout = 1800; my $API_obj = $FILER_API{$FILER_C}; my $status = abort_disk_qual( $Node, $API_obj, $timeout ); if ( $status == 1 ) { logcomment("$FILER_C : ERT Day $ERT_TEST_ID disk_qual process is stopped"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: ERT Day $ERT_TEST_ID Disk qual failed to stop - STEP : 2"); } { my $IOPS_datafile; # Calling the subroutine to capture the IOPS Statistics if ( $ERT_TEST_ID == 2 ) { # $IOPS_datafile = "$actual_log_dir/NADQ02_ERT_Maint_Day_$ERT_TEST_ID"."_Random_Read"."_"."$FILER_C"."\.IOPS_datafile"; # logcomment("IOPS Statistics for day : $ERT_TEST_ID will be captured in : $IOPS_datafile"); # _IOPS_data($Node,"NADQ02_ERT_Maint_Day_$ERT_TEST_ID"."_Random_Read",$IOPS_datafile); # # }elsif($ERT_TEST_ID == 2){ $IOPS_datafile = "$actual_log_dir/NADQ02_ERT_Maint_Day_$ERT_TEST_ID" . "_Mixed_Test" . "_" . "$FILER_C" . "\.IOPS_datafile"; logcomment("IOPS Statistics for day : $ERT_TEST_ID will be captured in : $IOPS_datafile"); _IOPS_data( $Node, "NADQ02_ERT_Maint_Day_$ERT_TEST_ID" . "_Mixed_Test", $IOPS_datafile ); } else { # Calling the subroutine to capture the IOPS Statistics if ( $noips_flag == 0 ) { $IOPS_datafile = "$actual_log_dir/NADQ02_ERT_Maint_Day_$ERT_TEST_ID" . "_" . "$FILER_C" . "\.IOPS_datafile"; logcomment("IOPS Statistics for day : $ERT_TEST_ID will be captured in : $IOPS_datafile"); _IOPS_data( $Node, "NADQ02_ERT_Maint_Day_$ERT_TEST_ID", $IOPS_datafile ); } } } disk_qual_drive_assign_maint( Node_ref => [$Node], disk_ref => $Fil_disk{$FILER_C} ); } ########################################################################################### ## Test log/results ########################################################################################### logcomment("Test $TC_name is completed."); logresult( "INFO", msg => $test_status ); } ##################################################################### # Cleanup - Post Process ##################################################################### sub cleanup() { logcomment(" $Filer_names - Clean up and post test process"); $Test->description("Preforming Post test process - Check filer state, Check disk, check root volume,halt hammer, halt scrub, filer config"); my $node_ref = \@Nodes; ########################################################################################### ## Post Test process - Category : "post_test" ########################################################################################## execute_post_test( filer_mode => $Mode, node_present => [@Nodes], Test => $Test, change_state_to => "MAINT", LOGDIR => $LOGDIR, test_setup => $TEST_SETUP, BOOT_TYPE => $BOOT_TYPE, ); return $TCD::PASS; } sub _execute_ert_cmd($) { my $Node = shift; my $FILER_C = $Node->name(); my $API_obj = $FILER_API{$FILER_C}; my $cmd_des = ""; my @actual_disks; my $disks_file = "$LOGDIR/../../../../../" . $FILER_C . "_disks_list"; my $timeout = 1800; if ( -f $disks_file ) { # logcomment("Inside disk_list"); @actual_disks = (); open( H_LOGFILE, "<$disks_file " ) or die ">>>>> read: Failed opening $disks_file <<<<<"; foreach my $line () { chomp($line); _trim($line); push @actual_disks, $line; } close H_LOGFILE; } else { logcomment("$FILER_C :: Could not find the file $disks_file"); logresult( 'FATAL', "Could not find the file $disks_file" ); } ## end if (-e $disks_file) logcomment("Executing disk_qual on disks: @actual_disks \n"); logcomment("$FILER_C : $TC_name for day $ERT_TEST_ID : Set disqual to stop on error "); $API_obj->execute_raw_command( command => "disk_qual setparams ContinueOnError 0" ); if ( $ERT_TEST_ID == 0 ) { $cmd_des = "Sequential Verify Test"; logcomment("$FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des"); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 18 -f 0 -l end -p 1 -d @disk_list_q"); $API_obj->execute_raw_command( command => "disk_qual start -t 18 -f fs_start -p 1 -d @disk_list_q" ); } ## end while (@actual_disks_t) _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 1 ) { $cmd_des = "Get SAS/SATA Phy error count initial values"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des"); my @actual_disks_t1 = @actual_disks; my @actual_dsk_use; if ( $actual_disks_t1[0] =~ /L\d+$/ ) { foreach my $dsk (@actual_disks_t1) { if ( $dsk =~ /L1$/ ) { push( @actual_dsk_use, $dsk ); } } } else { @actual_dsk_use = @actual_disks_t1; } while (@actual_dsk_use) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_dsk_use; $loop_no++; last if ( !(@actual_dsk_use) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $stsb_pre = STSB_Get_Status_Results( Node => $Node, disk_list => \@disk_list_q ); # $STSB_FOR_EACH_DISKS_PRE = STSB_Get_Status_Results(Node => $Node , disk_list => \@disk_list_q); } my $stsb_file = "$actual_log_dir/stsb_get_status_file_" . $FILER_C; open( H_LOGFILE, ">$stsb_file " ) or die ">>>>> read: Failed opening $stsb_file <<<<<"; %STSB_FOR_EACH_DISKS_PRE = %{$stsb_pre}; foreach my $stsb ( keys %STSB_FOR_EACH_DISKS_PRE ) { foreach my $var1 ( keys %{ $STSB_FOR_EACH_DISKS_PRE{$stsb} } ) { foreach my $var2 ( keys %{ $STSB_FOR_EACH_DISKS_PRE{$stsb}{$var1} } ) { my @values = @{ $STSB_FOR_EACH_DISKS_PRE{$stsb}{$var1}{$var2} }; my $stsb_data = "$stsb=>$var1=>$var2=@values\n"; print H_LOGFILE $stsb_data; } } } close(H_LOGFILE); $cmd_des = "Record the log sense data from each drive"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des "); my @actual_disks_t2 = @actual_disks; while (@actual_disks_t2) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t2; $loop_no++; last if ( !(@actual_disks_t2) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $disk_qual_12_s7{$FILER_C} = run_disk_qual_t12( API_Obj => $API_obj, disk_list_local => \@disk_list_q ); my $diskqual_file = "$actual_log_dir/disk_qual_t12_file_Day1_" . $FILER_C; open( H_LOGFILE, ">$diskqual_file " ) or die ">>>>> read: Failed opening $diskqual_file <<<<<"; foreach my $var ( keys %{ $disk_qual_12_s7{$FILER_C} } ) { foreach my $var1 ( keys %{ $disk_qual_12_s7{$FILER_C}{$var} } ) { my $qualdata = "$var=>$var1=$disk_qual_12_s7{$FILER_C}{$var}{$var1}\n"; print H_LOGFILE $qualdata; } } close(H_LOGFILE); disk_qual_drive_assign_maint( Node_ref => [$Node], disk_ref => \@disk_list_q ); } _wait_to_complete( $Node, 8, $test_wait_time, $cmd_des, $disks_file ); $cmd_des = "Single block random read, queue depth of 1 - No verify"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des for 8 hours "); my @actual_disks_t3 = @actual_disks; while (@actual_disks_t3) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t3; $loop_no++; last if ( !(@actual_disks_t3) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 3 -q 1 -n 1 -L 'forever' -d @disk_list_q -o noverify"); $API_obj->execute_raw_command( command => "disk_qual start -t 3 -q 1 -n 1 -f fs_start -L 'forever' -d @disk_list_q -o noverify" ); } _wait_to_complete( $Node, 16, $test_wait_time, $cmd_des, $disks_file ); logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Wait for all disks to report results and stop testing" . scalar( localtime() ) ); abort_disk_qual( $Node, $API_obj, $timeout ); logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Disk qual process stopped " . scalar( localtime() ) ); my $IOPS_datafile = "$actual_log_dir/NADQ02_ERT_Maint_Day_$ERT_TEST_ID" . "_Random_Read" . "_" . "$FILER_C" . "\.IOPS_datafile"; logcomment("IOPS Statistics for day : $ERT_TEST_ID will be captured in : $IOPS_datafile"); _IOPS_data( $Node, "NADQ02_ERT_Maint_Day_$ERT_TEST_ID" . "_Random_Read", $IOPS_datafile ); sleep 60; if ( -e $disks_file ) { @actual_disks = ''; open( H_LOGFILE, "< $disks_file " ) or die ">>>>> read: Failed opening $disks_file <<<<<"; foreach my $line () { chomp($line); _trim($line); push @actual_disks, $line; } close H_LOGFILE; } ## end if (-e $disks_file) } elsif ( $ERT_TEST_ID == 2 ) { $cmd_des = "Mixed Test, sector interval [0, end] group of 50 ops, 50% random read (4K) - No verify"; $Test->nlog( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $API_obj->execute_raw_command( command => "\013" ); $API_obj->execute_raw_command( command => "disk_qual start -t 16 -f fs_start -d @disk_list_q -o 50 50" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 3 ) { $cmd_des = "Sequential Verify test"; $Test->nlog( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 18 -f 0 -l end -p 1 -d @disk_list_q"); $API_obj->execute_raw_command( command => "disk_qual start -t 18 -f fs_start -p 1 -d @disk_list_q" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 4 ) { $cmd_des = "Write same test for 72 hrs (large block writes)"; $Test->nlog( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 18 -f 0 -l end -p 1 -d @disk_list_q"); $API_obj->execute_raw_command( command => "disk_qual start -t 9 -f fs_start -l fs_end -p 1 -d @disk_list_q -L calc -o noverify" ); } _wait_to_complete( $Node, 144, $test_wait_time, $cmd_des, $disks_file ); logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Wait for all disks to report results and stop testing" . scalar( localtime() ) ); abort_disk_qual( $Node, $API_obj, $timeout ); # Calling the subroutine to capture the IOPS Statistics # my $IOPS_datafile = "$LOGDIR/../../../../NADQ02_ERT_Maint_Day_$ERT_TEST_ID"."_"."$FILER_C"."\.IOPS_datafile"; # logcomment("IOPS Statistics for day : $ERT_TEST_ID will be captured in : $IOPS_datafile"); # _IOPS_data($Node,"NADQ02_ERT_Maint_Day_$ERT_TEST_ID",$IOPS_datafile); } elsif ( $ERT_TEST_ID == 7 ) { $cmd_des = "Perform verify same one pass (large 20480 blocks reads)"; $Test->nlog( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t7 = @actual_disks; while (@actual_disks_t7) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t7; $loop_no++; last if ( !(@actual_disks_t7) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 16 -d @disk_list_q -o 100 100"); $API_obj->execute_raw_command( command => "disk_qual start -t 18 -f 0 -l end -q 8 -n 20480 -d @disk_list_q" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); abort_disk_qual( $Node, $API_obj, $timeout ); my $IOPS_datafile = "$actual_log_dir/NADQ02_ERT_Maint_Day_$ERT_TEST_ID" . "_" . "$FILER_C" . "\.IOPS_datafile"; logcomment("IOPS Statistics for day : $ERT_TEST_ID will be captured in : $IOPS_datafile"); _IOPS_data( $Node, "NADQ02_ERT_Maint_Day_$ERT_TEST_ID", $IOPS_datafile ); $cmd_des = "Record the log sense data from each drive"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des "); my @actual_disks_t8 = @actual_disks; while (@actual_disks_t8) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t8; $loop_no++; last if ( !(@actual_disks_t8) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $disk_qual_12_s5{$FILER_C} = run_disk_qual_t12( API_Obj => $API_obj, disk_list_local => \@disk_list_q ); my $diskqual_file7 = "$actual_log_dir/disk_qual_t12_file_Day7_" . $FILER_C; open( H_LOGFILE, ">$diskqual_file7 " ) or die ">>>>> read: Failed opening $diskqual_file7 <<<<<"; foreach my $var ( keys %{ $disk_qual_12_s5{$FILER_C} } ) { foreach my $var1 ( keys %{ $disk_qual_12_s5{$FILER_C}{$var} } ) { my $qualdata = "$var=>$var1=$disk_qual_12_s5{$FILER_C}{$var}{$var1}\n"; print H_LOGFILE $qualdata; } } close(H_LOGFILE); disk_qual_drive_assign_maint( Node_ref => [$Node], disk_ref => \@disk_list_q ); } _wait_to_complete( $Node, 8, $test_wait_time, $cmd_des, $disks_file ); my $diskqual_file = "$actual_log_dir/disk_qual_t12_file_Day1_" . $FILER_C; my %disk_qual_12_s1; if ( -e $diskqual_file ) { open( H_LOGFILE, "<$diskqual_file" ) or die ">>>>> read: Failed opening $diskqual_file <<<<<"; foreach my $line () { chomp($line); if ( $line =~ /(.*)\=\>(.*)\=(.*)/ig ) { $disk_qual_12_s1{$FILER_C}{$1}{$2} = $3; } } close H_LOGFILE; } logcomment("$FILER_C - Record the log sense data completed"); if ( defined $disk_qual_12_s1{$FILER_C} ) { $cmd_des = "Compare log sense data of drives with ERT Day 1 output"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des "); my $ret = compare_disk_qual_t12( disk_qual_t12_before => $disk_qual_12_s1{$FILER_C}, disk_qual_t12_after => $disk_qual_12_s5{$FILER_C}, nm_error => 'N' ); if ( $ret == 0 ) { # $test_status = 1; logcomment("**WARNING**: $FILER_C: $TC_name for day $ERT_TEST_ID : Logsense data is different from captured in Day 1 and Day 7"); } else { logcomment("$FILER_C - Logsense data is matching from captured in Day 1 and Day 7"); } } else { logcomment("**WARNING**: $FILER_C : Comparison of logsense data cannot be done as ERT Test is not executed for Day 1"); } $cmd_des = "Get SAS/SATA Phy error count initial values"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des"); my @actual_disks_t9 = @actual_disks; my @actual_dsk_use; if ( $actual_disks_t9[0] =~ /L\d+$/ ) { foreach my $dsk (@actual_disks_t9) { if ( $dsk =~ /L1$/ ) { push( @actual_dsk_use, $dsk ); } } } else { @actual_dsk_use = @actual_disks_t9; } while (@actual_dsk_use) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_dsk_use; $loop_no++; last if ( !(@actual_dsk_use) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); # $stsb_post = STSB_Get_Status_Results(Node => $Node , disk_list => \@disk_list_q); $STSB_FOR_EACH_DISKS_POST = STSB_Get_Status_Results( Node => $Node, disk_list => \@disk_list_q ); } my $stsb_file = "$actual_log_dir/stsb_get_status_file_" . $FILER_C; my %STSB_FOR_EACH_DISKS_PRE1; if ( -e $stsb_file ) { open( H_LOGFILE, "<$stsb_file" ) or die ">>>>> read: Failed opening data.txt <<<<<"; foreach my $line () { chomp($line); if ( $line =~ /(.*)\=\>(.*)\=>(.*)=(.*)/ig ) { my @arr = split( " ", $4 ); $STSB_FOR_EACH_DISKS_PRE1{$1}{$2}{$3} = [@arr]; } } close H_LOGFILE; } my $STSB_after = \%STSB_FOR_EACH_DISKS_PRE1; if ( ( defined $STSB_after ) ) { $cmd_des = "Comparing the STSB get_status values of disks"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des "); if ( compare_stsb_get_status_disk_values( Node => $Node, before => $STSB_after, after => $STSB_FOR_EACH_DISKS_POST ) == 1 ) { logcomment("**WARNING** : some of the stsb get_status values are not matching from ERT Day 1 to ERT Day $ERT_TEST_ID"); } else { logcomment("stsb get_status values are matching from ERT Day 1 to ERT Day $ERT_TEST_ID"); } } else { logcomment("**WARNING** : Comparison of stsb get_status data cannot be done as ERT Test is not executed for ERT Day 1"); } } elsif ( $ERT_TEST_ID == 8 ) { $cmd_des = "Mixed Test, sector interval [0, end] group of 100 ops, 10% random read (4K)- No verify"; $Test->nlog( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 16 -d @disk_list_q -o 100 100"); $API_obj->execute_raw_command( command => "disk_qual start -t 16 -f fs_start -d @disk_list_q -q 6 -o 100 10" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 9 ) { $cmd_des = "Mixed Test, sector interval [0, OD] group of 200 ops, 10% random read (4K)- No verify"; $Test->nlog( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 16 -d @disk_list_q -o 100 100"); $API_obj->execute_raw_command( command => "disk_qual start -t 16 -f fs_start -l 150000000 -d @disk_list_q -o 200 10" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 10 ) { $cmd_des = "Mixed Test, sector interval [0, OD] group of 200 ops, 10% random read (64K)"; $Test->nlog( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 18 -f 0 -l end -p 1 -d @disk_list_q"); $API_obj->execute_raw_command( command => "disk_qual start -t 16 -f fs_start -l 150000000 -d @disk_list_q -n 128 -o 200 10" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 11 ) { logcomment("$FILER_C : $TC_name for day $ERT_TEST_ID : ATI Margin Test (off track write margin) - Write on small LBA band at OD & random seek away, repeat for 12 hours"); # logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des"); my @actual_disks_t = @actual_disks; my @MaxLBAval; my $TotalLBA; my $MaxLBA; my $SLBA; my $ELBA; eye_catcher( Test => $Test, string => "STEP 1 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Check the total LBAs of drives under test. Get MaxLBA.", ); # logcomment("Check the total LBAs of drives under test. Get MaxLBA."); foreach my $disk (@actual_disks_t) { my $output = $API_obj->execute_raw_command( command => "scsi capacity $disk" ); if ( $output =~ /(\d+) blocks/ ) { $TotalLBA = $1; } logcomment("The total LBA of disk $disk is $TotalLBA"); $MaxLBA = $TotalLBA - 1; logcomment("The max LBA of disk $disk is $MaxLBA"); push( @MaxLBAval, $MaxLBA ); # $Mlba_disk_val_18{$disk} = $MaxLBA; } my $maxlbavalue = ( sort { $b <=> $a } @MaxLBAval )[0]; logcomment("Max LBA value to be used for calculating Start LBA(SLBA) and End LBA(ELBA) : $maxlbavalue"); eye_catcher( Test => $Test, string => "STEP 2 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Calculate Start LBA(SLBA) for Write Zone. Set Start LBA write range at 10% OD: SLBA = MaxLBA*10%", ); # logcomment("Calculate Start LBA(SLBA) for Write Zone. Set Start LBA write range at 10% OD: SLBA = MaxLBA*10%"); $SLBA = int( $MaxLBA * .10 ); logcomment("Start LBA for Write Zone : $SLBA"); eye_catcher( Test => $Test, string => "STEP 3 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Calculate End LBA(ELBA) for Write Zone .Set End LBA for write range for 1% of disk capacity ELBA = SLBA + (MaxLBA*1%)", ); # logcomment("Calculate End LBA(ELBA) for Write Zone .Set End LBA for write range for 1% of disk capacity ELBA = SLBA + (MaxLBA*1%)"); $ELBA = int( $SLBA + ( $MaxLBA * .01 ) ); logcomment("End LBA for Write Zone : $ELBA"); eye_catcher( Test => $Test, string => "STEP 4 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Choose a random LBA number SLBA1 in the range of SLBA($SLBA) to ELBA-256($ELBA-256) to use in Step 5 and Choose a random LBA number SLBA2 in the range of 0 to MaxLBA -8($MaxLBA -8) to use in step 6", ); logcomment("Choose a random LBA number SLBA1 in the range of SLBA($SLBA) to ELBA-256($ELBA-256) to use in Step 6"); my $range = ( ( $ELBA - 256 ) - $SLBA ); my $min = $SLBA; my $SLBA1 = int( rand($range) ) + $min; logcomment("Random LBA number : SLBA1 : $SLBA1"); logcomment("Choose a random LBA number SLBA2 in the range of 0 to MaxLBA -8($MaxLBA -8) to use in step 7"); my $range2 = $maxlbavalue - 8; my $SLBA2 = int( rand($range) ); logcomment("Random LBA number : SLBA2 : $SLBA2"); eye_catcher( Test => $Test, string => "STEP 5 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Random 128K write in write LBA range of SLBA($SLBA) --> ELBA($ELBA) with starting LBA SLBA1($SLBA1)", ); $cmd_des = "Random 128K write in write LBA range of SLBA($SLBA) --> ELBA($ELBA) with starting LBA SLBA1($SLBA1)"; my @disks_step5 = @actual_disks_t; while (@disks_step5) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @disks_step5; $loop_no++; last if ( !(@disks_step5) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $API_obj->execute_raw_command( command => "disk_qual start -t 7 -L 1 -f $SLBA1 -q 8 -n 256 -d @disk_list_q -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); eye_catcher( Test => $Test, string => "STEP 6 : $FILER_C : $ERT_TEST_ID : Random 4K read somewhere on the disk (full seek length range) with Starting LBA SLBA2($SLBA2)", ); $cmd_des = "Random 4K read somewhere on the disk (full seek length range) with Starting LBA SLBA2($SLBA2)"; my @disks_step6 = @actual_disks_t; while (@disks_step6) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @disks_step6; $loop_no++; last if ( !(@disks_step6) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $API_obj->execute_raw_command( command => "disk_qual start -t 3 -L 1 -f $SLBA2 -q 9 -n 8 -d @disk_list_q -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); eye_catcher( Test => $Test, string => "STEP 7 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Repeat steps 4 to 6 for 12 hours", ); my $step = 1; my $time_left = 0; for ( $step = 1 ; $step <= 24 ; $step++ ) { logcomment("ITERATION NUMBER : $step"); eye_catcher( Test => $Test, string => "STEP 4 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Choose a random LBA number SLBA1 in the range of SLBA($SLBA) to ELBA-256($ELBA-256) to use in Step 5 and Choose a random LBA number SLBA2($SLBA2) in the range of 0 to MaxLBA -8($MaxLBA -8) to use in step 6", ); logcomment("Choose a random LBA number SLBA1 in the range of SLBA($SLBA) to ELBA-256($ELBA-256) to use in Step 6"); my $range = ( ( $ELBA - 256 ) - $SLBA ); my $min = $SLBA; my $SLBA1 = int( rand($range) ) + $min; logcomment("Random LBA number : SLBA1 : $SLBA1"); logcomment("Choose a random LBA number SLBA2($SLBA2) in the range of 0 to MaxLBA -8($MaxLBA -8) to use in step 7"); my $range2 = $maxlbavalue - 8; my $SLBA2 = int( rand($range) ); logcomment("Random LBA number : SLBA2 : $SLBA2"); eye_catcher( Test => $Test, string => "STEP 5 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Random 128K write in write LBA range of SLBA($SLBA) --> ELBA($ELBA) with starting LBA SLBA1($SLBA1)", ); $cmd_des = "Random 128K write in write LBA range of SLBA($SLBA) --> ELBA($ELBA) with starting LBA SLBA1($SLBA1)"; my @disk_rept_step5 = @actual_disks_t; while (@disk_rept_step5) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @disk_rept_step5; $loop_no++; last if ( !(@disk_rept_step5) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $API_obj->execute_raw_command( command => "disk_qual start -t 7 -L 1 -f $SLBA1 -q 8 -n 256 -d @disk_list_q -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); eye_catcher( Test => $Test, string => "STEP 6 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Random 4K read somewhere on the disk (full seek length range) with Starting LBA SLBA2($SLBA2)", ); $cmd_des = "Random 4K read somewhere on the disk (full seek length range) with Starting LBA SLBA2($SLBA2)"; my @disk_rept_step6 = @actual_disks_t; while (@disk_rept_step6) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @disk_rept_step6; $loop_no++; last if ( !(@disk_rept_step6) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $API_obj->execute_raw_command( command => "disk_qual start -t 3 -L 1 -f $SLBA2 -q 9 -n 8 -d @disk_list_q -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); sleep $test_wait_time; $time_left = ( $test_time_step_hr - ( ( $step * $test_wait_time ) / 3600 ) ); logcomment("Repeating Steps 4 to 6 . Test time left $time_left Hr \n Total time $test_time_step_hr Hrs"); } eye_catcher( Test => $Test, string => "STEP 8 : $FILER_C : ERT_TEST_ID - $ERT_TEST_ID : Perform sequential read scan for the entire range of SLBA-1000($SLBA-1000) --> ELBA+1000($ELBA+1000)", ); $cmd_des = "Perform sequential read scan for the entire range of SLBA-1000($SLBA – 1000) --> ELBA+1000($ELBA + 1000)"; my $slba10 = $SLBA - 1000; my $elba10 = $ELBA + 1000; my @disk_step8 = @actual_disks_t; while (@disk_step8) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @disk_step8; $loop_no++; last if ( !(@disk_step8) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $API_obj->execute_raw_command( command => "disk_qual start -t 1 -f $slba10 -l $elba10 -q 8 -n 256 -d @disk_list_q -o noverify -L calc" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 12 ) { $cmd_des = "Sequential Verify test"; $Test->nlog("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des "); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 18 -f 0 -l end -p 1 -d @disk_list_q"); $API_obj->execute_raw_command( command => "disk_qual start -t 18 -f fs_start -p 1 -d @disk_list_q" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); abort_disk_qual( $Node, $API_obj, $timeout ); my $IOPS_datafile = "$actual_log_dir/NADQ02_ERT_Maint_Day_$ERT_TEST_ID" . "_" . "$FILER_C" . "\.IOPS_datafile"; logcomment("IOPS Statistics for day : $ERT_TEST_ID will be captured in : $IOPS_datafile"); _IOPS_data( $Node, "NADQ02_ERT_Maint_Day_$ERT_TEST_ID", $IOPS_datafile ); $cmd_des = "Record the log sense data from each drive"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des "); my @actual_disks_t12 = @actual_disks; while (@actual_disks_t12) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t12; $loop_no++; last if ( !(@actual_disks_t12) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $disk_qual_12_s12{$FILER_C} = run_disk_qual_t12( API_Obj => $API_obj, disk_list_local => \@disk_list_q ); my $diskqual_file12 = "$actual_log_dir/disk_qual_t12_file_Day12_" . $FILER_C; open( H_LOGFILE, ">$diskqual_file12 " ) or die ">>>>> read: Failed opening $diskqual_file12 <<<<<"; foreach my $var ( keys %{ $disk_qual_12_s12{$FILER_C} } ) { foreach my $var1 ( keys %{ $disk_qual_12_s12{$FILER_C}{$var} } ) { my $qualdata = "$var=>$var1=$disk_qual_12_s12{$FILER_C}{$var}{$var1}\n"; print H_LOGFILE $qualdata; } } close(H_LOGFILE); disk_qual_drive_assign_maint( Node_ref => [$Node], disk_ref => \@disk_list_q ); } _wait_to_complete( $Node, 8, $test_wait_time, $cmd_des, $disks_file ); my $diskqual_file7 = "$actual_log_dir/disk_qual_t12_file_Day7_" . $FILER_C; my %disk_qual_12_s7_12; if ( -e $diskqual_file7 ) { open( H_LOGFILE, "<$diskqual_file7" ) or die ">>>>> read: Failed opening $diskqual_file7 <<<<<"; foreach my $line () { chomp($line); if ( $line =~ /(.*)\=\>(.*)\=(.*)/ig ) { $disk_qual_12_s7_12{$FILER_C}{$1}{$2} = $3; } } close H_LOGFILE; } logcomment("$FILER_C - Record the log sense data completed"); if ( defined $disk_qual_12_s7_12{$FILER_C} ) { logcomment("$FILER_C - Compare log sense data with ERT Day 7 output"); my $ret = compare_disk_qual_t12( disk_qual_t12_before => $disk_qual_12_s7_12{$FILER_C}, disk_qual_t12_after => $disk_qual_12_s12{$FILER_C}, nm_error => 'N' ); if ( $ret == 0 ) { # $test_status = 1; logcomment("**WARNING**: $FILER_C: $TC_name for day $ERT_TEST_ID : Logsense data is different from captured in Day 7 and Day 12"); } else { logcomment("$FILER_C - Logsense data is matching from captured in Day 7 and Day 12"); } } else { logcomment("**WARNING**: $FILER_C - Comparison of logsense data cannot be done as ERT Test is not executed for Day 7"); } } elsif ( $ERT_TEST_ID == 13 ) { $cmd_des = "Mixed Test, sector interval [Mid, ID] group of 100 ops, 30% random read (256K)"; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 16 -d @disk_list_q -n 512 -f id -o 100 30"); $API_obj->execute_raw_command( command => "disk_qual start -t 16 -d @disk_list_q -n 512 -f id -o 100 30" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 14 ) { $cmd_des = "Butterfly Wr, sector interval [MD] noverify (32K)"; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $API_obj->execute_raw_command( command => "disk_qual start -t 6 -d @disk_list_q -f md -n 64 -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 15 ) { $cmd_des = "Random Wr, sector interval [OD] noverify (0K)"; $Test->nlog( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); # $API_obj->execute_raw_command( command =>"disk_qual start -t 7 -d @disk_list_q -f od -n 1 -o noverify"); $API_obj->execute_raw_command( command => "disk_qual start -t 7 -d @disk_list_q -f fs_start -l 150000000 -n 1 -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 16 ) { $cmd_des = "Ripple Wr, sector interval [ID] noverify (24K)"; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $API_obj->execute_raw_command( command => "disk_qual start -t 8 -d @disk_list_q -f id -n 48 -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 17 ) { $cmd_des = "Butterfly Rd with large IO - Pre-Production Test to catch Max Connection time or Loop tendency related issue."; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); # $API_obj->execute_raw_command( command =>"disk_qual start -t 2 -f 0 -l end -p 1 -q 6 -n 512 -d @disk_list_q -o noverify"); $API_obj->execute_raw_command( command => "disk_qual start -t 2 -f fs_start -p 1 -q 6 -n 512 -d @disk_list_q -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 18 ) { $cmd_des = "Sequential Verify test"; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 18 -f 0 -l end -p 1 -d @disk_list_q"); $API_obj->execute_raw_command( command => "disk_qual start -t 18 -f fs_start -p 1 -d @disk_list_q" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); abort_disk_qual( $Node, $API_obj, $timeout ); my $IOPS_datafile = "$actual_log_dir/NADQ02_ERT_Maint_Day_$ERT_TEST_ID" . "_" . "$FILER_C" . "\.IOPS_datafile"; logcomment("IOPS Statistics for day : $ERT_TEST_ID will be captured in : $IOPS_datafile"); _IOPS_data( $Node, "NADQ02_ERT_Maint_Day_$ERT_TEST_ID", $IOPS_datafile ); $cmd_des = "Record the log sense data from each drive"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des "); my @actual_disks_t18 = @actual_disks; while (@actual_disks_t18) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t18; $loop_no++; last if ( !(@actual_disks_t18) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $disk_qual_12_s18{$FILER_C} = run_disk_qual_t12( API_Obj => $API_obj, disk_list_local => \@disk_list_q ); my $diskqual_file18 = "$actual_log_dir/disk_qual_t12_file_Day18_" . $FILER_C; open( H_LOGFILE, ">$diskqual_file18 " ) or die ">>>>> read: Failed opening $diskqual_file18 <<<<<"; foreach my $var ( keys %{ $disk_qual_12_s18{$FILER_C} } ) { foreach my $var1 ( keys %{ $disk_qual_12_s18{$FILER_C}{$var} } ) { my $qualdata = "$var=>$var1=$disk_qual_12_s18{$FILER_C}{$var}{$var1}\n"; print H_LOGFILE $qualdata; } } close(H_LOGFILE); disk_qual_drive_assign_maint( Node_ref => [$Node], disk_ref => \@disk_list_q ); } _wait_to_complete( $Node, 8, $test_wait_time, $cmd_des, $disks_file ); my $diskqual_file12 = "$actual_log_dir/disk_qual_t12_file_Day12_" . $FILER_C; my %disk_qual_12_s7_18; if ( -e $diskqual_file12 ) { open( H_LOGFILE, "<$diskqual_file12" ) or die ">>>>> read: Failed opening $diskqual_file12 <<<<<"; foreach my $line () { chomp($line); if ( $line =~ /(.*)\=\>(.*)\=(.*)/ig ) { $disk_qual_12_s7_18{$FILER_C}{$1}{$2} = $3; } } close H_LOGFILE; } logcomment("$FILER_C - Record the log sense data completed"); if ( defined $disk_qual_12_s7_18{$FILER_C} ) { logcomment("$FILER_C - Compare log sense data with ERT Day 12 output"); my $ret = compare_disk_qual_t12( disk_qual_t12_before => $disk_qual_12_s7_18{$FILER_C}, disk_qual_t12_after => $disk_qual_12_s18{$FILER_C}, nm_error => 'N' ); if ( $ret == 0 ) { # $test_status = 1; logcomment("**WARNING**: $FILER_C: $TC_name for day $ERT_TEST_ID : Logsense data is different from captured in Day 12 and Day 18"); } else { logcomment("$FILER_C - Logsense data is matching from captured in Day 12 and Day 18"); } } else { logcomment("**WARNING**: $FILER_C - Comparison of logsense data cannot be done as ERT Test is not executed for Day 12"); } } elsif ( $ERT_TEST_ID == 19 ) { $cmd_des = "Multi-Stream Sequential Read, sector interval [0, OD] test (4K)"; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); #$API_obj->execute_raw_command( command =>"setflag disk_qual_time_limit 10000"); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 24 -d @disk_list_q -o noverify"); $API_obj->execute_raw_command( command => "setflag disk_qual_time_limit 10000; disk_qual start -t 24 -f fs_start -d @disk_list_q -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 20 ) { $cmd_des = "Multi-Stream Sequential Read, sector interval [0, OD] test (64K)"; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); #$API_obj->execute_raw_command( command =>"setflag disk_qual_time_limit 10000"); my @actual_disks_t = @actual_disks; while (@actual_disks_t) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t; $loop_no++; last if ( !(@actual_disks_t) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 24 -q 6 -n 128 -d @disk_list_q -o noverify"); $API_obj->execute_raw_command( command => "setflag disk_qual_time_limit 10000; disk_qual start -t 24 -q 6 -n 128 -f fs_start -d @disk_list_q -o noverify" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); } elsif ( $ERT_TEST_ID == 21 ) { $cmd_des = "Seq. Write test, sector interval [0, OD] test (64K)"; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t21 = @actual_disks; while (@actual_disks_t21) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t21; $loop_no++; last if ( !(@actual_disks_t21) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 9 -f od -n 128 -d @disk_list_q"); $API_obj->execute_raw_command( command => "disk_qual start -t 9 -f fs_start -l 150000000 -n 128 -d @disk_list_q" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Wait for all disks to report results and stop testing" . scalar( localtime() ) ); abort_disk_qual( $Node, $API_obj, $timeout ); } elsif ( $ERT_TEST_ID == 22 ) { $cmd_des = "Seq. Verify test, sector interval [0, OD] test (64K)"; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t22 = @actual_disks; while (@actual_disks_t22) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t22; $loop_no++; last if ( !(@actual_disks_t22) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 18 -f od -n 128 -d @disk_list_q"); $API_obj->execute_raw_command( command => "disk_qual start -t 18 -f fs_start -l 150000000 -n 128 -d @disk_list_q" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); abort_disk_qual( $Node, $API_obj, $timeout ); my $IOPS_datafile = "$actual_log_dir/NADQ02_ERT_Maint_Day_$ERT_TEST_ID" . "_" . "$FILER_C" . "\.IOPS_datafile"; logcomment("IOPS Statistics for day : $ERT_TEST_ID will be captured in : $IOPS_datafile"); _IOPS_data( $Node, "NADQ02_ERT_Maint_Day_$ERT_TEST_ID", $IOPS_datafile ); $cmd_des = "Record the log sense data from each drive"; logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des "); my @actual_disks_t23 = @actual_disks; while (@actual_disks_t23) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t23; $loop_no++; last if ( !(@actual_disks_t23) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); $disk_qual_12_s22{$FILER_C} = run_disk_qual_t12( API_Obj => $API_obj, disk_list_local => \@disk_list_q ); disk_qual_drive_assign_maint( Node_ref => [$Node], disk_ref => \@disk_list_q ); } _wait_to_complete( $Node, 8, $test_wait_time, $cmd_des, $disks_file ); my $diskqual_file18 = "$actual_log_dir/disk_qual_t12_file_Day18_" . $FILER_C; my %disk_qual_12_s7_22; if ( -e $diskqual_file18 ) { open( H_LOGFILE, "<$diskqual_file18" ) or die ">>>>> read: Failed opening $diskqual_file18 <<<<<"; foreach my $line () { chomp($line); if ( $line =~ /(.*)\=\>(.*)\=(.*)/ig ) { $disk_qual_12_s7_22{$FILER_C}{$1}{$2} = $3; } } close H_LOGFILE; } logcomment("$FILER_C - Record the log sense data completed"); if ( defined $disk_qual_12_s7_22{$FILER_C} ) { logcomment("$FILER_C - Compare log sense data with ERT Day 18 output"); my $ret = compare_disk_qual_t12( disk_qual_t12_before => $disk_qual_12_s7_22{$FILER_C}, disk_qual_t12_after => $disk_qual_12_s22{$FILER_C}, nm_error => 'N' ); if ( $ret == 0 ) { # $test_status = 1; logcomment("**WARNING**: $FILER_C: $TC_name for day $ERT_TEST_ID : Logsense data is different from captured in Day 18 and Day 22"); } else { logcomment("$FILER_C - Logsense data is matching from captured in Day 18 and Day 22"); } } else { logcomment("**WARNING**: $FILER_C - Comparison of logsense data cannot be done as ERT Test is not executed for Day 18"); } } elsif ( $ERT_TEST_ID == 23 ) { $cmd_des = "Write Same test. Compile Write Same completion time average, sigma for entire population by capacity and interface type"; logcomment( "FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : Started executing $cmd_des " . scalar( localtime() ) ); my @actual_disks_t22 = @actual_disks; while (@actual_disks_t22) { my $loop_no = 0; my @disk_list_q = (); while ( $loop_no < $disk_max_sup ) { push @disk_list_q, pop @actual_disks_t22; $loop_no++; last if ( !(@actual_disks_t22) ); } my $timeout_val = 600 + ( scalar(@disk_list_q) * 300 ); $API_obj->set_timeout( "connectrec-timeout" => $timeout_val ); #$API_obj->execute_raw_command( command =>"disk_qual start -t 18 -f od -n 128 -d @disk_list_q"); $API_obj->execute_raw_command( command => "disk_qual start -t 18 -f fs_start -l 150000000 -n 128 -d @disk_list_q" ); } _wait_to_complete( $Node, 46, $test_wait_time, $cmd_des, $disks_file ); abort_disk_qual( $Node, $API_obj, $timeout ); } logresult( "INFO", msg => $test_status ); } sub _wait_to_complete($$$$$) { my ( $Node, $steps, $wait_step, $cmd_des, $disks_file ) = @_; my $FILER_C = $Node->name(); my $step = 1; my $time_left = 0; my $max_time = ( $steps * $wait_step ) / 3600; my @disk_array = @{ $Fil_disk{$FILER_C} }; my $Api = $FILER_API{$FILER_C}; for ( $step = 1 ; $step <= $steps ; $step++ ) { sleep $wait_step; $time_left = ( $max_time - ( ( $step * $wait_step ) / 3600 ) ); my $status = $Api->execute_raw_command( command => "disk_qual status" ); if ( $status !~ /idle|running/ig ) { sleep 30; ##Disk Qual status not captured successfully. Retrying the command again $status = $Api->execute_raw_command( command => "disk_qual status" ); if ( $status !~ /idle|running/ig ) { logcomment("**FATAL**: $FILER_C : Disk Qual status not captured."); return logresult( "INFO", msg => 1 ); } } if ( $status =~ /idle/ ) { my @fail_disk_array = (); my $disk_fail_status = 0; foreach my $disk (@disk_array) { my $return = $Api->execute_raw_command( command => "disk_qual report -d $disk" ); if ( $return !~ /Passed/i ) { $test_status = 1; logcomment("*FAIL* : $FILER_C : Disk qual stopped on error on disk : $disk"); $disk_fail_status = 1; #Adding the disk name in the failed disk array push( @fail_disk_array, "$disk.\n" ); } } if ( $disk_fail_status == 1 ) { $test_status = 1; logcomment("**FAIL**: $FILER_C: $TC_name for day $ERT_TEST_ID : Disk qual stopped on error."); } else { my $executiontime = ( $step * $wait_step ) / 3600; my ( $ss, $mm, $hh ) = gmtime( $executiontime * 3600 ); logcomment("$FILER_C : $TC_name for day $ERT_TEST_ID : Disk qual completed after $hh HRS $mm MINS $ss SECS"); } last; } else { logcomment("$FILER_C : Disk qual is running"); } my ( $s, $m, $h ) = gmtime( $time_left * 3600 ); my ( $s1, $m1, $h1 ) = gmtime( $max_time * 3600 ); logcomment("FILER- $FILER_C : $TC_name for day $ERT_TEST_ID : $cmd_des : Time left - $h HRS $m MINS $s SECS and Total time - $h1 HRS $m1 MINS $s1 SECS"); if ( $wait_step > 400 ) { my $Transit_obj = NACL::Transit->new( name => $FILER_C ); #my $filer_state = $Transit_obj->get_state(); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /FIRMWARE/ ) { logcomment("**FATAL**: $FILER_C : Terminating rest of the ERT tests as Filer PANIC"); return logresult( "INFO", msg => 1 ); } ## end if ($filer_state eq "FIRMWARE") } ## end if ($wait_step > 400) } ## end for ($step = 1 ; $step ... if ( $steps >= 144 ) { my $return_stats = $Api->execute_raw_command( command => "disk_qual status" ); if ( $return_stats =~ /running/i ) { $Api->execute_raw_command( command => "disk_qual stop -d all" ); } } my $my_log = "$LOGDIR/execute_ert_cmd_" . "$FILER_C" . ".log"; # logcomment("Subtest logfile :: $my_log"); open( H_LOGFILE, "< $my_log " ) || print ">>>>> read: Failed opening $my_log <<<<<"; foreach my $line () { if ( $line =~ /\[(\S+)\]:\s*disk_qual test failed/ ) { my $disk_found = 0; my $failed_disk = $1; my $flag = 0; #Capturing the dirve information for comparison my $drive_id = substr( $failed_disk, 3 ); logcomment("Drive ID selected for searching in the failed drives file is : $drive_id"); #Checking if the disk name is present in disk_list file #Set the value of $flag = 1, if the name exists open( DISK_LIST, "< $disks_file " ) or die ">>>>> read: Failed opening $disks_file <<<<<"; foreach () { chomp($_); if ( $_ =~ /$drive_id/ ) { $flag = 1; last; } } close(DISK_LIST); if ( $flag == 1 ) { $test_status = 1; $Api->execute_raw_command( command => "led_on $failed_disk" ); logcomment("$FILER_C : $TC_name for day $ERT_TEST_ID : Disk qual failed for drive : $failed_disk and LED is on for this drive"); _update_disk_list( $failed_disk, $disks_file ); logcomment("**FAIL**: $FILER_C: Disk qual failed for drive : $failed_disk "); } } } ## end foreach my $line () close H_LOGFILE; } sub _update_disk_list($$) { my ( $disk, $disks_file ) = @_; _trim($disk); open( H_LOGFILE, "< $disks_file " ) or die ">>>>> read: Failed opening $disks_file <<<<<"; my @temp_disks; foreach () { chomp($_); if ( !( $_ =~ /$disk/ ) ) { push @temp_disks, $_; } } close H_LOGFILE; system("rm $disks_file"); open( H_LOGFILE, ">$disks_file" ) or die "write:Failed opening $disks_file \n"; foreach my $line (@temp_disks) { print( H_LOGFILE "$line \n" ); } close H_LOGFILE; } ## end sub _update_disk_list($$) sub _IOPS_data($$$) { my ( $node, $testname, $IOPS_datafile ) = @_; my $FILER_C = $node->name(); my $Api_obj = $FILER_API{$FILER_C}; my ( $report_data, $count, $report_data_new ) = 0; my @disks = (); my $linecount = 0; logcomment("Filer - $FILER_C : Recording the IOPS values"); open( IOPSDATA, ">>$IOPS_datafile" ) or die "Failed opening $IOPS_datafile \n"; print IOPSDATA "\nTEST_ID : $testname\n\n"; print IOPSDATA "\nDISK : IOPS Value\n"; print IOPSDATA "========================\n"; my @disk_array = @{ $Fil_disk{$FILER_C} }; foreach my $disk (@disk_array) { try { $report_data = $Api_obj->execute_raw_command( 'command' => "disk_qual report -d $disk" ); } catch NACL::APISet::Exceptions::TimeoutException with { logcomment("Found timeout exception, So increasing the API set timeout and retrying again"); my $timeout = $Api_obj->get_timeout(); logcomment("Current API time out is $timeout, doubling the timeout"); $timeout = $timeout + 1800; $Api_obj->set_timeout( "connectrec-timeout" => $timeout ); $Api_obj->execute_raw_command( 'command' => "\013" ); sleep 10; $report_data = $Api_obj->execute_raw_command( 'command' => "disk_qual report -d $disk" ); }; if ( $report_data !~ /$disk/ ) { sleep(2); $report_data_new = $Api_obj->execute_raw_command( 'command' => "\013" ); } ##end if ($report_data =~ /$disk/) if ( ( $report_data =~ /\s*(\S+)\s*(IOPS:)\s*(\S+)/ ) || ( $report_data_new =~ /\s*(\S+)\s*(IOPS:)\s*(\S+)/ ) ) { my $datastring = "$1 $3\n"; print IOPSDATA "$datastring"; $linecount++; } else { my $datastring = "[$disk] 0\n"; print IOPSDATA "$datastring"; $count++; push( @disks, $disk ); } ##end if (($report_data =~ /\s*(\S+)\s*(IOPS:)\s*(\S+)/) || ($report_data_new =~ /\s*(\S+)\s*(IOPS:)\s*(\S+)/)) sleep(2); } close(IOPSDATA); if ($count) { logcomment("**WARNING**: $FILER_C - IOPS values are not generated by disk_qual report command, IOPS data is failed to generate for disks: @disks"); } $/ = undef; #reading the entire data at one time, so setting input record separator value to $/=undef; my ( $IOPSmax, $IOPSmin, $IOPSsum, $IOPSavg ) = ( undef, undef, 0, 0 ); #Reading the file to calculate the max and min and sigma of the IOPS data open( IOPSREAD, "$IOPS_datafile" ); my $data = ; my @total_iops = $data =~ /\[\S+\]:\s+(\d+)/g; @total_iops = sort { $a <=> $b } @total_iops; $IOPSmax = $total_iops[-1]; #finding the max IOPS value $IOPSmin = $total_iops[0]; #finding the min IOPS value map { $IOPSsum += $_ } @total_iops; #adding all the IOPS values in an array @total_iops close(IOPSREAD); $/ = "\n"; #resetting the input record separator to default value if ( $linecount > 0 ) { $IOPSavg = $IOPSsum / $linecount; } _trim($IOPSmax); _trim($IOPSmin); _trim($IOPSsum); _trim($IOPSavg); #Printing the final data in the IOPS datafile open( IOPSDATA, ">>$IOPS_datafile" ); print IOPSDATA "\nMaximum of all IOPS Values : $IOPSmax\n"; print IOPSDATA "Minimum of all IOPS Values : $IOPSmin\n"; print IOPSDATA "Sum of all IOPS Values : $IOPSsum\n"; print IOPSDATA "Total drives reporting IOPS values : $linecount\n"; print IOPSDATA "Average of all IOPS Values : $IOPSavg\n\n"; sleep(30); print IOPSDATA "Total drives on the system @{$Fil_disk{$FILER_C}}\n"; close(IOPSDATA); } sub _send_message($$) { my ( $mail_subject, $mail_body ) = @_; $mail_body = $mail_body . "\nLog location :$LOGDIR\n"; my %mail_params = ( HTML => $mail_body, Text => $mail_body, From => $MAIL_FROM, To => $MAIL_TO, Subject => $mail_subject, ); $mail_params{HTML} = "
$mail_params{Text}<\><\\HTML>";
    my $mail = HTML::Mail->new(%mail_params);
    $mail->send() or logresult( "CONFIG", "trouble sending mail" );
} ## end sub _send_message($$)

sub _trim($) {
    $_[0] =~ s/^\s+|\s+$//g if ( defined $_[0] );
}

sub _array_diff(\@\@) {
    my %e = map { $_ => undef } @{ $_[1] };
    return @{ [ ( grep { ( exists $e{$_} ) ? ( delete $e{$_} ) : (1) } @{ $_[0] } ), keys %e ] };
}

sub abort_disk_qual {
    my ( $Node, $API_Obj, $timeout ) = @_;
    my ( $match, @disks );
    my $FILER_C = $Node->name();

    #Checking the status of disk_qual
    $match = $API_Obj->execute_raw_command( 'command' => "disk_qual status" );
    if ( $match =~ /idle/ ) {
        logcomment("$FILER_C : disk qual is idle");
        return 1;
    }
    logcomment("$FILER_C : disk qual is not idle, waiting for the 'disk_qual stop' to complete");
    my $command = "disk_qual stop -d all";

    # $timeout = ((scalar(@disks) * 60 )+ 300);
    $API_Obj->set_timeout( "connectrec-timeout" => 18000 );
    my $dis_qua_stop = $API_Obj->execute_raw_command( 'command' => $command );
    logcomment("Waiting 120 seconds to get first prompt from 'disk_qual stop'");
    if ( $dis_qua_stop =~ /disk_qual not found/ ) {
        logcomment("**WARNING** : Filer : $FILER_C : disk_qual is not supported in the kernel");
        return (0);
    }
    sleep(120);
    $API_Obj->execute_raw_command( 'command' => "\013" );

    #***********************************************
    #setting the time out value
    #***********************************************
    my $kill_time = time + $timeout;
    while ( $kill_time > time ) {
        sleep 10;
        my $status = $API_Obj->execute_raw_command( 'command' => "disk_qual status" );
        if ( $status !~ /idle/ ) {
            sleep 120;    #waiting for 1 min and again checking the while loop condition
        }
        elsif ( $status =~ /disk_qual not found/ ) {
            logcomment("**WARNING** : Filer : $FILER_C : disk_qual is not supported in the kernel");
            return (0);
        }
        elsif ( $status =~ /idle/g ) {
            logcomment("disk_qual is stopped ");
            return (1);
        }
        else {
            if ( $kill_time < time ) {
                sleep 120;    #waiting for 1 min and again checking the while loop condition if time-out has not occurred
            }
            else {
                last;
            }
        }
    }
    ###################################################################################
    ##Checking again the status of disk_qual, if it is not idle return 0; else return 1;
    ###################################################################################
    my $stat = $API_Obj->execute_raw_command( 'command' => "disk_qual status" );
    if ( $stat !~ /idle/g ) {
        return (0);
    }
    else {
        return (1);
    }
}

=head
sub _drive_temp_check {
	my %Temp;
	my ($Node,$API_Obj) = @_;
        my $FILER_C = $Node->name();
        my @disks = @{$Fil_disk{$FILER_C}};
        logcomment("Display the temperature of all drives before disk_qual stop");
        foreach my $disk(@disks){
        	my $logsense_data = $API_Obj->execute_raw_command(command =>"scsi logsense -p 0xd  $disk");
                foreach( split("\n",$logsense_data) ){
                	if($_ =~ /^0000:/){
                        	my @arr = split(/\s+/,$_);
                                my $temp1 =substr($arr[10],2,1);
                                my $temp2 = substr($arr[10],3,1);
                                $temp1 =~s/(.)/$h2d{lc $1}/g;
                                $temp2 =~s/(.)/$h2d{lc $1}/g;
                                my $decimal = ($temp1*16)+$temp2;
                                $Temp{$disk} = $decimal;
                                last;
                         }
                }
        }
        sleep(5);
        logcomment("$FILER_C : The temperature values for all drives is as follows:");
        foreach my $dsk ( keys %Temp ){
        	logcomment("Disk: $dsk  Temperature : $Temp{$dsk} deg C\n");
        }
        logcomment("$FILER_C : Listing of temperature values completed ");
}
=cut