#!/usr/software/bin/perl # $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/CMODE/NADQ02_NVMe_Unmap_Error_Injection.thpl#1 $ # Copyright (c) 2005 Network Appliance, Inc. # All rights reserved # ## @summary Mode ## ATI Background Scan Performance Degradation Test. ## ## @description ## Measure IOPS performance degradation during ATI background scan ## ## ## @hardware configuration ## Applicable to one drive only.(use first disk from disk_list) ## Connect the filer to only one shelf with 14 Drives of EACH CAPACITY IN FAMILY ## ## @Test Mode ## Maintenance mode ## ## @Test bed setup ## SAS :Dual Path ## ## @usage ## The test can be run independently or with other tests as part of STEST. ## ## @dependencies ## None ## ## @steps ## The test will execute steps mentioned below: ## SubTest 1 - ATI Performance Test ## 1. Record kernel version ## 2. List of all drives on all loop(s). Verify the physical count. ## 3. Record SMART attributes or log pages. ## 4. Run the Program script0 on the single selected drive. ## 5. Run the Program script1 on the single selected drive. ## 6. Run the Program script2 on the single selected drive. ## 7. List diskqual script ## 8. Execute diskqual script ## 9. Parse IOPS data for each write loops (100 writes/loop) ## 10. Plot IOPS data vs. testloops.(Will be done by Python script) ## 11. Record SMART attributes or log pages. ## SubTest 2 - Same Track Erasure Tes ## 12. Check the drive under test total LBAs. ## 13. Run disk_qual: 50000 x Write 128 sectors (approx 0% of disk @ OD area) . ## 14. Run disk_qual: Read 256 sectors (approx 0% of disk @ OD area) . ## 15. Run disk_qual: 50000 x Write 128 sectors (approx 1% of disk @ OD area). ## 16. Run disk_qual: Read 256 sectors (approx 1% of disk @ OD area). ## 17. Run disk_qual: 50000 x Write 128 sectors (50% of disk @ MD area). ## 18. Run disk_qual: Read 256 sectors (50% of disk @ MD area). ## 19. Run disk_qual: 50000 x Write 128 sectors (95% of disk @ ID area). ## 20. Run disk_qual: Read 256 sectors (95% of disk @ ID area). ## 21. Record SMART attributes or log pages. ## SubTest 3 - Write Servo Margin Test ## 22. Check the drive under test total LBAs. ## 23. User input starting LBA value for write range. ## 24. User input ending LBA value for write range. ## 25. User input number of test loops for steps 26 & 28. ## 26. Choose a random LBA number ($SLBA1) in the range of $SLBA to ($ELBA-256)to use in step 27. ## 27. Random 128K write in write LBA range of $SLBA --> $ELBA with starting LBA ($SLBA1). ## 28. Random 4K read somewhere on the disk (full seek length range) with Starting LBA ($SLBA2). ## 29. Repeat steps 26 thru 28 for $Loop times. ## 30. Perform sequential read scan for the entire range of ($SLBA - 1000) --> ($ELBA + 1000). ## 31. Full surface sequential verify the disk one time. ## 32. Record SMART attributes or log pages. ## ## @param FILER - optional for cluster setup ## - required for Dual path setup Name of filer to be used ## @param FILERA - required for cluster setup ## - optional for Dual path setup Name of filer to be used ## @param FILERB - required for cluster setup ## - optional for Dual path setup Name of the partner node ## @param TEST_CONFIG - optional Type of the setup, C for cluster, D for dual path (default) ## @param TEST_SETUP - optional Default value set to 'FC' ## @param FILER_CONN - optional Type of connection, default is set to 'console' for this test ## @param MODE - optional Filer mode maint/normal, default set to 'maint' for this test ## @param FILER_PROMPT - optional prompt of the filer ## @param LOGDIR - optional This is required to generate 'END-LOG'. Default set to main log directory. ## @param EOE - optional 'default' is set to default value ## @param BOOT_MODE - optional Boot mode for filer , default set to '5' for this test ## @param BOOT_TYPE - optional Boot type, default set to 'A' (automatic) for this test ## @param SSD - optional Setup is SSD or not, default set to 'no' ## @param EMAIL - optional. This will indicate if the user wants to recieve mails for FAIL scenarios ## @param MAIL_TO - The email ids of the receiving users ## @param MAIL_FROM - The email id from where the above email will be send. ## @param FILER_TYPE - This is used to differentiate between BR and IC filers ## ## @status Automated ## @author kritiu@netapp.com ## @burt 1115436 ## @change YY/MM/DD from auther: Description of the change ## ############################################################################# ######################################### ### Library functions ########################################## use strict; use Storage::Common_Lib; use Storage::NVMe_Common_Lib; # Module imports use TCD; use List::Util qw[min max]; use NACL::APISet; use Data::Dumper; use NACL::Exceptions::NoElementsFound qw(:try); use NACL::Exceptions::InvalidMethod (); use Scalar::Util qw(blessed); use NACL::Transit; use Storage::Sasadmin qw(sasadmin_dev_stats sasadmin_expander_map sasadmin_inject ); use Storage::SASUtils qw(filer_cmd disable_dbg_msgs); ######################################### ### Initialization/declaration ######################################### ##### # Global parameters use vars qw( $FILER $MODE $TEST_CONFIG $TEST_SETUP $FILER_PROMPT $LOGDIR $EOE $TEST_WAIT_TIME $BOOT_MODE $BOOT_TYPE $SSD $EMAIL $MAIL_TO $MAIL_FROM $FILER_TYPE ); my $params = NATE::ParamSet->new( global => 1 ); $LOGDIR = $params->get( 'LOGDIR', default => '' ); $FILER = $params->get( 'FILER', default => '' ); $TEST_CONFIG = $params->get( 'TEST_CONFIG', default => 'D' ); $TEST_SETUP = $params->get( 'TEST_SETUP', default => 'FC' ); $MODE = $params->get( 'MODE', default => 'maint' ); $FILER_PROMPT = $params->get( 'FILER_PROMPT', default => '\*>' ); $EOE = $params->get( 'EOE', default => 'default' ); $TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' ); $BOOT_MODE = $params->get( 'BOOT_MODE', default => '5' ); $BOOT_TYPE = $params->get( 'BOOT_TYPE', default => 'A' ); $SSD = $params->get( 'SSD', default => 'no' ); $EMAIL = $params->get( 'EMAIL', default => 'y' ); $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); $FILER_TYPE = $params->get( 'FILER_TYPE', default => 'filer type' ); ################ # Testcase name ################ my $TC_name = "704_NADQ02_NVMe_Unmap_Error_Injection"; #Common variable declaration my ( $email_subject, $email_body ); my @Nodes = (); my $filer_names; my $Mode; my $test_status = 0; my %nodes_filer = (); my $Filer_state; $EOE = 1; ## Test Specific global variables my $disk_type; my $sanity1 = 0; my $sanity2 = 0; my $sanity3 = 0; my $sanity4 = 0; my $sanity4_verify = 0; my $sanity5 = 0; my $test1 = 0; my $test2 = 0; my $test3 = 0; my $test4 = 0; my $test5 = 0; my $time_taken1; my $time_taken2; my $time_taken3; my $time_taken5; my $inquiry = 0; my @disk_list_step1 = (); my %Fil_disk = (); my %Fil_all_disk = (); my $FILER_C; my @API_ARRAY = (); my %FILER_API = (); my $temp_ref; my $FILER_STATE; my $API_Object; my $disk_max_sup = 40; my @disk_info; my @disk_infoB; my $disk_list_result_3; my $node_ref; my $filers_ref; my $disk; my $TotalLBA; my $MaxLBA; my @all_disk_list_sub2n3 = (); my %filer_del_aggr = (); my %filer_nvol = (); my @vol_created = (); my $disk_maint_own; my %lba_disk_val = (); my %Mlba_disk_val_18 = (); # execute time params my $test_wait_time = 1800; ##equal to 30 minutes logcomment("Default wait time is 30 minutes"); if ( $TEST_WAIT_TIME == 1 ) { $test_wait_time = ( $test_wait_time / 10 ); my ( $sec, $min, $hour ) = gmtime($test_wait_time); logcomment("Total wait time is reduced to 10% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 1"); } elsif ( $TEST_WAIT_TIME == 2 ) { $test_wait_time = ( $test_wait_time / 2 ); my ( $sec, $min, $hour ) = gmtime($test_wait_time); logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 2"); } elsif ( $TEST_WAIT_TIME == 4 ) { $test_wait_time = ( $test_wait_time * .95 ); my ( $sec, $min, $hour ) = gmtime($test_wait_time); logcomment("Total wait time is reduced to 50% i.e $hour HOURS $min MINUTES $sec SECONDS as user selected the option -w = 4"); } ######################################### # Testcase available for execution ######################################### ## Pre-test processes ######################################### my @Testcases = ( Unmap_Error_Injection => "Unmap_Error_Injection" ); &main(); sub main { # Debug break point $DB::single = 2; # Create Test Case Driver object $Test = new TCD( -testcases => [@Testcases] ); if ( $Test->error ) { $Test->log( $Test->errmsg ); $test_status = 1; $email_subject = "$TC_name : Test FAILED: FAIL"; $email_body = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n"; send_message( mail_subject => $email_subject, mail_body => $email_body, MAIL_FROM => $MAIL_FROM, MAIL_TO => $MAIL_TO, EOE => $EOE, EMAIL => $EMAIL ); return $TCD::FAIL; } # Performs method callbacks $Test->run_test(); if ( $Test->error ) { $Test->log( $Test->errmsg ); return $TCD::FAIL; } exit(0); } ## end sub main ########## INIT ################################################### # init automatically called before executing tests #################################################################### sub init() { $Test->description(" Initialising all required variables and FILER connections "); $filer_names = $Test->get_param("FILER"); # Capturing Filer names from the param(Test_Suite) ##Check for duplicate node object and push unique node object in Nodes array. my @temp_nodes = NACL::C::Node->find(); # Find Nodes/Filers used in the test, Based on FILER param. foreach my $Node (@temp_nodes) { my $FILER_C = $Node->name(); $nodes_filer{$FILER_C} = [$Node]; } @Nodes = map { @{$_} } grep { (/\S/) } values(%nodes_filer); # Contains Node objects used for test execution grep { logcomment( "Filers passed to the ntest are " . $_->name() ) } @Nodes; logcomment("Changing state to maintenance MODE"); my @subtests; foreach my $node (@Nodes) { my $filer_name = $node->name(); push( @subtests, subtest( \&boot_filer_in_req_state, -runid, "boot_filer_in_MAINT_" . "$filer_name", -bg, "--", Test => $Test, filer_name => $filer_name, change_state_to => "MAINT", ) ); } Subtest::wait_finish( subtest => [@subtests] ); logcomment("Checking for execution mode"); $Mode = $Nodes[0]->mode(); return $TCD::PASS; } ########## SETUP ################################################### # Setup automatically called before executing tests ##################################################################### sub setup() { $Test->description("Setup the environment for the test exectuion "); logcomment("Mode of filer $filer_names : $Mode"); my $node_ref = \@Nodes; version_test( node_present => $node_ref, tc_name => $TC_name ); ##################################################################### # Pre test proces : call for pre_n_post test process ##################################################################### nvme_pre_test( node_present => $node_ref, Test => $Test, change_state_to => "MAINT", filer_mode => $Mode, ); ############################################################### #Creating API object and getting disk information ############################################################### foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment(" Creating API object for filer - $FILER_C"); my $Host = host($FILER_C); my $API_obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); push( @API_ARRAY, $API_obj ); $FILER_API{$FILER_C} = $API_obj; my @temp_disk_info = nvme_get_disk_owned( Node => $Node ); @temp_disk_info = sort(@temp_disk_info); my @drives_used; if ( !scalar @temp_disk_info ) { $test_status = 1; logcomment("**FATAL** : $FILER_C :NO DRIVES PRESENT ON THE SYSTEM. One drive is required for testing. Exiting from the test"); logresult( 'FATAL', "$FILER_C :NO DRIVES PRESENT ON THE SYSTEM. One drive is required for testing. Exiting from the test" ); } $Fil_all_disk{$FILER_C} = [@temp_disk_info]; push( @drives_used, $temp_disk_info[0] ); $Fil_disk{$FILER_C} = [@drives_used]; } my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/; my $disk_qual_path = $actual_log_dir . "/disk_qual_cnt_$TC_name.log"; logcomment("**DEBUG** : log path : $disk_qual_path"); open( FH, ">>$disk_qual_path" ) || die "Failed to open the write file $!"; close FH; return $TCD::PASS; } ########## TEST 1 ################################################### #NADQ02_Unmap_Error_Injection_ ##################################################################### sub Unmap_Error_Injection { my @subtests; logcomment( "Number of nodes are " . scalar @Nodes . " and the filers are $filer_names" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&Unmap_Error_Injection_sub, -runid, "Unmap_Error_Injection_$FILER_C", -bg, "--", $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); my $return = status_return( subtest_ref => [@subtests] ); if ( $return == 0 ) { return $TCD::PASS; } else { return $TCD::FAIL; } } ####################### ##Subtest_create ####################### sub Unmap_Error_Injection_sub { my @Nodes; my %disk_qual_cnt; push( @Nodes, shift(@_) ); $Test->description("Unmap_Error_Injection"); nvme_eye_catcher( Test => $Test, string => "Starting execution of test steps. Total steps to be executed as part of the script are: 27" ); #------------------------------------------------------------ ## Step 1 - Save disk_list entry output ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 1 of 17 : $FILER_C : Save disk_list entry output.", ); my $API = $FILER_API{$FILER_C}; my $return = nvme_list_only_nvme( node_present => [$Node] ); #my $return = disk_list_data( API_Obj => $API ); @disk_list_step1 = keys(%$return); if (@disk_list_step1) { logcomment( "$FILER_C : Disk list output contains " . scalar @disk_list_step1 . " and disks are @disk_list_step1" ); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: NO DRIVES PRESENT IN DISK_LIST OUTPUT - STEP : 1"); } } #------------------------------------------------------------ ## Step 2 - Issue mode page 0xB0 ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 2 of 17 : $FILER_C : Issue mode page 0xB0.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); my $scsi_output = $API->execute_raw_command( command => "scsi inquiry -p 0xb0 $dsk_use[0]" ); sleep(10); if ($scsi_output) { $inquiry = 1; logcomment("$FILER_C: scsi inquiry mode page 0xB0 completed"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: SCSI INQUIRY MODE PAGE 0XB0 FAILED - STEP : 1"); } } #------------------------------------------------------------ ## Step 3 - Pre-condition the SSD drive selected ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 3 of 17 : $FILER_C : Pre-condition the SSD drive selected.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); $disk_type = disk_type_maint( Node => $Node ); } my @subtests; foreach my $Node (@Nodes) { my $drive_type; my $FILER_C = $Node->name(); my @dsk_use = @{ $Fil_disk{$FILER_C} }; foreach my $dsk (@dsk_use) { $drive_type = $disk_type->{$dsk}->{'Type'}; last; } if ( $drive_type =~ /SSD/ ) { logcomment("Drive type is SSD, Preconditioning drives"); # PRECONDITION SSD DRIVE push( @subtests, subtest( \&pre_condition_ssd_drive, -runid, "pre_condition_ssd_drive_$FILER_C", -bg, "--", disks => \@dsk_use, node => $Node ) ); } } Subtest::wait_finish( subtest => [@subtests] ); #------------------------------------------------------------ ## Step 4 - TEST 1: Write-Same without unmap - clean ## Pass/Fail criteria: N/A #------------------------------------------------------------ my ( $s_t1, $m_t1, $h_t1 ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 4 of 17 : $FILER_C : TEST 1: Write-Same without unmap - clean.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); my $time_out_temp = 0; my $flag = 0; my $after1; my @end_time; my @start_time; my $check_pass = 0; my $check_fail = 0; my $max_time = 43200; #12 hours wait time to complete disk_qual grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; my $disk_qual = $API->execute_raw_command( command => "disk_qual start -t 9 -d $dsk_use[0] -o noverify nopattern" ); logcomment("Script will wait maximum of 43200 seconds for disk_qual to complete."); if ( $disk_qual =~ /Starting WRITE SAME/ig ) { while ( $time_out_temp < $max_time ) { $after1 = $API->execute_raw_command( 'command' => "" ); $disk_qual = $disk_qual . $after1; if ( ( $disk_qual =~ /WRITE SAME Passed/ig ) || ( $disk_qual =~ /WRITE SAME Failed/ig ) ) { foreach my $line ( split( /\n/, $disk_qual ) ) { if ( $line =~ /\[(\S+)\]:\s+WRITE SAME Passed/ ) { $check_pass = 1; } elsif ( $line =~ /\[(\S+)\]:\s+WRITE SAME Failed/ ) { $check_fail = 1; } elsif ( ( $line =~ /\[(\S+)\]:\s+Started/ ) && ( $line =~ /\s+(\d+)\:(\d+)\:(\d+)\s+/ ) ) { push( @start_time, $1, $2, $3 ); } elsif ( ( $line =~ /\[(\S+)\]:\s+Finished/ ) && ( $line =~ /\s+(\d+)\:(\d+)\:(\d+)\s+/ ) ) { push( @end_time, $1, $2, $3 ); } if (@end_time) { if ( $end_time[0] < $start_time[0] ) { $end_time[0] += 24; } ##Calculate WRITESAME_time = "Finished" time - "Started" time $time_taken1 = ( ( ( $end_time[0] - $start_time[0] ) * 3600 ) + ( ( $end_time[1] - $start_time[1] ) * 60 ) + ( $end_time[2] - $start_time[2] ) ); ( $s_t1, $m_t1, $h_t1 ) = gmtime($time_taken1); logcomment("Test time (t1) : $time_taken1 seconds (i.e) $h_t1 HRS $m_t1 MINS $s_t1 SECS"); last; } } $flag = 1; last; } else { sleep 300; $time_out_temp = $time_out_temp + 300; } } if ( ( $check_pass == 1 ) && ( $flag == 1 ) ) { $test1 = 1; logcomment("WRITE SAME Passed and Completed"); } elsif ( ( $check_fail == 1 ) && ( $flag == 1 ) ) { logcomment("WRITE SAME Failed and Completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C :DISK_QUAL NOT COMPLETED WITHIN THE GIVEN TIME - STEP : 4"); } } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: DISK_QUAL COULD NOT BE STARTED PROPERLY. - STEP : 4"); } disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@dsk_use ); } #------------------------------------------------------------ ## Step 5 - TEST 1: Sanity check read commands ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 5 of 17 : $FILER_C : TEST 1: Sanity check read commands.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; $API->execute_raw_command( command => "disk_qual start -t 1 -n 256 -q 32 -L 100000 -d $dsk_use[0] -o noverify" ); my $time_out_tmp = 0; my $flag; my $max_time = 6000; while ( $time_out_tmp < $max_time ) { $flag = 0; $API->execute_raw_command( 'command' => "\013" ); my $status = $API->execute_raw_command( command => "disk_qual status" ); if ( $status =~ /diskqual is idle/i ) { logcomment("Disk_qual is idle"); last; } else { $flag = 1; } sleep 120; $time_out_tmp = $time_out_tmp + 120; } if ( $flag == 0 ) { $sanity1 = 1; logcomment("$FILER_C: TEST 1: Sanity check completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C:DISK_QUAL COULDN'T COMPLETE WITHIN THE GIVEN TIME - STEP : 5"); } } #------------------------------------------------------------ ## Step 6 - TEST 2: Write-Same with unmap - clean ## Pass/Fail criteria: N/A #------------------------------------------------------------ my ( $s_t2, $m_t2, $h_t2 ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 6 of 17 : $FILER_C : TEST 2: Write-Same with unmap - clean.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); my $time_out_temp = 0; my $after1; my @end_time; my @start_time; my $check_pass = 0; my $check_fail = 0; my $flag = 0; my $max_time = 43200; #12 hours wait time to complete disk_qual grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; my $disk_qual = $API->execute_raw_command( command => "disk_qual start -t 9 -d $dsk_use[0] -o noverify nopattern unmap" ); logcomment("Script will wait maximum of 43200 seconds for disk_qual to complete."); if ( $disk_qual =~ /Starting WRITE SAME/ig ) { while ( $time_out_temp < $max_time ) { $after1 = $API->execute_raw_command( 'command' => "" ); $disk_qual = $disk_qual . $after1; if ( ( $disk_qual =~ /WRITE SAME Passed/ig ) || ( $disk_qual =~ /WRITE SAME Failed/ig ) ) { foreach my $line ( split( /\n/, $disk_qual ) ) { if ( $line =~ /\[(\S+)\]:\s+WRITE SAME Passed/ ) { $check_pass = 1; } elsif ( $line =~ /\[(\S+)\]:\s+WRITE SAME Failed/ ) { $check_fail = 1; } elsif ( ( $line =~ /\[(\S+)\]:\s+Started/ ) && ( $line =~ /\s+(\d+)\:(\d+)\:(\d+)\s+/ ) ) { push( @start_time, $1, $2, $3 ); } elsif ( ( $line =~ /\[(\S+)\]:\s+Finished/ ) && ( $line =~ /\s+(\d+)\:(\d+)\:(\d+)\s+/ ) ) { push( @end_time, $1, $2, $3 ); } if (@end_time) { if ( $end_time[0] < $start_time[0] ) { $end_time[0] += 24; } ##Calculate WRITESAME_time = "Finished" time - "Started" time $time_taken2 = ( ( ( $end_time[0] - $start_time[0] ) * 3600 ) + ( ( $end_time[1] - $start_time[1] ) * 60 ) + ( $end_time[2] - $start_time[2] ) ); ( $s_t2, $m_t2, $h_t2 ) = gmtime($time_taken2); logcomment("Test time (t2) : $time_taken2 seconds (i.e) $h_t2 HRS $m_t2 MINS $s_t2 SECS"); last; } } $flag = 1; last; } else { sleep 300; $time_out_temp = $time_out_temp + 300; } } if ( ( $check_pass == 1 ) && ( $flag == 1 ) ) { $test2 = 1; logcomment("WRITE SAME Passed and Completed"); } elsif ( ( $check_fail == 1 ) && ( $flag == 1 ) ) { logcomment("WRITE SAME Failed and Completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C :DISK_QUAL NOT COMPLETED WITHIN THE GIVEN TIME - STEP : 6"); } } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: DISK_QUAL COULD NOT BE STARTED PROPERLY. - STEP : 6"); } disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@dsk_use ); } #------------------------------------------------------------ ## Step 7 - TEST 2: Sanity check read commands ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 7 of 17 : $FILER_C : TEST 2: Sanity check read commands.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; $API->execute_raw_command( command => "disk_qual start -t 1 -n 256 -q 32 -L 100000 -d $dsk_use[0] -o noverify" ); my $time_out_tmp = 0; my $max_time = 6000; my $flag; while ( $time_out_tmp < $max_time ) { $flag = 0; $API->execute_raw_command( 'command' => "\013" ); my $status = $API->execute_raw_command( command => "disk_qual status" ); if ( $status =~ /diskqual is idle/i ) { logcomment("Disk_qual is idle"); last; } else { $flag = 1; } sleep 120; $time_out_tmp = $time_out_tmp + 120; } if ( $flag == 0 ) { $sanity2 = 1; logcomment("$FILER_C: TEST 2: Sanity check completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C: DISK_QUAL COULDN'T COMPLETE WITHIN THE GIVEN TIME - STEP : 7"); } } #------------------------------------------------------------ ## Step 8 - Random write 4KB aligned (n-hours) ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 8 of 17 : $FILER_C : Random write 4KB aligned (n-hours).", ); my $date_rate; my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); $API->set_timeout( "connectrec-timeout" => 1800 ); my $output = $API->execute_raw_command( command => "disk_latency -p rand_write -q 32 -t 30 -i 0 -m bcs -a 8 -s 4160 -f $dsk_use[0]" ); my $dsk_last = $dsk_use[0]; $API->execute_raw_command( command => "\013" ); $API->execute_raw_command( command => "\013" ); my @output_data = split( /\n/, $output ); foreach my $line (@output_data) { if ( $line =~ /$dsk_last\s+\d+\s+(\d+)\s+\d+\s+\d+\s+\d+$/ ) { logcomment("Data rate - $1"); $date_rate = $1; last; } } my $ssd_capacity; my $disk_list_result = nvme_list_only_nvme( node_present => [$Node] ); if ($date_rate) { foreach my $dsk ( keys %$disk_list_result ) { if ( $disk_list_result->{$dsk}->{'DISK'} =~ /$dsk_last$/ ) { my $disk_size = $disk_list_result->{$dsk}->{'BLOCKS'}; my $bps = $disk_list_result->{$dsk}->{'BPS'}; $ssd_capacity = int( ( $disk_size * $bps ) / 1000000 ); last; } } } else { $test_status = 1; logcomment("**FAIL** : $FILER_C: DATA RATE IS NOT CAPTURED PROPERLY - STEP : 8"); } logcomment(" Data rate - $date_rate and Drive capacity - $ssd_capacity"); logcomment("determine random write time"); my $write_time = int( $ssd_capacity / $date_rate ); #(1.6TB/147mb/s = 10884 seconds) my ( $s, $m, $h ) = gmtime($write_time); logcomment("$FILER_C: Write time : $write_time (i.e) $h HRS $m MINS $s SECS"); my $timeout = $write_time + 7200; $API->set_timeout( "connectrec-timeout" => $timeout ); logcomment("$FILER_C: Performing random writes to disk: $dsk_use[0]"); $API->execute_raw_command( command => "disk_latency -p rand_write -q 32 -t $write_time -i 0 -m bcs -a 8 -s 4160 -f $dsk_use[0]" ); $API->execute_raw_command( command => "\013" ); $API->execute_raw_command( command => "\013" ); } #------------------------------------------------------------ ## Step 9 - TEST 3: Write-Same with unmap - dirty (result time=t3) ## Pass/Fail criteria: N/A #------------------------------------------------------------ my ( $s_t3, $m_t3, $h_t3 ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 9 of 17 : $FILER_C : TEST 3: Write-Same with unmap - dirty.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); my $time_out_temp = 0; my $after1; my @end_time; my @start_time; my $check_pass = 0; my $check_fail = 0; my $flag = 0; my $max_time = 43200; #12 hours wait time to complete disk_qual grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; my $disk_qual = $API->execute_raw_command( command => "disk_qual start -t 9 -d $dsk_use[0] -o noverify nopattern unmap" ); sleep 60; logcomment("Script will wait maximum of 43200 seconds for disk_qual to complete."); if ( $disk_qual =~ /Starting WRITE SAME/ig ) { while ( $time_out_temp < $max_time ) { $after1 = $API->execute_raw_command( 'command' => "" ); $disk_qual = $disk_qual . $after1; if ( ( $disk_qual =~ /WRITE SAME Passed/ig ) || ( $disk_qual =~ /WRITE SAME Failed/ig ) ) { foreach my $line ( split( /\n/, $disk_qual ) ) { if ( $line =~ /\[(\S+)\]:\s+WRITE SAME Passed/ ) { $check_pass = 1; } elsif ( $line =~ /\[(\S+)\]:\s+WRITE SAME Failed/ ) { $check_fail = 1; } elsif ( ( $line =~ /\[(\S+)\]:\s+Started/ ) && ( $line =~ /\s+(\d+)\:(\d+)\:(\d+)\s+/ ) ) { push( @start_time, $1, $2, $3 ); } elsif ( ( $line =~ /\[(\S+)\]:\s+Finished/ ) && ( $line =~ /\s+(\d+)\:(\d+)\:(\d+)\s+/ ) ) { push( @end_time, $1, $2, $3 ); } if (@end_time) { if ( $end_time[0] < $start_time[0] ) { $end_time[0] += 24; } ##Calculate WRITESAME_time = "Finished" time - "Started" time $time_taken3 = ( ( ( $end_time[0] - $start_time[0] ) * 3600 ) + ( ( $end_time[1] - $start_time[1] ) * 60 ) + ( $end_time[2] - $start_time[2] ) ); ( $s_t3, $m_t3, $h_t3 ) = gmtime($time_taken3); logcomment("Test time (t3) : $time_taken3 seconds (i.e) $h_t3 HRS $m_t3 MINS $s_t3 SECS"); last; } } $flag = 1; last; } else { sleep 300; $time_out_temp = $time_out_temp + 300; } } if ( ( $check_pass == 1 ) && ( $flag == 1 ) ) { $test3 = 1; logcomment("WRITE SAME Passed and Completed"); } elsif ( ( $check_fail == 1 ) && ( $flag == 1 ) ) { logcomment("WRITE SAME Failed and Completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C : DISK_QUAL NOT COMPLETED WITHIN THE GIVEN TIME - STEP : 9"); } } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: DISK_QUAL COULD NOT BE STARTED PROPERLY. - STEP : 9"); } disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@dsk_use ); } #------------------------------------------------------------ ## Step 10 - TEST 3: Sanity check read commands ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 10 of 17 : $FILER_C : TEST 3: Sanity check read commands.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; $API->execute_raw_command( command => "disk_qual start -t 1 -n 256 -q 32 -L 100000 -d $dsk_use[0] -o noverify" ); my $time_out_tmp = 0; my $max_time = 6000; my $flag; while ( $time_out_tmp < $max_time ) { $flag = 0; $API->execute_raw_command( 'command' => "\013" ); my $status = $API->execute_raw_command( command => "disk_qual status" ); if ( $status =~ /diskqual is idle/i ) { logcomment("Disk_qual is idle"); last; } else { $flag = 1; } sleep 120; $time_out_tmp = $time_out_tmp + 120; } if ( $flag == 0 ) { $sanity3 = 1; logcomment("$FILER_C: TEST 3: Sanity check completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C: DISK_QUAL COULDN'T COMPLETE WITHIN THE GIVEN TIME - STEP : 10"); } } #------------------------------------------------------------ ## Step 11 - TEST 4: Write-Same with unmap and error injection ## Pass/Fail criteria: N/A #------------------------------------------------------------ my %retry_success = (); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 11 of 17 : $FILER_C : TEST 4: Write-Same with unmap and error injection.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); my $i = 1; my $check_pass; my $check_fail; my $flag; my @retry_command_time; while ( $i <= 20 ) { my $max_time = 600; my $time_out_temp = 0; logcomment("Iteration $i : Starting Write-Same with unmap and error injection"); $check_fail = 0; $check_pass = 0; $flag = 0; grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; $API->execute_raw_command( command => "disk_qual start -t 9 -d $dsk_use[0] -o noverify nopattern unmap" ); my $range = ( $time_taken3 - 5 ); my $randm_sleep_time = int( rand($range) ); my ( $s_rslt, $m_rslt, $h_rslt ) = gmtime($randm_sleep_time); logcomment("$FILER_C: Write same time $time_taken3 seconds (i.e) $h_t3 HRS $m_t3 MINS $s_t3 SECS ==== Random generated time of ($time_taken3 - 5): $randm_sleep_time seconds (i.e) $h_rslt HRS $m_rslt MINS $s_rslt SECS"); logcomment("$FILER_C: Sleep for $randm_sleep_time seconds (i.e) $h_rslt HRS $m_rslt MINS $s_rslt SECS before issuing snvmadmin nssr"); sleep($randm_sleep_time); my $dev_pwr_cyc = $API->execute_raw_command( command => "snvmadmin nssr $dsk_use[0]" ); while ( $time_out_temp < $max_time ) { my $after1 = $API->execute_raw_command( 'command' => "" ); my $pwr_cyc = $dev_pwr_cyc . $after1; if ( ( $pwr_cyc =~ /request successful/i ) || ( $pwr_cyc =~ /request failed/i ) ) { my @sysconf_lines = split( /\n/, $pwr_cyc ); foreach my $line (@sysconf_lines) { if ( $line =~ /\[\S+\:\s*scsi\.cmd\.retrySuccess:debug]:/ig ) { ( $retry_command_time[$i] ) = $line =~ /cdb\s*0x41:\S+\s*\((\d+)\)\.\s*$/; $retry_success{$i} = $retry_command_time[$i]; logcomment("Retry success time for iteration $i : $retry_command_time[$i]"); $check_pass = 1; last; } if ( $line =~ /request failed/i ) { $check_fail = 1; last; } } $flag = 1; last; } else { sleep 60; $time_out_temp = $time_out_temp + 60; } } if ( ( $check_pass == 1 ) && ( $flag == 1 ) ) { $test4 = 1; logcomment("Iteration $i : scsi cmd retrySuccess"); } elsif ( ( $check_fail == 1 ) && ( $flag == 1 ) ) { logcomment("Iteration $i : Request failed after retry"); } else { logcomment("**WARNING** : $FILER_C : Iteration $i : scsi cmd retry messages didn't appear within the given time - STEP : 11"); } my $timeout_qual = 0; my $time_qual = 3000; while ( $timeout_qual < $time_qual ) { my $diskqual = $API->execute_raw_command( command => "disk_qual report -d $dsk_use[0]" ); if ( ( $diskqual =~ /WRITE SAME Passed/i ) || ( $diskqual =~ /WRITE SAME Failed/i ) ) { logcomment("Iteration $i : WRITE SAME Completed"); last; } else { sleep 30; $timeout_qual = $timeout_qual + 30; } } $i++; } } #------------------------------------------------------------ ## Step 12 - TEST 4: Sanity check read commands ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 12 of 17 : $FILER_C : TEST 4: Sanity check read commands.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; $API->execute_raw_command( command => "disk_qual start -t 1 -n 256 -q 32 -L 100000 -d $dsk_use[0] -o noverify" ); my $time_out_tmp = 0; my $flag; my $max_time = 6000; while ( $time_out_tmp < $max_time ) { $flag = 0; $API->execute_raw_command( 'command' => "\013" ); my $status = $API->execute_raw_command( command => "disk_qual status" ); if ( $status =~ /diskqual is idle/i ) { logcomment("Disk_qual is idle"); last; } else { $flag = 1; } sleep 120; $time_out_tmp = $time_out_tmp + 120; } if ( $flag == 0 ) { $sanity4 = 1; logcomment("$FILER_C: TEST 4: Sanity check completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C: DISK_QUAL COULDN'T COMPLETE WITHIN THE GIVEN TIME - STEP : 12"); } } #------------------------------------------------------------ ## Step 13 - TEST 4: Sanity check write,read and verify commands ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 13 of 17 : $FILER_C : TEST 4: Sanity check write,read and verify commands.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; $API->execute_raw_command( command => "disk_qual start -t 5 -n 256 -q 32 -L 10000 -d $dsk_use[0] -o verify" ); my $time_out_tmp = 0; my $max_time = 6000; my $flag; while ( $time_out_tmp < $max_time ) { $flag = 0; $API->execute_raw_command( 'command' => "\013" ); my $status = $API->execute_raw_command( command => "disk_qual status" ); if ( $status =~ /diskqual is idle/i ) { logcomment("Disk_qual is idle"); last; } else { $flag = 1; } sleep 120; $time_out_tmp = $time_out_tmp + 120; } if ( $flag == 0 ) { $sanity4_verify = 1; logcomment("$FILER_C: TEST 4: Sanity check write,read and verify completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C: DISK_QUAL COULDN'T COMPLETE WITHIN THE GIVEN TIME - STEP : 13"); } } #------------------------------------------------------------ ## Step 14 - TEST 5: Clean drive back up for next test - Write-Same with unmap ## Pass/Fail criteria: N/A #------------------------------------------------------------ my ( $s_t5, $m_t5, $h_t5 ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 14 of 17 : $FILER_C : TEST 5: Clean drive back up for next test - Write-Same with unmap", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); my $time_out_temp = 0; my $after1; my @end_time; my @start_time; my $check_pass = 0; my $check_fail = 0; my $flag = 0; my $max_time = 43200; #12 hours wait time to complete disk_qual grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; my $disk_qual = $API->execute_raw_command( command => "disk_qual start -t 9 -d $dsk_use[0] -o noverify nopattern unmap" ); logcomment("Script will wait maximum of 43200 seconds for disk_qual to complete."); if ( $disk_qual =~ /Starting WRITE SAME/ ) { while ( $time_out_temp < $max_time ) { $after1 = $API->execute_raw_command( 'command' => "" ); $disk_qual = $disk_qual . $after1; if ( ( $disk_qual =~ /WRITE SAME Passed/i ) || ( $disk_qual =~ /WRITE SAME Failed/i ) ) { foreach my $line ( split( /\n/, $disk_qual ) ) { if ( $line =~ /\[(\S+)\]:\s+WRITE SAME Passed/ ) { $check_pass = 1; } elsif ( $line =~ /\[(\S+)\]:\s+WRITE SAME Failed/ ) { $check_fail = 1; } elsif ( ( $line =~ /\[(\S+)\]:\s+Started/ ) && ( $line =~ /\s+(\d+)\:(\d+)\:(\d+)\s+/ ) ) { push( @start_time, $1, $2, $3 ); } elsif ( ( $line =~ /\[(\S+)\]:\s+Finished/ ) && ( $line =~ /\s+(\d+)\:(\d+)\:(\d+)\s+/ ) ) { push( @end_time, $1, $2, $3 ); } if (@end_time) { if ( $end_time[0] < $start_time[0] ) { $end_time[0] += 24; } ##Calculate WRITESAME_time = "Finished" time - "Started" time $time_taken5 = ( ( ( $end_time[0] - $start_time[0] ) * 3600 ) + ( ( $end_time[1] - $start_time[1] ) * 60 ) + ( $end_time[2] - $start_time[2] ) ); ( $s_t5, $m_t5, $h_t5 ) = gmtime($time_taken5); logcomment("Test time (t5) : $time_taken5 seconds (i.e) $h_t5 HRS $m_t5 MINS $s_t5 SECS"); last; } } $flag = 1; last; } else { sleep 300; $time_out_temp = $time_out_temp + 300; } } if ( ( $check_pass == 1 ) && ( $flag == 1 ) ) { $test5 = 1; logcomment("WRITE SAME Passed and Completed"); } elsif ( ( $check_fail == 1 ) && ( $flag == 1 ) ) { logcomment("WRITE SAME Failed and Completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C : DISK_QUAL NOT COMPLETED WITHIN THE GIVEN TIME - STEP : 14"); } } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: DISK_QUAL COULD NOT BE STARTED PROPERLY. - STEP : 14"); } disk_qual_drive_assign_maint( Node_ref => \@Nodes, disk_ref => \@dsk_use ); } #------------------------------------------------------------ ## Step 15 - TEST 5: Sanity check read commands ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 15 of 17 : $FILER_C : TEST 5: Sanity check read commands.", ); my @dsk_use = @{ $Fil_disk{$FILER_C} }; my $API = $FILER_API{$FILER_C}; logcomment("$FILER_C: Drive that will be used for test : $dsk_use[0]"); grep { $disk_qual_cnt{$_}++ } $dsk_use[0]; $API->execute_raw_command( command => "disk_qual start -t 1 -n 256 -q 32 -L 100000 -d $dsk_use[0] -o noverify" ); my $time_out_tmp = 0; my $max_time = 6000; my $flag; while ( $time_out_tmp < $max_time ) { $flag = 0; $API->execute_raw_command( 'command' => "\013" ); my $status = $API->execute_raw_command( command => "disk_qual status" ); if ( $status =~ /diskqual is idle/i ) { logcomment("Disk_qual is idle"); last; } else { $flag = 1; } sleep 120; $time_out_tmp = $time_out_tmp + 120; } if ( $flag == 0 ) { $sanity5 = 1; logcomment("$FILER_C: TEST 5: Sanity check completed"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C: DISK_QUAL COULDN'T COMPLETE WITHIN THE GIVEN TIME - STEP : 15"); } } #------------------------------------------------------------ ## Step 16 - save disk_list entry output and compare with step 1 ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 16 of 17 : $FILER_C : Save disk_list entry output and compare with STEP 1.", ); my $API = $FILER_API{$FILER_C}; my $return = nvme_list_only_nvme( node_present => [$Node] ); #my $return = disk_list_data( API_Obj => $API ); my @disk_list_step16 = keys(%$return); if (@disk_list_step16) { logcomment( "$FILER_C : In Step 16 Disk list output contains " . scalar @disk_list_step16 . " and disks are @disk_list_step16" ); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: NO DRIVES PRESENT IN DISK_LIST OUTPUT - STEP : 16"); } if ( scalar @disk_list_step1 == scalar @disk_list_step16 ) { logcomment("Disk list output is same as the disk list output of Step 1"); } else { $test_status = 1; logcomment("**FAIL** : $FILER_C: DISK LIST OUTPUT IS DIFFERENT FROM THE DISK LIST OUTPUT OF STEP 1 - STEP : 16"); } } #------------------------------------------------------------ ## Step 17 - Display results ## Pass/Fail criteria: N/A #------------------------------------------------------------ foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "STEP 17 of 17 : $FILER_C : Display Results.", ); if ( $inquiry == 1 ) { logcomment("Read mode page 0xB0 Passed"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: READ MODE PAGE 0XB0 FAILED - STEP : 17"); } if ( $test1 == 1 ) { logcomment("TEST 1: WRITE SAME without unmap -clean ==== Test time(t1) : $time_taken1 sec (i.e) $h_t1 HRS $m_t1 MINS $s_t1 SECS"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 1: WRITE SAME WITHOUT UNMAP -CLEAN FAILED - STEP : 17"); } if ( $test2 == 1 ) { logcomment("TEST 2: WRITE SAME with unmap -clean ==== Test time(t2) : $time_taken2 sec (i.e) $h_t2 HRS $m_t2 MINS $s_t2 SECS"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 2: WRITE SAME WITH UNMAP -CLEAN FAILED - STEP : 17"); } if ( $test3 == 1 ) { logcomment("TEST 3: WRITE SAME with unmap -dirty ==== Test time(t3) : $time_taken3 sec (i.e) $h_t3 HRS $m_t3 MINS $s_t3 SECS"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: WRITE SAME WITH UNMAP -DIRTY FAILED - STEP : 17"); } if ( $test5 == 1 ) { logcomment("TEST 5: WRITE SAME with unmap -dirty ==== Test time(t5) : $time_taken5 sec (i.e) $h_t5 HRS $m_t5 MINS $s_t5 SECS"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 5: WRITE SAME WITH UNMAP -DIRTY FAILED - STEP : 17"); } if ( $test4 == 1 ) { my $i = 1; while ( $i <= 20 ) { if ( $retry_success{$i} ) { logcomment("TEST 4: Iteration $i : Retry success time - $retry_success{$i} msec"); } else { logcomment("**WARNING**: $FILER_C: TEST 4 : Iteration $i - Request failed after retry - STEP 17"); } $i++; } } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 4: SCSI CMD REQUEST FAILED AFTER RETRY FOR ALL 20 ITERATIONS - STEP 17"); } if ( $sanity1 == 1 ) { logcomment("TEST 1: Sanity read after WS test 1 Passed"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 1: SANITY READ AFTER WS TEST 1 FAILED - STEP 17"); } if ( $sanity2 == 1 ) { logcomment("TEST 2: Sanity read after WS test 2 Passed"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 2: SANITY READ AFTER WS TEST 2 FAILED - STEP 17"); } if ( $sanity3 == 1 ) { logcomment("TEST 3: Sanity read after WS test 3 Passed"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 3:SANITY READ AFTER WS TEST 3 FAILED - STEP 17"); } if ( $sanity4 == 1 ) { logcomment("TEST 4: Sanity read after WS test 4 Passed"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 3:SANITY READ AFTER WS TEST 4 FAILED - STEP 17"); } if ( $sanity4_verify == 1 ) { logcomment("TEST 4: Sanity WRV after WS test 4 Passed"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 4: SANITY WRV AFTER WS TEST 4 FAILED - STEP 17"); } if ( $sanity5 == 1 ) { logcomment("TEST 5: Sanity read after WS test 5 Passed"); } else { $test_status = 1; logcomment("**FAIL**: $FILER_C: TEST 5: SANITY READ AFTER WS TEST 5 FAILED - STEP 17"); } } #opening file to wrte the cnt of disk_qual start on drives for compare in the post test my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/; my $disk_qual_path = $actual_log_dir . "/disk_qual_cnt_$TC_name.log"; logcomment("**DEBUG** : log path : $disk_qual_path"); open( FH, "+>>$disk_qual_path" ) || die "Failed to open the write file disk_qual.log"; foreach my $dsk ( keys %disk_qual_cnt ) { logcomment("$dsk\t\t$disk_qual_cnt{$dsk}"); print FH "$dsk\t\t$disk_qual_cnt{$dsk}\n"; } close FH; ########################################################################################### ## Test log/results ########################################################################################### logcomment("Test $TC_name is completed."); logresult( "INFO", msg => $test_status ); } ############################################################# #Cleanup ############################################################# sub cleanup() { logcomment(" $filer_names - Clean up and post test process"); $Test->description("Preforming Post test process - Check filer state, Check disk, check root volume,halt hammer, halt scrub, filer config"); my $node_ref = \@Nodes; ########################################################################################### ## Post Test process - Category : "post_test" ########################################################################################## nvme_post_test( filer_mode => $Mode, node_present => [@Nodes], Test => $Test, change_state_to => "MAINT", ); return $TCD::PASS; }