#!/usr/software/bin/perl # $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/NDATE_NVMe/NADQ02_NVMe_Hmr_Scrub.thpl#1 $ # ## @summary Mode ## Disk Firmware Download in background/foreground mode # ## ## @description ## Verify that the drive can download disk firmware in foreground and background mode properly ## using the method available in the NetApp system. ## ## @Test Mode ## File system mode ## ## @Test bed setup ## FC : Cluster Fabric ## SATA : Cluster ## SAS : Cluster ## ## @usage ## The test can be run independently or with other tests as part of STEST. ## ## @dependencies ## @steps ## The test will execute steps mentioned below: ## ## ## @status Automated ######################################## ### Library functions ########################################## use strict; use Storage::NVMe_Common_Lib; use Storage::Common_Lib; use NACL::C::Statistics; use NACL::MTask::EventLogDetector; use List::Util qw(shuffle); ######################################### ######################################### ### Initialization/declaration ######################################### ##### # Global parameters use vars qw( $FILER $MODE $FILER_CONN $TEST_CONFIG $TEST_SETUP $FILER_PROMPT $LOGDIR $EOE $TEST_WAIT_TIME $BOOT_MODE $BOOT_TYPE $SSD $EMAIL $MAIL_TO $MAIL_FROM $FILER_TYPE $FIRMWARE $RUNID $USER_LEVL $ARMADILLO ); my $params = NATE::ParamSet->new( global => 1 ); $FILER = $params->get( 'FILER', default => 'Filer' ); $TEST_CONFIG = $params->get( 'TEST_CONFIG', default => 'E' ); $TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' ); $LOGDIR = $params->get( 'LOGDIR', default => undef ); $EOE = $params->get( 'EOE', default => 1 ); $EMAIL = $params->get( 'EMAIL', default => 'y' ); $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); $TEST_SETUP = $params->get( 'TEST_SETUP', default => 'SAS' ); $BOOT_MODE = $params->get( 'BOOT_MODE', default => '1' ); $BOOT_TYPE = $params->get( 'BOOT_TYPE', default => 'A' ); $RUNID = $params->get( 'RUNID', default => undef ); $SSD = $params->get( 'SSD', default => 'no' ); $ARMADILLO = $params->get( 'ARMADILLO', default => '2' ); $FILER_TYPE = $params->get( 'FILER_TYPE', default => 'BR' ); $FIRMWARE = $params->get( 'FIRMWARE', default => 'Firmware file not entered' ); $USER_LEVL = $params->get( 'USER_LEVL', default => 'high' ); ################ # Testcase name ################ my $TC_name; $TC_name = "402_NADQ02_NVMe_Hmr_Scrub"; #Common variable declaration my ( $email_subject, $email_body ); my @Nodes; my $filer_names; my $Mode; my $test_status = 0; my %nodes_filer; my $firmware_hash = {}; my @firmware_list; my $cnt = 1; my $disk_show_v; my %test_details; my $Home = $ENV{HOME}; my $scrub_start; my $scrub_end; my $test_wait_time = 12 * 3600; #12 Hrs test if ( $TEST_WAIT_TIME == 1 ) { $test_wait_time = ( $test_wait_time / 10 ); } elsif ( $TEST_WAIT_TIME == 2 ) { $test_wait_time = ( $test_wait_time / 2 ); } elsif ( $TEST_WAIT_TIME == 4 ) { $test_wait_time = ( ($test_wait_time) * (.95) ); } ######################################### # Test case available for execution ######################################### my @Testcases = ( NVMe_Hmr_Scrub => "Hammer Scrub" ); &main(); sub main { # Debug break point $DB::single = 2; # Create Test Case Driver object $Test = new TCD( -testcases => [@Testcases] ); if ( $Test->error ) { $Test->log( $Test->errmsg ); $test_status = 1; $email_subject = "$TC_name : Test FAILED: FAIL"; $email_body = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n"; send_message( mail_subject => $email_subject, mail_body => $email_body, MAIL_FROM => $MAIL_FROM, MAIL_TO => $MAIL_TO, EOE => $EOE, EMAIL => $EMAIL ); return $TCD::FAIL; } # Performs method callbacks $Test->run_test(); if ( $Test->error ) { $Test->log( $Test->errmsg ); return $TCD::FAIL; } exit(0); } ## end sub main ########## INIT ################################################### # This init subroutine will initialise the filer. #################################################################### sub init() { $Test->description(" Initialising all required variables and filer connections "); $filer_names = $Test->get_param("FILER"); # Capturing Filer names from the param(Test_Suite) ##Check for duplicate node object and push unique node object in Nodes array. my @temp_nodes = NACL::C::Node->find(); # Find Nodes/Filers used in the test, Based on FILER param. foreach my $Node (@temp_nodes) { my $FILER_C = $Node->name(); $nodes_filer{$FILER_C} = $Node; } @Nodes = values(%nodes_filer); # Contains Node object used for test execution. sort(@Nodes); $Mode = $Nodes[0]->mode(); logcomment("Checking for execution mode"); logcomment( "FILER- $filer_names : $TC_name : started, expected max completion time 30 Min : " . scalar( localtime() ) ); logcomment( "FILER- $filer_names : Log file for this test case: \n $LOGDIR/$TC_name" . ".log " . scalar( localtime() ) ); return $TCD::PASS; } ########## SETUP ################################################### # setup automatically called before executing tests #################################################################### sub setup() { $Test->description("Setup the environment for the test execution "); logcomment("Mode of filer $filer_names : $Mode"); my $node_ref = \@Nodes; version_test( node_present => $node_ref, tc_name => $TC_name ); ##################################################################### # Pre test proces : call for pre_n_post test process ##################################################################### nvme_pre_test( node_present => $node_ref, Test => $Test, change_state_to => "MAINT", filer_mode => $Mode ); return $TCD::PASS; } sub NVMe_Hmr_Scrub { my @subtests; my $status = 0; logcomment( "Number of nodes are " . scalar @Nodes . " and the filer are $filer_names" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&exe_hmar_scrub, -runid, "Exe_Hmr_Srb_$FILER_C", -bg, "--", $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); my $status = status_return( subtest_ref => [@subtests] ); logcomment("Total test status is : $status"); if ( $status == 0 ) { return $TCD::PASS; } else { return $TCD::FAIL; } } sub exe_hmar_scrub { $Test->description("Execute Hammer Scrub Command "); my ( @Nodes, %filer_nvol ); my $Home = $ENV{HOME}; my @NON_RT_AGG; my @NON_RT_VOLS; my %filer_nvol = (); my $create_agg_n_vol = 0; my $aggr_vol_found = 0; push( @Nodes, shift(@_) ); logcomment("Total Nodes - @Nodes"); logcomment( "Filer passed to the subtest :: " . $Nodes[0]->name() ); #STEP 1 nvme_eye_catcher( Test => $Test, string => "STEP 1 of 13 : List all drives " ); my $disk_list = nvme_list_only_nvme( node_present => [ $Nodes[0] ] ); # Check non root volumes nvme_eye_catcher( Test => $Test, string => "STEP 2 of 13 : Check non-root aggregate and create one non-root aggregate" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Check for non-root aggregate "); my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node ); if (@non_rt_agg) { logcomment("Total non-root aggregate is $non_rt_status and Non -root aggregate are @non_rt_agg"); logcomment("Check for non-root volume existance in available non root aggregate"); my @vol_created = check_non_root_volumes( Node => $Node ); if (@vol_created) { logcomment("$FILER_C : Volumes which are available in the filer are :: @vol_created"); $filer_nvol{$FILER_C} = [@vol_created]; $aggr_vol_found = 1; } else { $create_agg_n_vol = 1; vserver_delete( Node => $Node ); #Vserver deletion aggregate_volume_delete( Node => $Node ); } } else { logcomment("Non-root aggregate cannot be found"); $create_agg_n_vol = 1; } if ( $create_agg_n_vol == 1 ) { logcomment("No non-root volume or aggregate found, Create Aggregate and volume "); logcomment("Check for spare drive availability, Minimum 3 spares are required to create aggregate"); my $spare_list = nvme_sto_agg_show_spare_dsk( Node => $Node ); my @spare_drvs = keys %$spare_list; my $spare_count = @spare_drvs; @spare_drvs = shuffle(@spare_drvs); my $drv_ag = $spare_count - 2; logcomment("Total $drv_ag drives will be used to create aggregate, leaving 2 spare"); my @aggr_dr = splice @spare_drvs, 0, $drv_ag; logcomment("$FILER_C : Total spare are : $spare_count, $drv_ag drives will be used for aggregate creation and drives are : @aggr_dr"); if ( $spare_count > 2 ) { logcomment("Sufficient spares are available on Filer $FILER_C"); logcomment("Checking for non-root aggregate and creating aggregate "); my @aggr_lst = nvme_get_agg_dtls_n_create( Node => $Node, disk_count => $drv_ag ); logcomment("Aggregate created - @aggr_lst"); if ( !@aggr_lst ) { logcomment("Non-root Aggregate were not found, Cannot create non-root aggregate"); logcomment("**FATAL** : $FILER_C : Non-root Aggregate were not found, Cannot create non-root aggregate"); logresult( 'FATAL', "$FILER_C : Non-root Aggregate were not found, Cannot create non-root aggregate" ); } my $add_aggr = @aggr_lst; logcomment("Total Additional aggregate count is $add_aggr and aggregate are @aggr_lst is present"); @NON_RT_AGG = @aggr_lst; my $required_size; my $vol_size = 90; foreach my $agg (@aggr_lst) { logcomment("Get Usable size of Aggregate $agg"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" ); my $aggr_sz = $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate show -aggregate $agg -root false -fields state,availsize" ); foreach my $line ( split( /\n/, $aggr_sz ) ) { next if ( $line =~ /---/ ); if ( $line =~ /(\S+)\s+(\S+)\s+(online|offline)/ ) { my $agg_sz = $2; my ( $aggs, $unit ) = $agg_sz =~ /(\S+)(KB|MB|GB|TB|PB)/; logcomment("Aggregate $agg size is $aggs"); $required_size = int( ( $vol_size / 100 ) * $aggs ); $required_size = "$required_size" . "$unit"; } } } } else { logcomment("Spare available is not sufficient to create additional aggregate"); logcomment("**FATAL**:$FILER_C : Spare available is not sufficient to create additional aggregate"); logresult( 'FATAL', "$FILER_C : Spare available is not sufficient to create additional aggregate" ); } } else { if ( $aggr_vol_found == 1 ) { logcomment("Already aggregate and volume exists on filer :$FILER_C, re-using aggregates and volume"); } else { logcomment("**FATAL**:$FILER_C :Cannot get volume and aggregate status"); logresult( 'FATAL', "$FILER_C : Cannot get volume and aggregate status" ); } } } # STEP 3 Create Voluume a nvme_eye_catcher( Test => $Test, string => "STEP 3 of 13 : Check for non-root aggregate and create volume if needed" ); logcomment("DEBUG : Check Volume "); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("check for non-root volume"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); if ( (@NON_RT_AGG) && ( $aggr_vol_found == 0 ) ) { logcomment("NO NON_ROOT VOLUMES FOUND, CREATE VOLUMES ON AGGREGATE AVAILABLE"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); my $vserver_name = "vser_$FILER_C"; my $root = "volvs"; my $vser_cret = 0; logcomment("Creating vserver $vserver_name"); my $vser_out = $Api_Set_Obj->execute_raw_command( 'command' => "vserver show -fields vserver,rootvolume" ); if ( $vser_out =~ /($vserver_name)\s+(\S+)/ ) { logcomment("Vserver $vserver_name found on $FILER_C and root volume is $2 "); if ( $2 =~ /-/ ) { logcomment("Vserver does not have root volume, Destroy and create new vserver"); $Api_Set_Obj->execute_raw_command( 'command' => "vserver delete -vserver $vserver_name -foreground true" ); sleep 3; $vser_cret = 1; } } else { logcomment("Cannot Find Vserver on $FILER_C"); $vser_cret = 1; } if ( $vser_cret == 1 ) { logcomment("Create Vserver $vserver_name with root volume $root on filer $FILER_C"); my $prompts_answers = [ ".*Do you want to continue??.*" => 'y' ]; my $vser_o = $Api_Set_Obj->execute_raw_command( 'command' => "vserver create -vserver $vserver_name -rootvolume $root", 'connectrec-match_table' => $prompts_answers ); logcomment("DEBUGL : $vser_o"); } my $vol_name; my $nm = 1; foreach my $ag (@NON_RT_AGG) { my $vol_size = 90; my $agg_sz; my $required_size; $vol_name = "vol_" . "$FILER_C" . "$nm"; $vol_name =~ s/-/_/g; logcomment("Create Volume on aggregate $ag with 90% of aggregate size"); my $aggr_sz = $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate show -aggregate $ag -root false -fields state,availsize" ); foreach my $line ( split( /\n/, $aggr_sz ) ) { next if ( $line =~ /---/ ); if ( $line =~ /(\S+)\s+(\S+)\s+(online|offline)/ ) { my $agg_sz = $2; my ( $aggs, $unit ) = $agg_sz =~ /(\S+)(KB|MB|GB|TB|PB)/; logcomment("Aggregate $ag size is $aggs"); $required_size = int( ( $vol_size / 100 ) * $aggs ); if ( ( $required_size > 100 ) && ( $unit =~ /TB/ ) ) { logcomment("Volume size to be created is $required_size $unit, changing to 100TB"); $required_size = 100; } $required_size = "$required_size" . "$unit"; } } logcomment("Volume size to be created : $required_size"); logcomment("Aggregate $ag will be used to create vol $vol_name"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; my $ot = $Api_Set_Obj->execute_raw_command( 'command' => "vol create -vserver $vserver_name -volume $vol_name -aggregate $ag -size $required_size -state online", 'connectrec-match_table' => $prompts_answers ); logcomment("$ot"); if ( $ot =~ /Error|Invalid|Failed/ ) { logcomment("**FATAL** : Volume $vol_name cannot be created"); logresult( 'FATAL', "$FILER_C : Volume $vol_name cannot be created" ); } logcomment("Volume $vol_name created"); push( @NON_RT_VOLS, $vol_name ); $nm++; } logcomment("Volumes Created are :@NON_RT_VOLS"); $filer_nvol{$FILER_C} = [@NON_RT_VOLS]; } logcomment("Volume found on filer $FILER_C, re-using available volumes") if ( $aggr_vol_found == 1 ); } #STEP 4 : Verify that there is IO to the drives. nvme_eye_catcher( Test => $Test, string => "STEP 4 of 13 : Check for IO to the drives" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Show IO statistics of all drives"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_raw_command( 'command' => "statistics show-periodic -iterations 15 -interval 2 -summary false" ); logcomment("End of IO statistics"); } #STEP 5 :Start a thread of kernel tool "Hammer" with the file size 1234 nvme_eye_catcher( Test => $Test, string => "STEP 5 of 13 : Start a thread of kernel tool \"Hammer\" with the file size 1234" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my @n_root_vol = @{ $filer_nvol{$FILER_C} }; foreach my $vol (@n_root_vol) { logcomment("$FILER_C : Start a thread of kernel tool Hammer with the file size 1234 on volume $vol"); my $filename = "/vol/$vol/h0"; my $size = '1234'; my $result = hammer_start( Node => $Node, file_name => $filename, file_size => $size, Test => $Test, ); if ( $result == 1 ) { logcomment("- $FILER_C - Hammer with file $filename with size $size is not executed properly"); logcomment("- $FILER_C - Thread of kernel tool Hammer with file-size 1234 failed on $FILER_C"); logcomment("**FAIL** :THREAD OF KERNEL TOOL HAMMER WITH FILE-SIZE 1234 FAILED ON $FILER_C,skipping rest of the test steps : Step 2 "); return logresult( 'INFO', msg => 1 ); } else { logcomment("Thread of kernel tool Hammer with file-size 1234 started on $FILER_C"); } } } #STEP 6 :Start a thread of kernel tool "Hammer" with the file size 123456 nvme_eye_catcher( Test => $Test, string => "STEP 6 of 13 : Start a thread of kernel tool \"Hammer\" with the file size 123456" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my @n_root_vol = @{ $filer_nvol{$FILER_C} }; foreach my $vol (@n_root_vol) { logcomment("$FILER_C : Start a thread of kernel tool Hammer with the file size 123456 on volume $vol"); my $filename = "/vol/$vol/h1"; my $size = '123456'; my $result = hammer_start( Node => $Node, file_name => $filename, file_size => $size, Test => $Test, ); if ( $result == 1 ) { logcomment("- $FILER_C - Hammer with file $filename with size $size is not executed properly"); logcomment("- $FILER_C - Thread of kernel tool Hammer with file-size 123456 failed on $FILER_C"); logcomment("**FAIL** :THREAD OF KERNEL TOOL HAMMER WITH FILE-SIZE 1234 FAILED ON $FILER_C,skipping rest of the test steps : Step 2 "); return logresult( 'INFO', msg => 1 ); } else { logcomment("Thread of kernel tool Hammer with file-size 123456 started on $FILER_C"); } } } #STEP 7 nvme_eye_catcher( Test => $Test, string => "STEP 7 of 13 : Verify that hammer is executing." ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $return = hammer_status( Node => $Node, Test => $Test ); if ( $return == 0 ) { logcomment("$FILER_C: Hammer is running "); } else { logcomment("Hammer not running on $FILER_C"); } } #STEP 8 : Set scrub priority level nvme_eye_catcher( Test => $Test, string => "STEP 8 of 13 : Set scrub priority level" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); logcomment("Set scrub priority provided by user : $USER_LEVL"); $Api_Set_Obj->execute_raw_command( 'command' => "system node run -node local -command \"priv set -q test; options raid.scrub.perf_impact $USER_LEVL\"" ); logcomment("Scrub priority level set to : $USER_LEVL"); $Api_Set_Obj->execute_raw_command( 'command' => "system node run -node local -command \"priv set -q test; options raid.scrub.perf_impact\"" ); } #STEP 9 : Start a disk SCRUB / nvme_eye_catcher( Test => $Test, string => "STEP 9 of 13 : Start scrub" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); foreach my $aggr (@NON_RT_AGG) { logcomment("Start Scrub on Aggregate : $aggr"); $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate scrub -action start -aggregate $aggr" ); logcomment("Aggregate scrub started"); } logcomment("Scrub started "); $scrub_start = time; } #STEP 10: nvme_eye_catcher( Test => $Test, string => "STEP 10 of 13 : Chek IO stats" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Show IO statistics of all drives"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_raw_command( 'command' => "statistics show-periodic -iterations 15 -interval 2 -summary false" ); logcomment("End of IO statistics"); } #STEP 11 nvme_eye_catcher( Test => $Test, string => "STEP 11 of 13 : Execute aggregate Scrub for 12 Hrs " ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Execute aggregate Scrub for 12 Hrs, and check for IO in between"); my $timeout = $test_wait_time; my ( $lsec, $lmin, $lhrs ) = gmtime($timeout); logcomment("Test executin time : $lhrs HOURS $lmin MINUTES and $lsec SECONDS"); my $curtime = time; my $endtime = $curtime + $timeout; my $loop = 0; while ( $curtime < $endtime ) { logcomment("Current time : $curtime and Endtime :$endtime"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); logcomment("Check hammer Status"); my $return = hammer_status( Node => $Node, Test => $Test ); my $response = $Api_Set_Obj->hammer_status(); sleep 10; logcomment("Hammer Status - $response "); logcomment("Check IO statistics"); $Api_Set_Obj->execute_raw_command( 'command' => "statistics show-periodic -iterations 15 -interval 2 -summary false" ); logcomment("Wait for 5 min"); logcomment("Check Aggregate Scrub Status "); my $src_sta = $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate show-scrub-status -node $FILER_C" ); sleep 800; $curtime = time; } } #STEP 12 nvme_eye_catcher( Test => $Test, string => "STEP 12 of 13 : Abort Hammer and stop scrub" ); logcomment("Aborting hammer "); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); my $return = hammer_status( Node => $Node, Test => $Test ); my $response = $Api_Set_Obj->hammer_status(); logcomment("Hammer Status - $response "); my $response = $Api_Set_Obj->hammer_abort(); logcomment("Hammer thread stopped "); logcomment("Stop scrub on all aggregate"); foreach my $aggr (@NON_RT_AGG) { logcomment("Start Scrub on Aggregate : $aggr"); $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate scrub -action stop -aggregate $aggr" ); logcomment("Aggregate scrub stopped"); } $scrub_end = time; } my $total_scrub_time = $scrub_end - $scrub_start; my ( $lsec, $lmin, $lhrs ) = gmtime($total_scrub_time); logcomment("Scrub executed for : $lhrs HOURS $lmin MINUTES and $lsec SECONDS , along with hammer"); #STEP 13 nvme_eye_catcher( Test => $Test, string => "STEP 13 of 13 : Check IO stats" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Show IO statistics of all drives"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_raw_command( 'command' => "statistics show-periodic -iterations 15 -interval 2 -summary false" ); logcomment("End of IO statistics"); } } ##################################################################### # Cleanup - Post Process ##################################################################### sub cleanup() { $Test->nlog(" $filer_names - Clean up and post test process"); @Nodes = node_new_obj( node_array => [@Nodes] ); my $node_ref = \@Nodes; nvme_post_test( node_present => $node_ref, Test => $Test, change_state_to => "MAINT", filer_mode => $Mode ); return $TCD::PASS; }