#$Id: //depot/prod/test/Rsierranevada/storage/hdd/NADQ_SEA/FS/Common_Lib.pm#39 $ # # Library of utility functions for HDD qualification tests # # ## @summary Library of routines to help simply HDD qualfication test development ## @description Library of routines to help in simplify the verfication required for various test cases oh hdd test plan. ## ## @synopsis ## use Storage::Common_Lib; ## ## ## @author ## arunak@netapp.com #################################################################### ## ## Design guidelines to be followed ## ## All subroutine must start with nvme_ ## All Subriutine must return has with all details or values in output ## Each subroutine must use validate options ## Each subroutine must contain author and usage of subroutines ## Every optional value must contain default value assigned ## ## ################################################################### package Storage::NVMe_Common_Lib; use TCD; use CR::Setup::TCD qw(nlog); use strict; use Storage::Common_Lib; use Tharn; use NACL::C::StorageAggregate; use NACL::APISet; use Data::Dumper; use NACL::C::Disk; use NACL::C::Volume; use NACL::C::Aggregate; use NACL::C::Vserver; use NACL::C::VolumeMember; use NACL::CS::StorageAggregateMember; use NACL::MTask::EventLogDetector; use Utils; use HTML::Mail; use NACL::MTask::SCP; use NACL::C::Client; use NACL::STask::SecurityLogin; use NACL::STask::Node; use NACL::MTask::SystemLogDetector; use boot; use San; use List::Util qw( min max shuffle); use Utils qw(make_conn); use Storage::SASUtils qw(filer_cmd); use Storage::Misc qw(filer_cmd); use install; use misc qw(cli_localhost); use Storage::Storage qw(storage_show_adapter); use POSIX; use NACL::Transit; use Storage::SASUtils qw(disable_dbg_msgs); use HTML::Mail; use NACL::C::Node; use Tharn qw(sleep); use Tharn qw(param); use Hostrec; use Server; use Options; use Expect; use SayRetry; use mount; use NACL::MTask::EventLogDetector; use NACL::STask::StorageFailover; use NACL::CS::Cluster; use lib "/usr/PNATE/pnate/lib"; use PNATE; use lib "/usr/PNATE/nate/lib"; use NATE; use Sys::Hostname; my $params = NATE::ParamSet->new( global => 1 ); my $FILER = $params->get( 'FILER', default => undef ); my $Kernel = $params->get( 'KERNEL', default => 'SN' ); my $EMAIL = $params->get( 'EMAIL', default => undef ); my $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); my $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); my $RUNID = $params->get( 'RUNID', default => 'undef' ); my $STEST_FILE = $params->get( 'STEST_FILE', default => '' ); my $BOOT_MODE = $params->get( 'BOOT_MODE', default => undef ); my $Filer_mode = $params->get( 'Filer_mode', default => 'F' ); my $TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' ); my $GROUP_NO = $params->get( 'GROUP_NO', default => undef ); my $DRIVE_TYPE = $params->get( 'DRIVE_TYPE', default => '' ); my $ERT_VOL = $params->get( "ERT_VOL", default => '1' ); my $LOGDIR = $params->get( "LOGDIR", default => 'NA' ); my $PRE_DETAILS; my $POST_DETAILS; my @pre_disk_count; my @post_disk_count; my $pre_post_recon_copy; my %Non_Medium_Model; logcomment("Kernel $Kernel"); logcomment("Id $RUNID"); my @run_id = split( '/', $RUNID ); my $parse_id = $run_id[-1]; #LOCAL VAR my $SCSI_DISK_ERRORS = {}; our $GET_PHY_STATE; our $STO_DISK_SHOW = {}; my @Filers; if ( $FILER =~ /\,/ ) { @Filers = split( /\,/, $FILER ); } else { push( @Filers, $FILER ); } if ( $Kernel =~ /FS|LB/i ) { require NACL::C::StorageDiskPartition; require NACL::CS::StorageDiskPartition; } # Import exceptions # use NACL::APISet::Exceptions::ResponseException qw(:try); use NACL::APISet::Exceptions::CommandFailedException qw (:try); use NACL::APISet::Exceptions::MissingRequiredParamException qw(:try); use NACL::APISet::Exceptions::InvalidParamValueException qw(:try); use NACL::APISet::Exceptions::MethodNotFoundException qw(:try); use NACL::Exceptions::NoElementsFound qw(:try); use NATE::BaseException qw(:try); use NACL::Transit::Exceptions::Timeout qw(:try); use NACL::Transit::Exceptions::RetriesExceeded qw(:try); use NACL::Transit::Exceptions::TransitException qw(:try); use NACL::APISet::Exceptions::TimeoutException qw(:try); use NACL::Exceptions::EventCheckFailure qw(:try); use NACL::APISet::Exceptions::MgwdCrashedException qw(:try); use NACL::APISet::Exceptions::NoMatchingEntriesException qw(:try); #use NATE::Result::Fatal qw(:try); # use Params::Validate qw(validate OBJECT SCALAR HASHREF ARRAYREF); use Tharn qw( host logcomment ); BEGIN { Params::Validate::validation_options( on_fail => sub { my @msgs = @_; chomp(@msgs); Carp::confess( Tharn::script_result(@msgs) ); } ); } require Exporter; use vars ( '@ISA', '@EXPORT' ); @ISA = ('Exporter'); @EXPORT = qw( nvme_eye_catcher nvme_list_only_nvme nvme_get_all_disk nvme_map_fw_dsk_rev_model nvme_disk_fw_update_low_or_high nvme_create_aggregate nvme_check_non_root_aggr nvme_get_agg_dtls_n_create nvme_rename_fw_files nvme_get_disk_owned nvme_dsk_qual_outpt_parsed nvme_sto_agg_show_spare_dsk nvme_pre_test nvme_post_test nvme_halt_Node nvme_set_env_var nvme_boot_to_maint nvme_clean_config nvme_drv_assign nvme_pre_condition_drvs nvme_pre_con_locl_drv_sub nvme_boot_in_req_mode nvme_multi_aggr_with_rd_wr_vol_n_fsio nvme_get_local_spares nvme_get_drv_by_user_sel_mod_id boot_in_lemur lemur_get_drv_info lemur_get_smart_drv_info get_scm_flash_cache_drv nvme_report_failed_drives nvme_del_nonroot_stale_aggr_and_start_zeroing check_system_health dump_drv_failure_info scm_pre_condit_drv scm_pre_test scm_get_nvme_flash_cache_ctrl_id nvme_get_sk_trace_logs nvme_chk_unowned_drv nvme_get_tahiti_shelf_log ); sub scm_get_nvme_flash_cache_ctrl_id { my (%opts) = validate( @_, { Node => { type => OBJECT }, handle => { type => SCALAR, optional => 1, default => 0 }, } ); my $scm_dsk_result = {}; my $Node = $opts{Node}; my $FILER_C = $Node->name(); logcomment("$FILER_C : Check for SCM Flash Cache NVMe Drive"); my $Host = host($FILER_C); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); my $sysshell_out = $Api_Set_Obj->execute_raw_command( 'command' => "systemshell -node $FILER_C sudo nvmecontrol devlist" ); my ( $drv_n, $drvid, $drv_cap ); foreach my $line ( split( /\r?\n/, $sysshell_out ) ) { if ( $line =~ /nvme\d+/ ) { if ( $line =~ /(nvme\d+):\s+(.*)/ ) { $drv_n = $1; } if ( $line =~ /(\S+)\s+\((\S+)\)/ ) { $drvid = $1; $drv_cap = $2; } $scm_dsk_result->{$drv_n} = { 'DRIVEID' => $drv_n, 'DRVNAME' => $drvid, 'DRVCAP' => $drv_cap, } } } return $scm_dsk_result; } sub scm_pre_test { my (%opts) = validate( @_, { filer_mode => { type => SCALAR }, node_present => { type => ARRAYREF }, Test => { type => OBJECT }, handle => { type => SCALAR, optional => 1, default => 0 }, LOGDIR => { type => SCALAR, optional => 1, default => '' }, test_setup => { type => SCALAR, optional => 1, default => 'SCM' }, } ); my @Nodes; #handle_run_local_error(); if ( defined( @{ $opts{node_present} } ) ) { @Nodes = @{ $opts{node_present} }; } else { @Nodes = NACL::C::Node->find(); } Storage::Common_Lib::handle_run_local_error( Nodes => [@Nodes] ); my $Filer_mode = $opts{filer_mode}; my $Test = $opts{Test}; my $handle = $opts{handle}; my $flag = 0; my $test_setup = $opts{test_setup}; my $disk_capacity_total = 0; my $LOGDIR = $opts{LOGDIR}; my $params = NATE::ParamSet->new( global => 1 ); my $EOE = $params->get( 'EOE', default => 'default' ); my $EMAIL = $params->get( 'EMAIL', default => 'y' ); my $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); my $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); my @Filer_array = (); my $execution_state; my @disk_list; my $node_ref; my @disk_array; my %FILER_API = (); if ( $opts{LOGDIR} ) { $LOGDIR = $opts{LOGDIR}; } else { $LOGDIR = $ENV{HOME}; } my $TEST_NAME; opendir( DIR_LOG31, $LOGDIR ); my @files_DIR_LOG31 = readdir(DIR_LOG31); foreach my $file (@files_DIR_LOG31) { if ( ( $file =~ /(.*_NADQ02.*)\.log/ ) && ( $file !~ /results|END_FILER_CONFIG/ ) ) { $TEST_NAME = $1; } } $node_ref = \@Nodes; foreach my $temp_node (@Nodes) { my $Filer = $temp_node->name(); push( @Filer_array, $Filer ); } # Check state of filer and Execute based on state logcomment("Nodes present - @Filer_array"); logcomment("DEBUG:Log loction - $LOGDIR"); nvme_eye_catcher( Test => $Test, string => "@Filer_array : Starting PRE_TEST - SCM Specific" ); logcomment("Check for SCM FLASH CACHE DRIVE and get all logpage"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Host = host($FILER_C); logcomment("Executing PRE TEST on FILER : $FILER_C"); my $Api_Set_Obj_SS; my $execution_state = "CLI"; if ( $execution_state eq 'CLI' ) { my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); logcomment("Check Current Privilage Level "); $Api_Set_Obj->execute_raw_command( 'command' => "set" ); logcomment("Set privilage level to diag"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set diag", 'connectrec-match_table' => $prompts_answers ); logcomment("Privilage level diag set"); $Api_Set_Obj->execute_raw_command( 'command' => "set" ); logcomment("Get password to login in system shell"); my $Passwd = $Host->default_password(); logcomment("Filer password : $Passwd"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj_SS = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Systemshell" ); } else { my $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); my $Passwd = $Host->default_password(); my $prompts_answers = [ ".*password*" => "$Passwd" ]; $API_Object->execute_raw_command( 'command' => "systemshell", 'connectrec-match_table' => $prompts_answers ); $API_Object->execute_raw_command( 'command' => "\013" ); $Api_Set_Obj_SS = $API_Object; } logcomment("System shell API created, getting log pages "); my $flash_cache_drv = get_scm_flash_cache_drv( Node => $Node ); my @res_disks; @res_disks = keys $flash_cache_drv if ($flash_cache_drv); if (@res_disks) { logcomment("Flash Cache Drive present: @res_disks"); foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Get logpages for drive :$ctrl_id"); logcomment("Recording controller infromation on $ctrl_id"); $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol identify $ctrl_id" ); logcomment("Recorded controller infromation on $ctrl_id"); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording Error Information log page 1 data for drive $ctrl_id"); my $lgpg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 1 $ctrl_id" ); if ( $lgpg =~ /No error entries found/ ) { logcomment("NO LOG PAGE 1 ENTRIES FOUND for drive $ctrl_id"); } logcomment("Recorded Error Information log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording SMART/Health log page 2 data for drive $ctrl_id"); my $smlg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 2 $ctrl_id " ); logcomment("Recorded SMART/Health log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording Firmware log page 3 data for drive $ctrl_id "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 3 $ctrl_id " ); logcomment("Recorded Firmware log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording 0xC0 log page data for drive $ctrl_id "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 0xC0 $ctrl_id " ); logcomment("Recorded Firmware log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording 0xC0 log page data for drive $ctrl_id "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 0xCA $ctrl_id " ); logcomment("Recorded Firmware log page data for drive $ctrl_id "); } logcomment("$FILER_C : Basic log pages Completed for Flash Cache NVMe Drive"); # $Api_Set_Obj_SS->execute_raw_command( 'command' => "exit" ); } else { logcomment("SCM /Flash cachec drive not found "); } } } #################### SCM pre condition based on user choice #################### sub scm_pre_condit_drv { my (%opts) = validate( @_, { Node => { type => OBJECT }, handle => { type => SCALAR, optional => 1, default => 0 }, pre_condition_opt => { type => SCALAR }, scm_drive => { type => SCALAR, optional => 1, default => 0 }, runtime => { type => SCALAR, optional => 1, default => 3600 }, ctrl => { type => SCALAR, optional => 1, default => 0 }, namesp => { type => SCALAR, optional => 1, default => 0 }, lbas => { type => SCALAR, optional => 1, default => 256 }, } ); my $Node = $opts{Node}; my $handle = $opts{handle}; my $scm_drive = $opts{scm_drive}; my $runtime = $opts{runtime}; my $pre_condition_opt = $opts{pre_condition_opt}; my $FILER_C = $Node->name(); my $ctrl = $opts{ctrl}; my $namesp = $opts{namesp}; my $lbas = $opts{lbas}; my $flag = 0; logcomment("$FILER_C : Pre condition SCM Drive using -controller : $ctrl, -namespace : $namesp and -lbas-per-op : $lbas"); logcomment("$FILER_C : User selected pre condition option : $pre_condition_opt "); logcomment("$FILER_C : user selected runtime for pre-condition : $runtime seconds"); logcomment("$FILER_C : Pre condition SCM Drive using -controller : $ctrl, -namespace : $namesp and -lbas-per-op : $lbas"); my $cmd; if ( $pre_condition_opt =~ /random/ ) { logcomment("$FILER_C : User selected option : $pre_condition_opt"); $cmd = "system node nvme stress start -controller $ctrl -namespace $namesp -threads 16 -ops-per-thread 32 -rd-percent 0 -rd-verify false -lbas-per-op $lbas -runtime $runtime -rd-iter 0 -rd-op seq -wr-iter 0 -wr-op rand -wr-skip-freq 0 -wr-skip-amt 0"; } elsif ( $pre_condition_opt =~ /sequential/ ) { logcomment("$FILER_C : User selected option : $pre_condition_opt"); $cmd = "system node nvme stress start -controller $ctrl -namespace $namesp -threads 16 -ops-per-thread 32 -rd-percent 0 -rd-verify false -lbas-per-op $lbas -runtime $runtime -rd-iter 0 -rd-op seq -wr-iter 0 -wr-op seq -wr-skip-freq 0 -wr-skip-amt 0"; } else { logcomment("$FILER_C : User Selected not to pre condition"); } my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); my $time_out = 999999999999; $Api_Set_Obj->set_timeout( "connectrec-timeout" => $time_out ); logcomment("$FILER_C : Api Set Created "); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); sleep 2; $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); if ( $pre_condition_opt =~ /sequential|random/ ) { my $out = $Api_Set_Obj->execute_raw_command( 'command' => $cmd ); logcomment("$FILER_C : Pre condition started on -controller $ctrl -namespace $namesp with -lbas-per-op $lbas "); if ( $out =~ /Error|command failed/ ) { logcomment("$FILER_C : pre condition on -controller $ctrl -namespace $namesp with -lbas-per-op $lbas failed"); logcomment("**FAIL** : pre condition on -controller $ctrl -namespace $namesp with -lbas-per-op $lbas failed"); $flag = 1; } } else { logcomment("As user selected \'No Precondition\', Pre-condition will be skipped"); } return $flag; } #################### Prints one drive fialure info BURT #burt1168970 sub dump_drv_failure_info { my (%opts) = validate( @_, { Node => { type => OBJECT }, handle => { type => SCALAR, optional => 1, default => 0 }, filer_state => { type => SCALAR }, drive => { type => SCALAR }, } ); my $Node = $opts{Node}; my $handle = $opts{handle}; my $execution_state = $opts{filer_state}; my $drive = $opts{drive}; my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Api_Set_Obj_SS; my $drv_result = {}; logcomment("FAILED/BROKEN DRIVE : $drive"); logcomment("FILER STATE IS : $execution_state"); my $flag_cli = 0; if ( $execution_state eq 'CLI' ) { my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); logcomment("Check Current Privilage Level "); $Api_Set_Obj->execute_raw_command( 'command' => "set" ); logcomment("Set privilage level to diag"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set diag", 'connectrec-match_table' => $prompts_answers ); logcomment("Privilage level diag set"); $Api_Set_Obj->execute_raw_command( 'command' => "set" ); logcomment("Get password to login in system shell"); my $Passwd = $Host->default_password(); logcomment("Filer password : $Passwd"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj_SS = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Systemshell" ); } else { my $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); my $Passwd = $Host->default_password(); my $prompts_answers = [ ".*password*" => "$Passwd" ]; $API_Object->execute_raw_command( 'command' => "systemshell", 'connectrec-match_table' => $prompts_answers ); $API_Object->execute_raw_command( 'command' => "\013" ); $Api_Set_Obj_SS = $API_Object; } logcomment("System shell API created, getting log pages "); my $ss_disk_list = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol devlist" ); foreach my $line ( split( /\n/, $ss_disk_list ) ) { my ( $drv, $ns, $nsz ); if ( $line =~ /(\S+)\:\s+(\S+)/ ) { $drv = $1; $drv_result->{$drv}->{'PRDID'} = $2; } if ( $line =~ /(\S+)ns\d+/ ) { $drv = $1; if ( $line =~ /(\S+)\s+\((\S+)\)/ ) { $ns = $1; $nsz = $2; $drv_result->{$drv}->{'NAME_SP'} = $ns; $drv_result->{$drv}->{'NS_SIZE'} = $nsz; } } } my @nv_drv = keys(%$drv_result); logcomment("System Shell Drive(s) : @nv_drv"); my ($dd) = $drive =~ /\S+\.(\S+)/; logcomment("Drive ID : $dd"); $Api_Set_Obj_SS->set_timeout( "connectrec-timeout" => 1200000000 ); foreach my $nv_d (@nv_drv) { logcomment("DEBUG : Check for drive $nv_d"); my ($nv) = $nv_d =~ /.*nvme(\S+)/; logcomment("NVMe controller ID : $nv"); if ( $nv == $dd ) { if ( $execution_state eq 'CLI' ) { logcomment("dump failure info for drive $nv_d"); logcomment("Pull memory dumps from the failed drive $nv_d"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 120000 ); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "systemshell -node $FILER_C -command sudo peg_nvmecontrol vu_dump_ss retrieve -o memory $nv_d" ); sleep 2; logcomment("Pull debug dumps from failed drive $nv_d"); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 120000 ); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "systemshell -node $FILER_C -command sudo peg_nvmecontrol vu_dump_ss retrieve -o debug $nv_d" ); sleep 3; logcomment("Pull crash dumps from failed drive $nv_d"); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 120000 ); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "systemshell -node $FILER_C -command sudo peg_nvmecontrol vu_dump_ss retrieve -o crash $nv_d" ); $flag_cli = 1; } else { logcomment("Filer is in $execution_state state, Cannot dump logs"); last; } } } if ( $flag_cli == 1 ) { logcomment("Copy .dmp files to $LOGDIR"); my $Client = NACL::C::Client->find(); my $scp; my $flag; try { $scp = NACL::MTask::SCP->new( target_command_interface => $Node, source_command_interface => $Client, ); } catch NATE::BaseException with { $flag = 1; logcomment("$FILER_C : Could NOT create SCP object, please check the connections"); }; if ($flag) { return; } my $num; my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/; if ( $parse_id =~ /(\d+)\_\S+/ ) { $num = $1; } my $log_dir = "$actual_log_dir\/FAILED_DRIVE_LOGS\/" . $FILER_C; if ( !( -d "$log_dir" ) ) { system("sudo mkdir -p $log_dir"); } system("sudo chmod 777 $log_dir"); my $Filer_apiset = $Node->apiset( category => 'Node', interface => 'CLI', set => "Systemshell" ); my @filer_logfile_names = ('/mroot/etc/log/*.dmp'); foreach my $log_path (@filer_logfile_names) { my $File_on_filer; try { $File_on_filer = $Filer_apiset->execute_raw_command( command => "ls $log_path" ); if ( $File_on_filer =~ /No match|No such/i ) { logcomment("$FILER_C : NO $log_path files found on the filer"); next; } } catch NATE::BaseException with { logcomment("$FILER_C : $log_path CANNOT be accessed"); }; my @files = $File_on_filer =~ /(\S+)/sg; foreach my $file (@files) { next if ( $file =~ /core/ ); #Burt - NVMe - Ssytem logs huge #1167332 if ( $log_path =~ /crash/i ) { $file = $log_path . "/" . $file; } chomp($file); try { $scp->copy_from_target_to_source( 'files_to_copy' => [$file], set_directory_on_source => "$log_dir", ); } catch NATE::BaseException with { logcomment("$FILER_C : Could NOT copy file :$file"); }; } } } logcomment("Completed dumping failure info for failed drives"); } ### ONLY ON FILE SYSTEM MODE sub get_scm_flash_cache_drv { my (%opts) = validate( @_, { Node => { type => OBJECT }, handle => { type => SCALAR, optional => 1, default => 0 }, } ); my $Node = $opts{Node}; my $handle = $opts{handle}; my $FILER_C = $Node->name(); my $scm_dsk_result = {}; my $check = 0; logcomment("$FILER_C : Check for SCM Flash Cache NVMe Drive"); logcomment("$FILER_C : HANDLE : $handle"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); my $sys_out = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test; sysconfig -v\"" ); chomp( my @sysconf_lines = split( /\r?\n/, $sys_out ) ); foreach my $line (@sysconf_lines) { my $slot; if ( $line =~ /slot\s+(\S+)\:\s+Flash Cache NVMe/ ) { $slot = $1; logcomment("$FILER_C : Flash Cache drive found at slot $slot\n"); my ( $serial, $fw, $mod, $cap, $state ); my $slot_out = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test; sysconfig $slot\"" ); foreach my $line ( split( /\r?\n/, $slot_out ) ) { if ( $line =~ /Serial Number:\s+(\S+)/ ) { $serial = $1; } if ( $line =~ /Firmware Version:\s+(\S+)/ ) { $fw = $1; } if ( $line =~ /Model Name:\s+(\S+)/ ) { $mod = $1; } if ( $line =~ /Capacity:\s+(\S+)/ ) { $cap = $1; } if ( $line =~ /State:\s+(\S+)/ ) { $state = $1; } } my $sysshell_out = $Api_Set_Obj->execute_raw_command( 'command' => "systemshell -node $FILER_C sudo nvmecontrol devlist" ); my ( $drv_n, $drvid, $drv_cap ); foreach my $line ( split( /\r?\n/, $sysshell_out ) ) { if ( $line =~ /nvme1/ ) { if ( $line =~ /nvme1:\s+(.*)/ ) { $drv_n = $1; } if ( $line =~ /(\S+)\s+\((\S+)\)/ ) { $drvid = $1; $drv_cap = $2; } } } $scm_dsk_result->{$drvid} = { 'DRIVEID' => $drvid, 'FILER' => $FILER_C, 'SLOT' => $slot, 'VENDOR_N' => $drv_n, 'SYSHEL_CAP' => $drv_cap, 'SERIAL' => $serial, 'FIRMWARE' => $fw, 'MODEL' => $mod, 'CAPACITY' => $cap, 'STATE' => $state }; logcomment("$FILER_C :Captured SCM Flash Cache Drive "); logcomment( "$FILER_C : Dumper " . Dumper($scm_dsk_result) ); $check = 1; } } if ( $check == 1 ) { return $scm_dsk_result; } else { logcomment("FLASH CACHE NVME DRIVE IS NOT PRESENT"); return undef; } } sub lemur_get_smart_drv_info { my (%opts) = validate( @_, { Filer => { type => SCALAR }, handle => { type => SCALAR, optional => 1, default => 0 }, Test => { type => OBJECT }, } ); my $Test = $opts{Test}; my $handle = $opts{handle}; my $FILER_C = $opts{Filer}; my $prompt_fw; my $host = Tharn::host($FILER_C); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); logcomment("**DEBUG** : Create Server object"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); nvme_eye_catcher( Test => $Test, string => "List all nvme drives" ); my $output = $Server->console( cmd => "lemur nvme list", ignore => $ignore_strings, additional_prompts_timeout => "36000", additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "->" => "FINAL_PROMPT", "]->" => "FINAL_PROMPT", } ); my @lines = split( /\n/, $output ); my $fail_flag = 1; my @nvme_lem_drv; foreach my $line (@lines) { if ( $line =~ /(NVME-DRIVE.*)\(/ ) { logcomment("NVMe Drive - $1"); push( @nvme_lem_drv, $1 ); } if ( $line =~ /RESULT:(\S+)/ ) { logcomment("Drive capture result : $1"); if ( $1 =~ /PASSED/ ) { $fail_flag = 0; } else { $fail_flag = 1; } } } my $total_nvm_drv = @nvme_lem_drv; logcomment("Total Drive present : $total_nvm_drv "); logcomment("Drive are : @nvme_lem_drv"); my $dsk_smrt_result = {}; foreach my $drv (@nvme_lem_drv) { my $output = $Server->console( cmd => "lemur storage utility smart $drv", ignore => $ignore_strings, additional_prompts_timeout => "36000", additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "->" => "FINAL_PROMPT", "]->" => "FINAL_PROMPT", } ); my @lines = split( /\n/, $output ); foreach my $line (@lines) { if ( $line =~ /(.*)\:\s+(.*)/ ) { my $param = $1; my $val = $2; $dsk_smrt_result->{$drv}->{'CRIT_WARN'} = $val if ( $param =~ /critical_warning/ ); $dsk_smrt_result->{$drv}->{'TEMP'} = $val if ( $param =~ /temperature/ ); $dsk_smrt_result->{$drv}->{'AVL_SPR'} = $val if ( $param =~ /available_spare/ ); $dsk_smrt_result->{$drv}->{'AVL_SPR_THRES'} = $val if ( $param =~ /available_spare_threshold/ ); $dsk_smrt_result->{$drv}->{'PRE_USED'} = $val if ( $param =~ /percentage_used/ ); $dsk_smrt_result->{$drv}->{'DAT_UN_READ'} = $val if ( $param =~ /data_units_read/ ); $dsk_smrt_result->{$drv}->{'DAT_UN_WRITE'} = $val if ( $param =~ /data_units_written/ ); $dsk_smrt_result->{$drv}->{'HOST_READ'} = $val if ( $param =~ /host_read_commands/ ); $dsk_smrt_result->{$drv}->{'HOST_WRITE'} = $val if ( $param =~ /host_write_commands/ ); $dsk_smrt_result->{$drv}->{'CTRL_BUSY_TIME'} = $val if ( $param =~ /controller_busy_time/ ); $dsk_smrt_result->{$drv}->{'PWR_CYCL'} = $val if ( $param =~ /power_cycles/ ); $dsk_smrt_result->{$drv}->{'POH'} = $val if ( $param =~ /power_on_hours/ ); $dsk_smrt_result->{$drv}->{'UNSAFE_SHUTDW'} = $val if ( $param =~ /unsafe_shutdowns/ ); $dsk_smrt_result->{$drv}->{'MEDIA_ERROR'} = $val if ( $param =~ /media_errors/ ); $dsk_smrt_result->{$drv}->{'ERR_LOG_ENTR'} = $val if ( $param =~ /num_err_log_entries/ ); $dsk_smrt_result->{$drv}->{'WARN_TEMP_TIME'} = $val if ( $param =~ /Warning Temperature Time/ ); $dsk_smrt_result->{$drv}->{'CRIT_TEMP'} = $val if ( $param =~ /Critical Composite Temperature Time/ ); $dsk_smrt_result->{$drv}->{'TEMP_SENSOR_1'} = $val if ( $param =~ /Temperature Sensor 1/ ); $dsk_smrt_result->{$drv}->{'TEMP_SENSOR_2'} = $val if ( $param =~ /Temperature Sensor 2/ ); $dsk_smrt_result->{$drv}->{'TEMP_SENSOR_3'} = $val if ( $param =~ /Temperature Sensor 3/ ); $dsk_smrt_result->{$drv}->{'TEMP_SENSOR_4'} = $val if ( $param =~ /Temperature Sensor/ ); $dsk_smrt_result->{$drv}->{'TEMP_SENSOR_5'} = $val if ( $param =~ /Temperature Sensor 5/ ); $dsk_smrt_result->{$drv}->{'TEMP_SENSOR_6'} = $val if ( $param =~ /Temperature Sensor 6/ ); $dsk_smrt_result->{$drv}->{'TEMP_SENSOR_7'} = $val if ( $param =~ /Temperature Sensor 7/ ); $dsk_smrt_result->{$drv}->{'TEMP_SENSOR_8'} = $val if ( $param =~ /Temperature Sensor 8/ ); } else { #logcomment("Some of the parameters could not be captured for drive : $drv :: $line"); } } } logcomment("FLAG : $fail_flag"); return $dsk_smrt_result; } sub lemur_get_drv_info { my (%opts) = validate( @_, { Filer => { type => SCALAR }, handle => { type => SCALAR, optional => 1, default => 0 }, Test => { type => OBJECT }, } ); my $Test = $opts{Test}; my $handle = $opts{handle}; my $FILER_C = $opts{Filer}; my $prompt_fw; my $host = Tharn::host($FILER_C); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); logcomment("**DEBUG** : Create Server object"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); nvme_eye_catcher( Test => $Test, string => "List all nvme drives" ); my $output = $Server->console( cmd => "lemur nvme list", ignore => $ignore_strings, additional_prompts_timeout => "36000", additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "->" => "FINAL_PROMPT", "]->" => "FINAL_PROMPT", } ); my @lines = split( /\n/, $output ); my $fail_flag = 1; my @nvme_lem_drv; foreach my $line (@lines) { if ( $line =~ /(NVME-DRIVE.*)\(/ ) { logcomment("NVMe Drive - $1"); push( @nvme_lem_drv, $1 ); } if ( $line =~ /RESULT:(\S+)/ ) { logcomment("Drive capture result : $1"); if ( $1 =~ /PASSED/ ) { $fail_flag = 0; } else { $fail_flag = 1; } } } my $total_nvm_drv = @nvme_lem_drv; logcomment("Total Drive present : $total_nvm_drv "); logcomment("Drive are : @nvme_lem_drv"); my $dsk_result = {}; foreach my $drv (@nvme_lem_drv) { my $output = $Server->console( cmd => "lemur nvme show all $drv", ignore => $ignore_strings, additional_prompts_timeout => "36000", additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "->" => "FINAL_PROMPT", "]->" => "FINAL_PROMPT", } ); my @lines = split( /\n/, $output ); foreach my $line (@lines) { if ( $line =~ /(.*)\:\s+(.*)/ ) { my $param = $1; my $val = $2; $dsk_result->{$drv}->{'NODE_NAME'} = $val if ( $param =~ /Node Name/ ); $dsk_result->{$drv}->{'BLOCK_NAME'} = $val if ( $param =~ /Block Node Name/ ); $dsk_result->{$drv}->{'MODEL_NAME'} = $val if ( $param =~ /Model Name/ ); $dsk_result->{$drv}->{'SERIAL'} = $val if ( $param =~ /Serial Number/ ); $dsk_result->{$drv}->{'HW_VER'} = $val if ( $param =~ /HW Version/ ); $dsk_result->{$drv}->{'FW_VER'} = $val if ( $param =~ /FW Version/ ); $dsk_result->{$drv}->{'PCI_BDF'} = $val if ( $param =~ /PCI BDF/ ); $dsk_result->{$drv}->{'PCI_SPEED'} = $val if ( $param =~ /PCI Speed/ ); $dsk_result->{$drv}->{'PCI_WDT'} = $val if ( $param =~ /PCI Width/ ); $dsk_result->{$drv}->{'NAMESPACE_ID'} = $val if ( $param =~ /Namespace ID/ ); $dsk_result->{$drv}->{'USER_CAP'} = $val if ( $param =~ /User Capacity/ ); $dsk_result->{$drv}->{'BLOCK_SIZE'} = $val if ( $param =~ /Block Size/ ); $dsk_result->{$drv}->{'EU_ID'} = $val if ( $param =~ /EUI64/ ); $dsk_result->{$drv}->{'TEMP'} = $val if ( $param =~ /Temperature/ ); } else { logcomment("Some of the parameters could not be captured for drive : $drv :: $line"); } } } logcomment("FLAG : $fail_flag"); =head foreach my $dsk ( keys %$dsk_result ) { my $disk_size = $dsk_result->{$dsk}->{'FW_VER'}; my $bps = $dsk_result->{$dsk}->{'SERIAL'}; my $mod = $dsk_result->{$dsk}->{'MODEL_NAME'}; nvme_eye_catcher ( Test => $Test, string => "Drive : $dsk | FW : $disk_size | SERIAL : $bps | MODEL : $mod "); } =cut return $dsk_result; } sub boot_in_lemur { my (%opts) = validate( @_, { Filer => { type => SCALAR }, Image => { type => SCALAR }, handle => { type => SCALAR, optional => 1, default => 0 }, Test => { type => OBJECT }, } ); my $Test = $opts{Test}; my $handle = $opts{handle}; my $FILER_C = $opts{Filer}; my $LM_IMG = $opts{Image}; logcomment("**DEBUG** : Handle = $handle"); logcomment("Boot filer : $FILER_C in LEMUR using image :$LM_IMG"); nvme_eye_catcher( Test => $Test, string => "Change filer state to LOADER and configure e0M port" ); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Transit_obj->change_state( to => "FIRMWARE" ); sleep 60; my ( $host, $passwd, $mgmt_port, $mgmt_ip, $mgmt_mask, $mgmt_gw, $mgmt_domain ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /LOADER|FIRMWARE/ ) { logcomment("Filer :$FILER_C is in LOADER, Proceed with next steps"); $host = Tharn::host($FILER_C); $passwd = $host->{'default_password'}; $mgmt_port = $host->{'MGMT_PORT'}; $mgmt_ip = $host->{'MGMT_IP'}; $mgmt_mask = $host->{'MGMT_MASK'}; $mgmt_gw = $host->{'MGMT_GATEWAY'}; $mgmt_domain = "lab.netapp.com"; logcomment("$FILER_C : Management port : $mgmt_port"); logcomment("$FILER_C : Management IP : $mgmt_ip"); logcomment("$FILER_C : Management mask and gateway : $mgmt_mask and $mgmt_gw "); logcomment("$FILER_C : Domain : $mgmt_domain "); } else { logcomment("**FATAL** : Filer : $FILER_C is not in FIRMWARE or LOADER "); logresult( "FATAL", "Filer : $FILER_C is not in FIRMWARE or LOADER " ); return $TCD::FAIL; } nvme_eye_catcher( Test => $Test, string => "Set environment variables" ); my $prompt_fw; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); logcomment("**DEBUG** : Create Server object"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); my $env_1 = "LINUX_BOOTARGS \"console=ttyS0,115200 console=tty0 mem=7936m intel_iommu=off acpi_rsdp=\$bootarg.acpi_rsdp nopat\""; logcomment("Configure $mgmt_port using ifconfig"); my $ifconfig = "ifconfig $mgmt_port -addr=$mgmt_ip -mask=$mgmt_mask -gw=$mgmt_gw -domain=$mgmt_domain -dns=172.19.2.30"; logcomment("Ifconfig command : $ifconfig"); logcomment("DEBUG:Set execution server"); $Server->set_execution_server( -Server => $FILER_C ); logcomment("$FILER_C : Setenv : $env_1"); $Server->console( cmd => "setenv $env_1", ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "::>" => "FINAL_PROMPT", } ); logcomment("Configure $mgmt_port using ifconfig"); sleep 10; $Server->console( cmd => "$ifconfig", ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "::>" => "FINAL_PROMPT", } ); sleep 10; logcomment("set newtork device eth1"); $Server->console( cmd => "setenv NETWORK_DEVICE eth1", ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "::>" => "FINAL_PROMPT", } ); sleep 10; nvme_eye_catcher( Test => $Test, string => " Boot LEMUR image " ); logcomment("LEMUR IMAGE : $LM_IMG"); my $boot_img = "boot -bzimage " . "$LM_IMG"; logcomment("Ignore- @$ignore_strings"); $Server->console( cmd => "$boot_img", ignore => $ignore_strings, additional_prompts_timeout => "36000", additional_prompts => { "]->" => "FINAL_PROMPT", } ); sleep 10; logcomment("Filer :$FILER_C is in LEMUR mode "); nvme_eye_catcher( Test => $Test, string => "Set $mgmt_port address" ); my $addr = "ifconfig i210-ob1p0 $mgmt_ip netmask $mgmt_mask\;"; $Server->console( cmd => "$addr", ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "->" => "FINAL_PROMPT", "]->" => "FINAL_PROMPT", } ); sleep 10; my $rout = "route add default gw $mgmt_gw"; $Server->console( cmd => "$rout", ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "->" => "FINAL_PROMPT", "]->" => "FINAL_PROMPT", } ); sleep 10; logcomment("Verify configured route "); $Server->console( cmd => "ifconfig i210-ob1p0; route", ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "->" => "FINAL_PROMPT", "]->" => "FINAL_PROMPT", } ); } sub nvme_get_drv_by_user_sel_mod_id { my (%opts) = validate( @_, { API_con => { type => OBJECT }, Node => { type => OBJECT }, handle => { type => SCALAR, optional => 1, default => 0 }, model => { type => SCALAR }, local_name => { type => SCALAR }, Filer => { type => SCALAR }, } ); my $Api_Set_Obj = $opts{API_con}; my $handle = $opts{handle}; my $FILER_C = $opts{Filer}; my $MODEL = $opts{model}; my $local_name = $opts{local_name}; my $Node = $opts{Node}; my @drv_2_return; logcomment(" $FILER_C : Get all drives "); my $disk_list_result = nvme_get_all_disk( node_present => [$Node] ); my @dsk = keys %$disk_list_result; my @disk_info = sort(@dsk); my @user_sel_drv; logcomment("Drive Present : @disk_info"); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); if ( $filer_state =~ /CLI|UP/ ) { logcomment("Get spare belongs to $FILER_C"); my @local_spare_drv = nvme_get_local_spares( API_con => $Api_Set_Obj, Filer => $FILER_C ); logcomment("$FILER_C : Spare drives are @local_spare_drv"); logcomment("USER SEELCTED MODEL : $MODEL"); foreach my $dsk (@disk_info) { my $model_name = $disk_list_result->{$dsk}->{'MODEL'}; if ( $model_name =~ /$MODEL/ ) { push( @user_sel_drv, $dsk ); } } logcomment("Drives with model : $MODEL are :"); logcomment("@user_sel_drv"); my @drv_node_scp; if ( $local_name =~ /yes/i ) { logcomment("Retrun drives with local name"); return @user_sel_drv; } else { logcomment("Get node scope name and return drives"); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200000 ); $Api_Set_Obj->execute_raw_command( command => "run local \"set -rows 0;system timeout modify 0\"" ); my $spare_list = nvme_sto_agg_show_spare_dsk( Node => $Node ); @drv_node_scp = keys %$spare_list; logcomment("Drive Belongs to Filer : $FILER_C : @drv_node_scp"); foreach my $drv (@drv_node_scp) { if ( $drv =~ /\w+\.\w+\.\w+/ ) { my ($chk) = $drv =~ /\w+\.\w+\.(\w+)/; foreach my $usr_sel (@user_sel_drv) { my ($matc) = $usr_sel =~ /\w+\.(\S+)/; if ( $matc == $chk ) { push( @drv_2_return, $drv ); } } } elsif ( $drv =~ /\w+\w+\:\w+\.\w+/ ) { my ($chk) = $drv =~ /\w+\w+\:\w+\.(\w+)/; foreach my $usr_sel (@user_sel_drv) { my ($matc) = $usr_sel =~ /\w+\.(\S+)/; if ( $matc == $chk ) { push( @drv_2_return, $drv ); } } } } @drv_2_return = unique(@drv_2_return); return @drv_2_return; } } elsif ( $filer_state =~ /MAINT/ ) { logcomment("Filer is in Maintenece mode - Get drives owned by node"); logcomment("USER SEELCTED MODEL : $MODEL"); my @fil_drvs = nvme_get_disk_owned( Node => $Node ); foreach my $dsk (@disk_info) { my $model_name = $disk_list_result->{$dsk}->{'MODEL'}; if ( $model_name =~ /$MODEL/ ) { push( @user_sel_drv, $dsk ); } } logcomment("Drives with model : $MODEL are : @user_sel_drv"); logcomment("$FILER_C : All Drives : @fil_drvs"); logcomment("$FILER_C : Check 1 - Drives user model selected returned - @user_sel_drv"); foreach my $all_drv (@fil_drvs) { foreach my $loc_own (@user_sel_drv) { if ( $all_drv eq $loc_own ) { logcomment("Drive with $MODEL owned by $FILER_C : $all_drv"); push( @drv_2_return, $all_drv ); } } } @drv_2_return = unique(@drv_2_return); return @drv_2_return; } } sub nvme_get_local_spares { my (%opts) = validate( @_, { API_con => { type => OBJECT }, handle => { type => SCALAR, optional => 1, default => 0 }, Filer => { type => SCALAR }, } ); my $Api_Set_Obj = $opts{API_con}; my $handle = $opts{handle}; my $FILER_C = $opts{Filer}; my @spare_disk_list; my @not_zeroed_spares; $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000 ); try { $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); my $output = $Api_Set_Obj->execute_command( 'command' => "aggr status -s" ); my $n = 0; my @lines = split( /\n/, $output ); while ( $n <= $#lines ) { last if ( $lines[$n] =~ /^-----/ ); $n++; } $n++; foreach my $line ( @lines[ $n .. $#lines ] ) { if ( $line =~ /spare\s+(\S+)/ ) { logcomment("Spare drive : $1"); push( @spare_disk_list, $1 ); } if ( $line =~ /spare\s+(\S+)\s*(.*)\s*\((.*)\)/ ) { logcomment("drive is : $1"); logcomment("zeroed or not zeroed drive ?: $3"); my $var = $3; push( @not_zeroed_spares, $1 ) if ( $var =~ /not zeroed/i ); } } } catch NACL::Exceptions::NoElementsFound with { logcomment("NO SPARE DISKS FOUND FOR THE FILER $FILER_C"); }; logcomment( "$FILER_C : The number of spare disks present are " . scalar @spare_disk_list . " and the disks are: @spare_disk_list " ) if (@spare_disk_list); logcomment("$FILER_C : No Spare drives found ") if ( !@spare_disk_list ); if ( $handle == 1 ) { if (@not_zeroed_spares) { while (1) { my $output = $Api_Set_Obj->execute_command( 'command' => "disk zero spares" ); sleep(60); my $out = $Api_Set_Obj->execute_command( 'command' => "aggr status -s" ); last if ( $out =~ /fast zeroed/i ); } } } elsif ( $handle == 2 ) { # return @not_zeroed_spares if (@not_zeroed_spares); return @not_zeroed_spares; } return @spare_disk_list; } sub nvme_multi_aggr_with_rd_wr_vol_n_fsio { my (%opts) = validate( @_, { API_con => { type => OBJECT }, filer => { type => SCALAR }, fsio => { type => SCALAR, optional => 1, default => 30 }, dset => { type => SCALAR, optional => 1, default => 50 }, loc_drvs => { type => ARRAYREF }, handle => { type => SCALAR, optional => 1, default => 0 }, Node => { type => OBJECT }, } ); my $Api_Set_Obj = $opts{API_con}; my @local_sp_drv = @{ $opts{loc_drvs} }; my $fsio_inst = $opts{fsio}; my $data_set = $opts{dset}; my $total_spare = @local_sp_drv; my $handle = $opts{handle}; my $drv_per_aggr = 5; my $spare_buffer; my $aggr_flag = 0; if ( $handle =~ /custom/ ) { $spare_buffer = 1; } else { $spare_buffer = 3; } logcomment("Custom Handle : $handle"); my $FILER_C = $opts{filer}; my $Node = $opts{Node}; my $FILERSIO_THRDS = {}; my @fsio_cmd; my $drv_count = $total_spare - $spare_buffer; $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000 ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); sleep 30; $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( 'command' => "event config modify -console-log-level INFO -console off" ); logcomment("Total Spare Available on $FILER_C : $total_spare"); logcomment("$FILER_C : Spare drives are : @local_sp_drv"); logcomment("Pre-defined total spares to be left : $spare_buffer"); logcomment("Pre-defined total drives per aggregate : $drv_per_aggr"); logcomment("User entered filersio instance : $fsio_inst"); logcomment("User entered data set size : $data_set"); logcomment("$FILER_C : $drv_count drives will be used to create aggregate"); logcomment("Aggregate(s) of $drv_per_aggr drives must be created"); my $aggr_count = int( $drv_count / $drv_per_aggr ); logcomment("$FILER_C : Total $aggr_count aggregate with $drv_per_aggr drives will be created"); my $rd_vol_perc = 0.05; #5% of read my $wr_vol_perc = 0.85; #85% of write logcomment("Pre-defined - 5% of read volume and 85% of write volume will be created from availabel aggregate(s) size"); if ( $aggr_count <= 0 ) { logcomment("**FATAL** : $FILER_C : Minimum 5 non-spare drives is needed, Total Spare :$total_spare , Reserved Spare :$spare_buffer, Remaiing Drives:$drv_count"); logresult( "FATAL", " $FILER_C : Minimum 5 non-spare drives is needed, Total Spare :$total_spare , Reserved Spare :$spare_buffer, Remaiing Drives:$drv_count" ); return 0; } my @non_root_aggrs; my @all_vols; sleep 20; $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); if ( $handle =~ /custom/ ) { logcomment("Single Aggregate with Specified drives will be created"); logcomment("Total Spare :$total_spare "); logcomment("Spare Drives : @local_sp_drv "); my $i = 1; my $aggr_name = "aggr_io_" . "$FILER_C" . "_" . "$i"; $aggr_name =~ s/\-/_/g if ( $aggr_name =~ /\-/ ); logcomment("Aggregate $aggr_name will be created on $FILER_C"); try { logcomment("$FILER_C : Creating aggregate $aggr_name... "); my $job; my $stask_aggr_obj = NACL::STask::Aggregate->create( command_interface => $Node, aggregate => $aggr_name, node => $FILER_C, nacltask_if_exists => 'reuse', job_component => \$job, disklist => \@local_sp_drv, ); } catch NACL::APISet::Exceptions::MgwdCrashedException with { logcomment("Management gateway restarted in between, Check for any coredump"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*to page down, for next line, or 'q' to quit.*" => 'q' ]; $Api_Set_Obj->execute_raw_command( command => "set -rows 0;system node coredump show -node local -type application -corename mgwd.* -instance", 'connectrec-match_table' => $prompts_answers ); logcomment("Captured system node core dump for management gateway"); $aggr_flag = 1; } catch NATE::BaseException with { logcomment("Management gate way restarted, Re-try again "); sleep 30; my $prompts_answers = [ ".*to page down, for next line, or 'q' to quit.*" => 'q' ]; $Api_Set_Obj->execute_raw_command( command => "set -rows 0;system node coredump show -node local -type application -corename mgwd.* -instance", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( command => " " ); $aggr_flag = 1; }; if ( $aggr_flag == 1 ) { logcomment("**FATAL** : $FILER_C : Aggregate $aggr_name is NOT created ."); logresult( "FATAL", " $FILER_C : Aggregate $aggr_name is NOT created ." ); return 0; } else { logcomment(" $FILER_C : Aggregate $aggr_name created."); sleep 20; } } else { for ( my $i = 1 ; $i <= $aggr_count ; $i++ ) { $aggr_flag = 0; $Node->refresh_command_interface(); my $aggr_name = "aggr_io_a_" . "$FILER_C" . "_" . "$i"; $aggr_name =~ s/\-/_/g if ( $aggr_name =~ /\-/ ); logcomment("Aggregate $aggr_name will be created on $FILER_C"); try { logcomment("$FILER_C : Creating aggregate $aggr_name... "); my $job; my $stask_aggr_obj = NACL::STask::Aggregate->create( command_interface => $Node, aggregate => $aggr_name, node => $FILER_C, nacltask_if_exists => 'reuse', job_component => \$job, diskcount => 5, ); } catch NACL::APISet::Exceptions::MgwdCrashedException with { logcomment("Management gateway restarted in between, Check for any coredump"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*to page down, for next line, or 'q' to quit.*" => 'q' ]; $Api_Set_Obj->execute_raw_command( command => "set -rows 0;system node coredump show -node local -type application -corename mgwd.* -instance", 'connectrec-match_table' => $prompts_answers ); logcomment("Captured system node core dump for management gateway"); $aggr_flag = 1; } catch NATE::BaseException with { logcomment("Management gate way restarted, Re-try again "); sleep 30; my $prompts_answers = [ ".*to page down, for next line, or 'q' to quit.*" => 'q' ]; $Api_Set_Obj->execute_raw_command( command => "set -rows 0;system node coredump show -node local -type application -corename mgwd.* -instance", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( command => " " ); $aggr_flag = 1; }; if ( $aggr_flag == 1 ) { logcomment("**FATAL** : $FILER_C : Aggregate $aggr_name is NOT created."); logresult( "FATAL", " $FILER_C : Aggregate $aggr_name is NOT created." ); return 0; } else { logcomment(" $FILER_C : Aggregate $aggr_name created."); sleep 20; } } } sleep 30; logcomment("Get all aggregate created details"); $Api_Set_Obj->execute_raw_command( 'command' => " " ); sleep 20; re_ex: $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); my $output = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;aggr show -root false -node $FILER_C" ); foreach my $line ( split( /\n/, $output ) ) { next if ( $line =~ /RAID\s+Status/ ); if ( $line =~ /no entries matching/ ) { logcomment("NO NON-ROOT Aggregate FOUND\n"); } elsif ( $line =~ /(aggr.*)\s+(\S+)\s+(\S+)\s+(\S+)\s+(online)\s+(\S+)\s+(\S+)\s+(\S+)/ ) { logcomment("Aggregate : $1"); push( @non_root_aggrs, $1 ); } } logcomment("Create test vserver for @non_root_aggrs aggregates "); if ( !@non_root_aggrs ) { logcomment("Aggregate cannot be captured, Re-trying...."); goto re_ex; } my $count = 1; $FILERSIO_THRDS->{$FILER_C}->{'NON_ROOT_AGGR'} = [@non_root_aggrs]; foreach my $aggregate_name (@non_root_aggrs) { logcomment("Create vserver , read and write volumes for aggregate - $aggregate_name"); my $vserver_name = "vser_test_" . "$FILER_C" . "_" . "$count"; $vserver_name =~ s/\-/_/g if ( $vserver_name =~ /\-/ ); logcomment("$FILER_C : Creating vserver $vserver_name..."); my $root = "vol0"; my $ns = "file"; my $sec = "unix"; my $vserver_obj = NACL::STask::Vserver->create( command_interface => $Node, vserver => $vserver_name, subtype => 'default', rootvolume => $root, aggregate => $aggregate_name, "rootvolume-security-style" => $sec, nacltask_if_exists => 'reuse', ); logcomment("Created test vserver for all aggregates "); my $output = $Api_Set_Obj->execute_raw_command( 'command' => "vserver show" ); if ( $output =~ /command failed/i ) { logcomment("**FATAL** : $FILER_C : Vserver $vserver_name is NOT created."); logresult( "FATAL", " $FILER_C : Vserver $vserver_name is NOT created.." ); return 0; } else { logcomment("Created test vserver $vserver_name for all aggregates "); } my $aggr_size; my $output = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0; aggr show $aggregate_name -fields availsize" ); foreach my $line ( split( /\n/, $output ) ) { if ( ( $line =~ /$aggregate_name\s+(\S+)[GT]B/i ) || ( $line =~ /$aggregate_name\s+(\S+)\s+[GT]B/i ) ) { $aggr_size = $1; $aggr_size = $aggr_size * 1024 if ( $line =~ /^$aggregate_name\s+(\S+)TB/i ); my ($val) = readable_disk_size( GB => $aggr_size ); logcomment("$FILER_C : Size of created aggregate --> $val"); my $ag_wr_per = $rd_vol_perc * 100; my $ag_rd_per = $wr_vol_perc * 100; logcomment("Two volumes has to be created write volume of $ag_wr_per% and read volume of $ag_rd_per%"); my $wrt_vol = int( $aggr_size * $wr_vol_perc ); my $rd_vol = int( $aggr_size * $rd_vol_perc ); logcomment("Write volume of size : $wrt_vol and Read volume of size : $rd_vol will be created"); logcomment("disabling volume policy"); $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule daily -newcount 0" ); $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule weekly -newcount 0" ); $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify-schedule -policy default -schedule hourly -newcount 0" ); $Api_Set_Obj->execute_raw_command( 'command' => "volume snapshot policy modify -policy default -enabled false" ); logcomment("Disabled snapshot policy"); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); my ( $volsize_g, $vol_name ); for ( my $v = 1 ; $v <= 2 ; $v++ ) { $volsize_g = $wrt_vol . "g" if ( $v == 2 ); $volsize_g = $rd_vol . "g" if ( $v == 1 ); my $vol_wr = "vol_write_$v" . "_" . "$FILER_C" . "_ag_" . "$count" if ( $v == 2 ); my $vol_rd = "vol_read_$v" . "_" . "$FILER_C" . "_ag_" . "$count" if ( $v == 1 ); $vol_rd =~ s/\-/_/g if ( $vol_rd =~ /\-/ ); $vol_wr =~ s/\-/_/g if ( $vol_wr =~ /\-/ ); $vol_name = $vol_wr if ( $v == 2 ); $vol_name = $vol_rd if ( $v == 1 ); logcomment("DEBUG : Volume creation details : $v : vol name : $vol_name vol size : $volsize_g"); my $cmd = "vol create -vserver $vserver_name -volume $vol_name -aggregate $aggregate_name -size $volsize_g -state online -policy default -unix-permissions ---rwxr-xr-x"; logcomment("Creating volume : $vol_name of size $volsize_g on aggregate $aggregate_name using vserver $vserver_name"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "$cmd", 'connectrec-match_table' => $prompts_answers ); logcomment("Created $vol_name on aggregate : $aggregate_name"); sleep 30; push( @all_vols, $vol_name ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); } } } sleep 30; $count++; sleep 30; } logcomment("Volumes created are : @all_vols"); sleep 30; foreach my $vol_name (@all_vols) { logcomment("Get volume size and create file to read and initiate filersio threads on both read and write volume"); logcomment("DEBUG : Checking ofr volume : $vol_name"); if ( $vol_name =~ /vol_read/ ) { logcomment("$FILER_C : Executing df on volume $vol_name"); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); sleep 20; $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); my $df_output = $Api_Set_Obj->execute_raw_command( 'command' => "run local df $vol_name" ); my $vol_size_df; foreach my $line ( split( /\n/, $df_output ) ) { next if ( $line =~ /snapshot/ ); if ( $line =~ /vol_read/ ) { ($vol_size_df) = $line =~ /\S+\s+\d+\s+\d+\s+(\d+)/i; } } logcomment("DF out for volume : $vol_name : $vol_size_df"); logcomment("User entered total filersio thread or instances : $fsio_inst"); my $rd_fsio_isnt = $fsio_inst * $rd_vol_perc; logcomment("Total filer sio threads on read volume : $rd_fsio_isnt"); $vol_size_df = int( $vol_size_df / $rd_fsio_isnt ); my ($val) = readable_disk_size( KB => $vol_size_df ); logcomment("DEBUG : HARDCODED FILE SIZe 2G for Read files"); $val = "2G"; logcomment("DEBUG : file of size $val will be created on volume"); $val =~ s/( |B|\.\d+)//ig; for ( my $fi = 1 ; $fi <= $rd_fsio_isnt ; $fi++ ) { logcomment("Creating file : /vol/$vol_name/f_rd$fi"); $Api_Set_Obj->execute_raw_command( 'command' => "run local mkfile $val /vol/$vol_name/f_rd$fi" ); logcomment("file created"); logcomment("Initiate filersio instance on read volume : $vol_name"); my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio asyncio_active 100 -r 100 64k 0 $val -1 16 /vol/$vol_name/f_rd$fi" ); my $cmd = "run local filersio asyncio_active 100 -r 100 64k 0 $val -1 16 /vol/$vol_name/f_rd$fi"; push( @fsio_cmd, $cmd ); if ( $output =~ /failed to start/ig ) { logcomment("**FAIL** : $FILER_C : FILERSIO FAILED TO START"); } } $FILERSIO_THRDS->{$FILER_C}->{'SIO_CMD'} = [@fsio_cmd]; $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); } if ( $vol_name =~ /vol_write/ ) { logcomment("$FILER_C : Executing df on write volume : $vol_name"); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); my $df_output = $Api_Set_Obj->execute_raw_command( 'command' => "run local df $vol_name" ); my $vol_size_df; foreach my $line ( split( /\n/, $df_output ) ) { next if ( $line =~ /snapshot/ ); if ( $line =~ /vol_write/ ) { ($vol_size_df) = $line =~ /\S+\s+\d+\s+\d+\s+(\d+)/i; } } logcomment("DF out for volume : $vol_name : $vol_size_df"); logcomment("User entered total filersio thread or instances : $fsio_inst"); my $wr_fsio_isnt = $fsio_inst * $wr_vol_perc; logcomment("Total filer sio threads on write volume : $wr_fsio_isnt"); $vol_size_df = int( $vol_size_df / $wr_fsio_isnt ); my ($val) = readable_disk_size( KB => $vol_size_df ); $val =~ s/( |B|\.\d+)//ig; $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); for ( my $wfi = 1 ; $wfi <= $wr_fsio_isnt ; $wfi++ ) { logcomment("Create file and starting fioersio on write volume"); my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local filersio asyncio_active 0 -r 0 64k 0 $val -1 16 /vol/$vol_name/f$wfi -create" ); my $cmd = "run local filersio asyncio_active 0 -r 0 64k 0 $val -1 16 /vol/$vol_name/f$wfi -create"; push( @fsio_cmd, $cmd ); logcomment("Created filersio on write"); } $FILERSIO_THRDS->{$FILER_C}->{'SIO_CMD'} = [@fsio_cmd]; } sleep 60; } logcomment("Check for filersio status "); my $i = 0; my $filersio_stat = $Api_Set_Obj->execute_command( 'command' => "filersio status" ); if ( $filersio_stat =~ /filersio: instance $i: asyncio_active running/i ) { $i++; logcomment( "$FILER_C : Thread " . ( $i + 1 ) . " : filersio is running" ); } logcomment("Created vserver , read and write volumes for aggregate on filer $FILER_C"); sleep(20); return ($FILERSIO_THRDS); } sub nvme_boot_in_req_mode { my (%opts) = validate( @_, { Trans => { type => OBJECT }, boot_to => { type => SCALAR }, handle => { type => SCALAR, optional => 1, default => 0 }, } ); my $Transit_obj = $opts{Trans}; my $handle = $opts{handle}; my $BOOT_TO = $opts{boot_to}; logcomment("Required Boot state is $BOOT_TO"); $Transit_obj->set( 'timeout' => 7690000, maxcycle => 24 ); $Transit_obj->set( 'change_state_timeout' => 72000000 ); logcomment("DEBUG : Transit Object time set"); my $FILER_STATE = $Transit_obj->get_state( 'timeout' => 79000000, 'get_state_timeout' => 72000000 ); $Transit_obj->change_state( to => $BOOT_TO ); logcomment("DEBUG : Booted to $BOOT_TO"); logcomment("Verify filer state"); my $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 72000 ); logcomment("FILER is in $FILER_STATE state"); } sub nvme_pre_condition_drvs { my (%opts) = validate( @_, { Nodes => { type => ARRAYREF }, Filer_drv => { type => HASHREF, optional => 1, default => undef }, handle => { type => SCALAR, optional => 1, default => 0 }, } ); my @Nodes = @{ $opts{Nodes} }; my %FIL_DRV = %{ $opts{Filer_drv} } if ( $opts{Filer_drv} ); my $handle = $opts{handle}; my %FILER_API = (); my $FILER_O_STATE; my %FILER_DISK = (); my $status = 0; my $drvs = "all"; $drvs = "custom" if (%FIL_DRV); logcomment("DEBUG : CUSTOM FALG : $drvs drives"); logcomment("Get original state of filer before proceeding with preconditioning"); #logcomment( "Dumper " . Dumper(%FIL_DRV) ); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); logcomment("Get state of filer :$FILER_C"); $FILER_O_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("FILER- $FILER_C - Is in $FILER_O_STATE state"); if ( $FILER_O_STATE =~ /CLI|UP/ ) { my $API_obj_cli = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $API_obj_cli->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); } } logcomment("Filer was in $FILER_O_STATE state, It will be booted to MAINTENANCE mode after pre-conditioning"); $FILER_O_STATE = "MAINT"; logcomment("Filers will be booted to $FILER_O_STATE state after preconditioning"); logcomment("PRE CONDITIONING all the drives, This will boot in maintenance mode "); my @subtests_maint; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Node->refresh_command_interface(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); logcomment("Get state of filer :$FILER_C"); my $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("FILER- $FILER_C - Is in $FILER_STATE state"); if ( $FILER_STATE !~ /MAINT/ ) { push( @subtests_maint, subtest( \&nvme_boot_to_maint, -runid, "boot_in_main_$FILER_C", -bg, "--", Node => $Node ) ); } } Subtest::wait_finish( subtest => [@subtests_maint] ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Node->refresh_command_interface(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); logcomment("Checking filer state again "); my $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $FILER_STATE =~ /MAINT/ ) { my $host = host($FILER_C); my $API_Object = NACL::APISet->new( hostobj => $host, category => "Node", interface => "CLI", set => "Maintenance" ); $FILER_API{$FILER_C} = $API_Object; } logcomment("Get drives attached to filer $FILER_C"); my @temp_disk_info = nvme_get_disk_owned( Node => $Node ); @temp_disk_info = sort(@temp_disk_info); $FILER_DISK{$FILER_C} = [@temp_disk_info]; logcomment("Drive owned by $FILER_C : @temp_disk_info"); } my @subtests; if ( $drvs eq "all" ) { logcomment("PRE CONDITIONING ALL DRIVES .... "); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $FILER_C = $Node->name(); my @array = @{ $FILER_DISK{$FILER_C} }; my $API_Obj = $FILER_API{$FILER_C}; $API_Obj->execute_command( 'command' => 'disk_qual clear -d all' ); my @disk_array; grep { if ( $_ !~ /P\d+$/ ) { push( @disk_array, $_ ) } } @array; logcomment("$FILER_C : pre-conditioning drives : @disk_array"); push( @subtests, subtest( \&nvme_pre_con_locl_drv_sub, -runid, "pre_condit_drv_$FILER_C", -bg, "--", API => $API_Obj, Filer => $FILER_C, Drivs => \@disk_array, Node => $Node ) ); } } else { foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $FILER_C = $Node->name(); my @drvs = @{ $FIL_DRV{$FILER_C} }; my $API_Obj = $FILER_API{$FILER_C}; $API_Obj->execute_command( 'command' => 'disk_qual clear -d all' ); if ( !@drvs ) { logcomment("No Drives owned by $FILER_C, Check for next node"); next; } else { logcomment("$FILER_C : CUSTOM : Pre-conditioning drives : @drvs"); push( @subtests, subtest( \&nvme_pre_con_locl_drv_sub, -runid, "pre_condit_drv_$FILER_C", -bg, "--", API => $API_Obj, Filer => $FILER_C, Drivs => \@drvs, Node => $Node ) ); } last if ( $RUNID =~ /NVMe_Disk_IO_Latency/ ); } } Subtest::wait_finish( subtest => [@subtests] ); $status = status_return( subtest_ref => [@subtests] ); logcomment("DEBUG :pre-condition status is : $status"); my @subtests_post; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Node->refresh_command_interface(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); logcomment("Get state of filer :$FILER_C"); my $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("FILER- $FILER_C - Is in $FILER_STATE state"); logcomment("Filer was in $FILER_O_STATE state, Booting filer in original state : $FILER_O_STATE"); if ( $FILER_STATE !~ /$FILER_O_STATE/ ) { push( @subtests_post, subtest( \&nvme_boot_in_req_mode, -runid, "boot_back_to_org_$FILER_C", -bg, "--", Trans => $Transit_obj, boot_to => $FILER_O_STATE ) ); } } Subtest::wait_finish( subtest => [@subtests_post] ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Node->refresh_command_interface(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); logcomment("Checking filer state again "); my $FILER_STATE = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("FILER- $FILER_C - Booted in $FILER_STATE state"); } logcomment("PRE CONDITIONING of ALL DRIVES COMPLETED"); return $status; #1- Failure 0-Passed } sub nvme_pre_con_locl_drv_sub { my (%opts) = validate( @_, { API => { type => OBJECT }, Filer => { type => SCALAR }, Drivs => { type => ARRAYREF }, handle => { type => SCALAR, optional => 1, default => 0 }, Node => { type => OBJECT }, } ); logcomment("Pre-conditioning drives "); my $Node = $opts{Node}; my @drves = @{ $opts{Drivs} }; my $filer = $opts{Filer}; my $API_Obj = $opts{API}; my $handle = $opts{handle}; logcomment("$filer : @drves"); my $drv_pass = 0; logcomment("Starting - PRE-CONDITION for Drive : @drves"); my $dq_status = $API_Obj->execute_raw_command( 'command' => "disk_qual status" ); if ( $dq_status =~ /idle/ ) { logcomment("Disk qual status is idle, Proceed with pre-conditioning of drive @drves"); my $output = $API_Obj->execute_raw_command( 'command' => "disk_qual start -t 9 -d @drves -o noverify nopattern unmap" ); sleep(5); } else { logcomment("WARNING : Disk qual is not idle, Cannot proceed with pre-conditioning of dirve :@drves"); } my $time_out_temp = 0; my $max_time = 180000; $API_Obj->set_timeout( "connectrec-timeout" => $max_time ); while ( $time_out_temp < $max_time ) { my $after1 = $API_Obj->execute_raw_command( 'command' => "disk_qual status" ); if ( $after1 =~ /idle/ ) { $drv_pass = 1; logcomment("Precondition completed for all drives"); last; } else { logcomment("pre-conditioning of drive is in progress"); sleep(120); $time_out_temp = $time_out_temp + 120; } } #Burt1124013 -Starta $drv_pass = 0; my $time_out_temp_p2 = 0; my $max_time_p2 = 604800; #1 wk timeout $API_Obj->set_timeout( "connectrec-timeout" => $max_time_p2 ); logcomment("Starting - SEQUENTIAL WRITE with noverify nodata on all drives : @drves"); my $dq_status = $API_Obj->execute_raw_command( 'command' => "disk_qual status" ); if ( $dq_status =~ /idle/ ) { logcomment("Disk qual status is idle, Proceed with SEQUENTIAL WRITE with noverify nodata on all drives : @drves"); my $output = $API_Obj->execute_raw_command( 'command' => "disk_qual start -t 5 -n 2048 -q 32 -p 2 -d @drves -o noverify nodata" ); sleep(5); } else { logcomment("WARNING : Disk qual is not idle, Cannot proceed with pre-conditioning of dirve :@drves"); } while ( $time_out_temp_p2 < $max_time_p2 ) { my $after1 = $API_Obj->execute_raw_command( 'command' => "disk_qual status" ); if ( $after1 =~ /idle/ ) { $drv_pass = 1; logcomment("Precondition completed for all drives"); last; } elsif ( $after1 =~ /test failed/ ) { $after1 =~ /\[(\S+)\]\:\s+/; my $drv_failed = $1; logcomment("**WARNING** : PRE-CONDITION FAILED ON DRIVE : $drv_failed"); } else { logcomment("pre-conditioning of drive is in progress"); sleep(1800); $time_out_temp_p2 = $time_out_temp_p2 + 1800; } } logcomment("PRE-CONDITION of Drive @drves - Completed") if ( $drv_pass == 1 ); logcomment("PRE-CONDITION of Drive @drves - FAILED") if ( $drv_pass == 0 ); #Burt1124013 - End =head #Burt1124013 my $SGL = "bcs"; my $time_out_temp = 0; my $test_wait_time = 7200; my $flag; my %data_rate = (); my %rates = (); my $FILER_C = $filer; # determine data rate $API_Obj->execute_raw_command( command => "\013" ); $API_Obj->execute_raw_command( command => "\013" ); sleep 120; $API_Obj->execute_raw_command( command => "\013" ); logcomment("Determine data rate"); my $output = $API_Obj->execute_raw_command( command => "disk_latency -p seq_write -q 32 -t 60 -i 0 -m $SGL -l 832000 -s 133120 -f @drves" ); logcomment("==============================================================\n"); my @output_data = split( /\n/, $output ); foreach my $line (@output_data) { if ( ( $line =~ /(\w+\.\w+\.\w+)\s+\d+\s+(\d+)\s+\d+\s+\d+\s+\d+$/ ) || ( $line =~ /(\w+\.\w+)\s+\d+\s+(\d+)\s+\d+\s+\d+\s+\d+$/ ) ) { #Nvme my $dsk = $1; my $data_rate = $2; push( @{ $rates{$FILER_C}{$data_rate} }, $dsk ); last if ( $line =~ /^Latency Histogram:/i ); } } my @rate_arr = keys %{ $rates{$FILER_C} }; @rate_arr = sort { $a <=> $b } @rate_arr; my @arr_disk = @{ $rates{$FILER_C}{ $rate_arr[0] } }; my $dsk_last = $arr_disk[0]; my $drv; if ( ( $dsk_last =~ /\w+\.(\S+)/ ) || ( $dsk_last =~ /(\w+\.\w+)/ ) ) { $drv = $1; } logcomment("Disk - $drv :: Data rate - $rate_arr[0]"); my $ssd_capacity; sleep 60; my $disk_list_result = nvme_list_only_nvme( node_present => [$Node] ); foreach my $dsk ( keys %$disk_list_result ) { if ( $disk_list_result->{$dsk}->{'DISK'} =~ /$drv$/ ) { my $disk_size = $disk_list_result->{$dsk}->{'BLOCKS'}; my $bps = $disk_list_result->{$dsk}->{'BPS'}; $ssd_capacity = int( ( $disk_size * $bps ) / 1000000 ); last; } } if ( $rate_arr[0] < 500 ) { logcomment("**WARNING** : $FILER_C : Data rate: $rate_arr[0]MB/s, Data rate is less than 500MB/s"); } logcomment("Disk - $dsk_last :: Data rate - $rate_arr[0] :: Drive capacity - $ssd_capacity"); my $precond_time = int( 1.5 * ( $ssd_capacity / $rate_arr[0] ) ); logcomment(" Pre condition time - $precond_time"); # perform sequential precondition (q=32, s=128KB) logcomment("Perform sequential precondition (q=32, s=128KB)"); my $timeout = $precond_time + 7200; $API_Obj->set_timeout( "connectrec-timeout" => $timeout ); $API_Obj->execute_raw_command( command => "disk_latency -p seq_write -q 32 -t $precond_time -i 0 -m $SGL -s 133120 -f @drves" ); logcomment("Precondition completed"); =cut } sub nvme_drv_assign { my (%opts) = validate( @_, { Nodes => { type => ARRAYREF }, force => { type => SCALAR, optional => 1, default => 0 }, check => { type => SCALAR, optional => 1, default => 0 }, assign => { type => SCALAR, optional => 1, default => 'no' }, drv_asgn_A => { type => ARRAYREF, optional => 1, default => undef }, drv_asgn_B => { type => ARRAYREF, optional => 1, default => undef }, } ); my @Nodes = @{ $opts{Nodes} }; my $force = $opts{force}; my $check = $opts{check}; my $assign = $opts{assign}; my @Drv_A; my @Drv_B; my $custom = 0; if ( ( $opts{drv_asgn_A} ) || ( $opts{drv_asgn_B} ) ) { logcomment("Custom flage passed"); $custom = 1; @Drv_A = @{ $opts{drv_asgn_A} }; @Drv_B = @{ $opts{drv_asgn_B} }; } my $custom = 0; my $flag = 0; =head if ( $custom == 1 ) { my $cont = 1; logcomment("Custom flag is selected , Drives and system id has been passed, Assign drive directly"); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $Api_Set_Obj; logcomment("Get node name"); my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); my $Host = host($FILER_C); my $output; if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk show -a" ); } elsif ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $output = $Api_Set_Obj->execute_raw_command( command => "disk show -a" ); } my @Drvs = @Drv_A if ( $cont == 1 ); my @Drvs = @Drv_B if ( $cont == 2 ); foreach my $line ( split( /\n/, $output ) ) { if ( ( $line =~ /$FILER_C\s+(\S+)/ ) || ( $line =~ /Local System ID\:\s+(\S+)/ ) ) { my $sysid = $1; ($sysid) = $sysid =~ /\((\d+)\)/ if ( $sysid =~ /\(/ ); logcomment("Filer : $FILER_C and System ID : $sysid"); if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk assign @Drvs -s $sysid -o $FILER_C\"" ); $cont++; last; } elsif ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj->execute_raw_command( command => "disk assign @Drvs -s $sysid -o $FILER_C" ); $cont++; last; } } else { logcomment("Sysstem ID couldnt be captured, Cannot assign drives"); } } } } =cut if ( $force == 1 ) { logcomment("Mandatory force flage is set, Remove ownership forcefully"); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $Api_Set_Obj; logcomment("Get node name"); my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); my $output; my $Host = host($FILER_C); if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk show -n\"" ); } elsif ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "disk show -n" ); } $Api_Set_Obj->execute_raw_command( 'command' => "disk remove_ownership all -f" ); sleep 60; $Api_Set_Obj->execute_raw_command( 'command' => "disk remove_ownership all -f" ); $Api_Set_Obj->execute_raw_command( 'command' => "storage release disks -f" ); } logcomment("Removed all ownership forcefully, As force option is selected"); } if ( $custom == 0 ) { foreach my $Node (@Nodes) { my $Api_Set_Obj; $Node->refresh_command_interface(); logcomment("Get node name"); my $FILER_C = $Node->name(); logcomment("LOG DIR : $LOGDIR"); logcomment("Check for Unowned drives on setup $FILER_C"); logcomment("Check for any other parameter passed : $force and $check is passed"); logcomment("Check for drive details file in main log directory"); my $Home = $ENV{HOME}; my @unowned; my $unowned = 0; my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); my $output; my $Host = host($FILER_C); if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk show -n\"" ); } elsif ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "disk show -n" ); } foreach my $line ( split( /\n/, $output ) ) { if ( $line =~ /Not Owned|Unowned/ ) { if ( $line =~ /(\S+)\s+\S+\s+\S+\s+\S+/ ) { logcomment("DEBUG DRIVE : $1"); push( @unowned, $1 ); $unowned = 1; } } } if ( ( $unowned == 1 ) && ( $assign =~ /yes/ ) ) { logcomment("Some drive s are unowned"); logcomment("Check for drive details file"); my @actual_disks; my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/; my $disks_file = "$actual_log_dir" . "/Pre_post_Drives_Details_" . "$FILER"; if ( -f $disks_file ) { open( H_LOGFILE, "<$disks_file " ) or die ">>>>> read: Failed opening $disks_file <<<<<"; foreach my $line () { chomp($line); next if ( $line !~ /\S/ ); $line =~ s/(^\s+)|(\s+$)//ig; push @actual_disks, $line; } } my $output; logcomment("Drives captured from support file for $FILER_C : @actual_disks"); logcomment("Get total and assign drive equally to both node"); my $drv_count = @unowned; logcomment("Total $drv_count drives are unowned $FILER_C"); logcomment("DEBUG : FLAG : $flag"); if ( ( $flag == 0 ) && ( $drv_count >= 2 ) ) { $drv_count = floor( $drv_count / 2 ); } logcomment("DEBUG : Drv to assign : $drv_count"); if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;storage show initiators\"" ); } elsif ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "storage show initiators" ); } else { logcomment("Filer state cannot be captured : $FILER_C"); } if ( $drv_count > 0 ) { if ( $filer_state =~ /UP|CLI/i ) { $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk show -a" ) if ( $output !~ /$FILER_C/ ); } elsif ( $filer_state =~ /MAINT/i ) { $output = $Api_Set_Obj->execute_raw_command( command => "disk show -a" ) if ( $output !~ /$FILER_C/ ); } foreach my $line ( split( /\n/, $output ) ) { if ( ( $line =~ /$FILER_C\s+(\S+)/ ) || ( $line =~ /Local System ID\:\s+(\S+)/ ) ) { my $sysid = $1; ($sysid) = $sysid =~ /\((\d+)\)/ if ( $sysid =~ /\(/ ); logcomment("Filer : $FILER_C and System ID : $sysid"); if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk assign -n $drv_count -s $sysid -o $FILER_C\"" ); } elsif ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj->execute_raw_command( command => "disk assign -n $drv_count -s $sysid -o $FILER_C" ); } $flag++; sleep 3; } else { logcomment("System -ID cannot be captured, Existing from assign subroutine") if ( $flag == 0 ); } } } else { logcomment("No un-owned drives found on filer $FILER_C"); } } elsif ( ( $unowned == 1 ) && ( $assign =~ /no/ ) ) { logcomment("Un-owned Drives found ... "); logcomment("**WARNING** : Unowned drive found, Drive are @unowned"); } else { logcomment("No -Un-owned drives found"); } } } # Custom flag - 0 } sub nvme_clean_config { my (%opts) = validate( @_, { Node => { type => OBJECT }, env_var => { type => SCALAR, optional => 1, default => undef }, env_file => { type => SCALAR, optional => 1, default => undef }, } ); my $Node = $opts{Node}; my $env_data = $opts{env_var}; my $ENV_FIL_LOC = $opts{env_file}; my $filer_env_details = {}; logcomment("DEBUG CHCEK _ $Node"); logcomment("Get node name"); my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $passwd = $Host->default_password(); logcomment("Filer :$FILER_C Password : $passwd"); logcomment("Filer :$FILER_C halt file , set env and wipe config"); if ( defined( $opts{env_var} ) ) { logcomment("Env variable to be used $env_data"); } else { logcomment("Check for ENV file and get environment variable"); my $Home = $ENV{HOME}; my $ENV_FIL_LOC = "$Home/NDATE/FILER_INFO/$FILER_C"; open( ENV_FL, "$ENV_FIL_LOC" ) || die "Can not open file $ENV_FIL_LOC"; my @env_data = ; logcomment("Get path for filer "); my ( $env, $nb ); foreach my $line (@env_data) { chomp($line); if ( $line =~ /FILER=(\S+)/ ) { print "Filer : $1\n"; $FILER_C = $1; $filer_env_details->{$FILER_C} = { 'filer' => $1 }; } if ( $line =~ /PATH=(\S+)/ ) { $env = $1; print "ENV : $env\n"; $filer_env_details->{$FILER_C} = { 'env_p' => $env }; } if ( $line =~ /NETBOOT_IMG=(\S+)/ ) { $nb = $1; print "NB : $nb\n"; } } logcomment( "Dumper " . Dumper($filer_env_details) ); foreach my $fil ( keys %$filer_env_details ) { logcomment("Get Env path to be set for filer $fil"); if ( $fil =~ /$FILER_C/ ) { $env_data = $filer_env_details->{$fil}->{'env_p'}; logcomment("Env variable to be used : $env_data"); } else { $env_data = "NA"; } } } nvme_halt_Node( Node => $Node ); logcomment("FIELR : $FILER_C is in loader"); logcomment("Filer : $FILER : executing wipe-config on the filer"); nvme_set_env_var( Node => $Node, env_path => $env_data ); my $prompt_fw; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $host = host($FILER_C); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); #set the execution server to the current node my @filer_working = $Server->servers(); logcomment("Number of filers working are @filer_working"); my $timeout = 7200; #Burt: 1219973 my $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Transit_obj->set( 'timeout' => 600, maxcycle => 5 ); logcomment("Filer : $FILER : Changing state to boot-menu"); try { $Transit_obj->change_state( to => "BOOT_MENU" ); } catch NATE::BaseException with { logcomment("New prompt occured"); $Server->console( cmd => "4", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { ".*Are you sure you want to continue.*" => 'yes', ".*Zero disks, reset config and install a new file system.*" => 'yes', ".*This will erase all the data on the disks.*" => 'yes', ".*Override system ID.*" => "yes", ".Override system ID? {y|n}.*" => "yes", ".{y|n}.*" => 'y', ".*To override this delay, press \'c\' followed by .*" => 'c', ".*followed by \'Enter\'.*" => "\013", ".*Are you sure you want to continue (y or n).*" => 'y', "$prompt_fw" => "FINAL_PROMPT", } ); }; logcomment("DEBUG:Filer : $FILER_C executing wipeconfig on the boot_menu"); try { $Server->console( cmd => "4", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { ".*Are you sure you want to continue.*" => 'yes', ".*Zero disks, reset config and install a new file system.*" => 'yes', ".*This will erase all the data on the disks.*" => 'yes', ".*are you sure.*" => 'yes', "login" => "admin", ".*Type yes to confirm and continue.*" => 'exit', ".*Zero disks, reset config and install a new file system.*" => "yes", ".*reset config and install a new file.*" => "yes", ".*erase all the data on the disks.*" => "yes", ".*Override system ID.*" => "yes", ".*System ID mismatch. This usually occurs.*" => "yes", ".*To override this delay, press \'c\' followed by .*" => 'c', ".*followed by \'Enter\'.*" => "\013", ".*Are you sure you want to continue (y or n).*" => 'y', "$prompt_fw" => "FINAL_PROMPT", } ); } catch NATE::BaseException with { logcomment("Filer : $FILER_C inside first base exception"); }; logcomment("DEBUG:Completed executing wipeconfig command"); logcomment("Set environment variables"); nvme_set_env_var( Node => $Node, env_path => $env_data ); logcomment("Rebooting to finish wipeconfig request : $FILER_C"); try { $Server->console( cmd => "boot_ontap", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", ".*are you sure.*" => 'yes', "login" => "admin", ".*Type yes to confirm and continue.*" => 'exit', ".*To override this delay, press \'c\' followed by .*" => 'c', ".*followed by \'Enter\'.*" => "\013", ".*Are you sure you want to continue (y or n).*" => 'y', ".*Zero disks, reset config and install a new file system.*" => "yes", ".*reset config and install a new file.*" => "yes", ".*erase all the data on the disks.*" => "yes", ".*management interface port.*" => "exit", "::>" => "FINAL_PROMPT", } ); } catch NATE::BaseException with { logcomment("Filer : $FILER_C inside second base exception"); }; logcomment("Compelted wipe_config on filer $FILER_C"); logcomment("Check filer "); logcomment("Renaming node "); try { $Server->console( cmd => "node rename -newname $FILER_C", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "::>" => "FINAL_PROMPT", "login" => "admin", ".*Password.*" => "$passwd", } ); logcomment("Changing session timeout Value"); $Server->console( cmd => "set -rows 0;system timeout modify -timeout 0", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "::>" => "FINAL_PROMPT", "login" => "admin", ".*Password.*" => "$passwd", } ); } catch NACL::Transit::Exceptions::TransitException with { my $exception_object = shift; logcomment( "Exception is of type: " . ref($exception_object) ); logcomment( "Error text is: " . $exception_object->text() ); }; logcomment("Compelted re-initilizing, Proceed with cluster setup"); } sub nvme_boot_to_maint { my (%opts) = validate( @_, { Node => { type => OBJECT }, env_var => { type => SCALAR, optional => 1, default => undef }, } ); my $Node = $opts{Node}; my $env_data; if ( defined( $opts{env_var} ) ) { $env_data = $opts{env_var}; } else { $env_data = "NA"; } my $FILER_C = $Node->name(); logcomment("Filer :$FILER_C to be changed to maintenance mode"); my $Original_State; logcomment("Set environment variables"); $Node->refresh_command_interface(); # nvme_set_env_var( Node => $Node, env_path => $env_data ); logcomment("Check current state of filer"); logcomment("FILER : $FILER_C"); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $Api_Set_Obj; try { $Transit_obj->change_state( to => "MAINT" ); my $Host = host($FILER_C); $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); } catch NACL::Transit::Exceptions::RetriesExceeded with { logcomment("Expected: Caught a RetriesExceeded exception"); } catch NACL::Transit::Exceptions::TransitException with { my $exception_object = shift; logcomment( "Exception is of type: " . ref($exception_object) ); logcomment( "Error text is: " . $exception_object->text() ); } finally { logcomment("Reset fields 'timeout' and 'maxcycle'"); $Transit_obj->set( 'timeout' => 70000, maxcycle => 30 ); $Transit_obj->set( 'change_state_timeout' => 60000 ); logcomment( "New object settings for timeout and maxcycle are: " . Dumper $Transit_obj->get( 'timeout', 'maxcycle' ) ); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Transit_obj->change_state( to => "MAINT" ); }; logcomment("Filer booted in maintenance mode"); } sub nvme_set_env_var { my (%opts) = validate( @_, { Node => { type => OBJECT }, env_path => { type => SCALAR, optional => 1, default => undef }, } ); my $Node = $opts{Node}; my $env_path; my $FILER_C = $Node->name(); if ( defined( $opts{env_path} ) ) { $env_path = $opts{env_path}; } else { $env_path = "NA"; } logcomment("User Entered ENV variable: $env_path"); my ( $ENV1, $ENV2 ) = $env_path =~ /(\S+)\;(\S+)/; logcomment("Environment variableto be used \n ENV_1 : $ENV1 \n ENV_2 : $ENV2\n"); my @env_paths = split( /\;/, $env_path ); logcomment("DEBUG : @env_paths"); my $Host = host($FILER_C); my $prompt_array = $Host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; logcomment("Get parameters from filer config file to configure netboot "); my $Home = $ENV{HOME}; my $host_file = "$Home/NDATE/FILER_INFO/$FILER_C"; my $filer_config = `cat $host_file`; my $def_pass; my $mask; my $mgmport; my $def_addr; my $gw; my $netboot_img; foreach my $line ( split /\n/, $filer_config ) { if ( $line =~ /^default_password=(.*)/ ) { $def_pass = $1; } if ( $line =~ /^MGMT_IP=(.*)/ ) { $def_addr = $1; } if ( $line =~ /^MGMT_PORT=(.*)/ ) { $mgmport = $1; } if ( $line =~ /^MGMT_MASK=(.*)/ ) { $mask = $1; } if ( $line =~ /^MGMT_GATEWAY=(.*)/ ) { $gw = $1; } } logcomment("FILER: $FILER_C: IP:$def_addr | PORT:$mgmport | MASK:$mask | GATEWAY:$gw will be used for netboot"); try { $Server->console( cmd => "priv set test", additional_prompts_timeout => 600, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", "\>" => "FINAL_PROMPT" } ); logcomment("Privilage level test has been set"); } catch NATE::BaseException with { logcomment("FILER - $FILER_C : CANNOT SET TEST Privilage"); }; try { $Server->console( cmd => "ifconfig $mgmport -addr=$def_addr -mask=$mask -gw=$gw", additional_prompts_timeout => 600, ignore => $ignore_strings, additional_prompts => { ".*login.*" => "admin", ".*Password.*" => $def_pass, ".*::>.*" => "FINAL_PROMPT", ".*Login incorrect.*" => 'FATAL_PROMPT', "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", "\>" => "FINAL_PROMPT" } ); logcomment("Configured netboot parameters"); } catch NATE::BaseException with { logcomment("FILER - $FILER_C : CANNOT CONFIGURE NETTBOOT"); }; logcomment("Set default environment variables "); logcomment("Set Environment varible for filer $FILER_C from $env_path"); foreach my $env_p (@env_paths) { try { $Server->console( cmd => "run $env_p", additional_prompts_timeout => 6000, ignore => $ignore_strings, additional_prompts => { ".*login.*" => "admin", ".*Password.*" => $def_pass, ".*::>.*" => "FINAL_PROMPT", ".*Login incorrect.*" => 'FATAL_PROMPT', "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", "\>" => "FINAL_PROMPT" } ); logcomment("Default Environment variable for $FILER_C has been set"); } catch NATE::BaseException with { logcomment("FILER - $FILER_C : CANNOT SET DEFAULT ENV VARIABLES"); }; } logcomment("Environment variable set, Proceed with booting"); } # Enmd of setenv sub nvme_halt_Node { my (%opts) = validate( @_, { Node => { type => OBJECT }, } ); my $Node = $opts{Node}; my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $prompt_array = $Host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Transit_obj->set( 'timeout' => 600, maxcycle => 20 ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer is in $state state "); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; if ( $state =~ /CLI|UP/ ) { $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; try { $Server->console( cmd => "set test;halt -node $FILER_C -ignore-quorum-warnings true -skip-lif-migration-before-shutdown true", additional_prompts_timeout => 60000, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", ".*LOADER.*" => "FINAL_PROMPT", ".*Are you sure you want to halt node.*" => "y", ".*To override this delay, press \'c\' followed by .*" => 'c', ".*followed by \'Enter\'.*" => "\013", ".*Are you sure you want to continue (y or n).*" => 'y', ".*Do you want to continue.*" => "y", } ); } catch NATE::BaseException with { logcomment("FILER - $FILER_C : CANNOT halt system"); }; } else { $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; try { $Server->console( cmd => "halt -f -t 0", additional_prompts_timeout => 60000, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", ".*LOADER.*" => "FINAL_PROMPT" } ); } catch NATE::BaseException with { logcomment("FILER - $FILER_C : CANNOT halt system"); }; } logcomment("Filer halted, Set AUTOBOOT to FALSE"); $Server->console( cmd => 'setenv AUTOBOOT false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); } ######## BURT: 1199624 ###### sub nvme_get_sk_trace_logs { my (%opts) = validate( @_, { Node => { type => OBJECT }, Destination => { type => SCALAR }, runtime => { type => SCALAR }, } ); my $Node = $opts{Node}; my $destination = $opts{Destination}; my $FILER_C = $Node->name(); my $runtime = $opts{runtime}; my $sk_trace_path = "/mroot/etc/log/mlog/"; my $sk_trace_dir = "$FILER_C" . "_" . "sk_trace_logs"; my $main_dir = "SK_TRACE_LOGS"; my $client = hostname(); my @clienta = split( /\./, $client ); my $CLIENT = $clienta[0]; my $Client = NACL::C::Client->new( client => $CLIENT ); logcomment("Client : $Client - $CLIENT "); logcomment("RUNID : $RUNID"); my $run_id_dir; if ( $RUNID =~ /\S+\/\S+\/\S+\/(\S+)/ ) { $run_id_dir = $1; } logcomment("DIR Location : $run_id_dir"); my $num; if ( $parse_id =~ /(\d+)\_\S+/ ) { $num = $1; } $run_id_dir = $num if ( $run_id_dir == '' ); my ($actual_log_dir) = $destination =~ /(\S+)\/HDD/; my $log_dir_2_copy = "$actual_log_dir\/SK_TRACE_LOGS\/$run_id_dir"; system("sudo mkdir -p $log_dir_2_copy"); logcomment("$log_dir_2_copy Created"); system("sudo chmod 777 $log_dir_2_copy"); logcomment("############### $FILER_C : SK TRACE LOG CAPTURE START ###############"); logcomment("Copy sk_trace logs from filer :$FILER_C "); logcomment("$FILER_C : sk_trace runtime : $runtime seconds"); logcomment("$FILER_C : sk_trace log path : $sk_trace_path"); logcomment("$FILER_C : sk_trace log destination : $log_dir_2_copy"); #my $Client = NACL::C::Client->find(); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; logcomment("$FILER_C : Set privilage to diag"); $Api_Set_Obj->execute_raw_command( 'command' => "set -privilege diag", 'connectrec-match_table' => $prompts_answers ); logcomment("$FILER_C : Start sk_trace and wait for $runtime seconds"); $Api_Set_Obj->execute_raw_command( 'command' => "systemshell -node $FILER_C -command sudo sysctl sysvar.sktrace.DISK_enable=-1" ); logcomment("Wait for $runtime seconds"); sleep($runtime); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); logcomment("$FILER_C : $runtime seconds completed , Stop sk_trace and copy logs to $log_dir_2_copy"); $Api_Set_Obj->execute_raw_command( 'command' => "systemshell -node $FILER_C -command sudo sysctl sysvar.sktrace.DISK_enable=0" ); logcomment("$FILER_C : sk_trace stopped, copy logs"); my $scp; my $flag = 0; try { $scp = NACL::MTask::SCP->new( target_command_interface => $Node, source_command_interface => $Client, ); } catch NATE::BaseException with { $flag = 1; logcomment("Could NOT create SCP object, please check the connections"); }; if ($flag) { return 1; } logcomment("$FILER_C : SCP object created"); my $Filer_apiset = $Node->apiset( category => 'Node', interface => 'CLI', set => "Systemshell" ); my $File_on_filer = $Filer_apiset->execute_raw_command( command => "ls $sk_trace_path" ); chomp($File_on_filer); logcomment("File on filer >$File_on_filer<"); my $files = [$File_on_filer]; my @temp_array = @$files; @temp_array = split( "\t+|\n", $temp_array[0] ); foreach my $file (@temp_array) { chomp($file); if ( $file =~ /sktrace\.log/ ) { $file = "/mroot/etc/log/mlog/" . "$file"; $Filer_apiset->execute_raw_command( command => "chmod 777 $file" ); logcomment("Copy sk_trace $file logs to $log_dir_2_copy directory"); try { $scp->copy_from_target_to_source( 'files_to_copy' => [$file], set_directory_on_source => "$log_dir_2_copy", ); } catch NATE::BaseException with { logcomment("$FILER_C : Could NOT copy file :$file"); }; } } logcomment("$FILER_C : sk_trace logs copied"); logcomment("$FILER_C : SK TRACE LOG CAPTURE END "); } # BURT: 1217461 - Kiranp1 sub nvme_get_tahiti_shelf_log { my (%opts) = validate( @_, { Node => { type => OBJECT }, Destination => { type => SCALAR }, } ); my $Node = $opts{Node}; my $destination = $opts{Destination}; my $FILER_C = $Node->name(); my $tahiti_shelf_log_path = "/etc/log/shelflog/asup/"; my $shelf; logcomment("RUNID : $RUNID"); #Get Tahiti Shelf ID. my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $API_obj = $Node->get_7m_or_nodescope_apiset(); $API_obj->set_timeout( "connectrec-timeout" => 1200000 ); $API_obj->execute_raw_command( command => "set -rows 0" ); $API_obj->execute_raw_command( command => "system timeout modify 0" ); $API_obj->execute_raw_command( command => "\013" ); sleep 2; $API_obj->execute_raw_command( command => "\013" ); $API_obj->execute_raw_command( 'command' => "event config modify -console-log-level INFO -console off" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $API_obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); my $dr_out = $API_obj->execute_raw_command( 'command' => "run local disk_list" ); foreach my $line ( split( /\n/, $dr_out ) ) { if ( $line =~ /(\S+\.(\S+)\.\S+\.\S+)/ ) { $shelf = $2; } } logcomment("Tahiti shelf ID : $shelf"); #Command Execution to Generate Shelf Logs. $API_obj->execute_raw_command( command => "\013" ); sleep 2; $API_obj->execute_raw_command( command => "\013" ); $API_obj->execute_raw_command( 'command' => "run local psmadmin inband_cli 0x.$shelf 'psmmgr_tp 14 0'" ); sleep 2; $API_obj->execute_raw_command( command => "\013" ); $API_obj->execute_raw_command( 'command' => "run local psmadmin inband_cli 0x.$shelf 'psmmgr_tp 14 1'" ); sleep 5; $API_obj->execute_raw_command( command => "\013" ); #Check whether Logs are present at the specified location. my $File_on_filer; try { $File_on_filer = $API_obj->execute_raw_command( command => "run local ls $tahiti_shelf_log_path" ); if ( $File_on_filer =~ /No match|No such/i ) { logcomment("$FILER_C : NO $tahiti_shelf_log_path files found on the filer"); next; } } catch NATE::BaseException with { logcomment("$FILER_C : $tahiti_shelf_log_path CANNOT be accessed"); }; my $actual_log_dir = $destination; my $log_dir_2_copy = "$actual_log_dir\/TAHITI_SHELF_LOG\/"; system("sudo mkdir -p $log_dir_2_copy"); logcomment("$log_dir_2_copy Created"); system("sudo chmod 777 $log_dir_2_copy"); logcomment("$FILER_C : TAHITI SHELF LOG CAPTURE START"); logcomment("Copy tahiti_shelf_log from filer :$FILER_C "); logcomment("$FILER_C : tahiti_shelf_log path : $tahiti_shelf_log_path"); logcomment("$FILER_C : tahiti_shelf_log destination : $log_dir_2_copy"); #Creation of SCP object for copying. my $scp; my $Client = NACL::C::Client->find(); my $flag = 0; try { $scp = NACL::MTask::SCP->new( target_command_interface => $Node, source_command_interface => $Client, ); } catch NATE::BaseException with { $flag = 1; logcomment("Could NOT create SCP object, please check the connections"); }; if ($flag) { return 1; } logcomment("$FILER_C : SCP object created"); #API Object creation for switching to system Shell my $Filer_apiset = $Node->apiset( category => 'Node', interface => 'CLI', set => "Systemshell" ); #Get file name from ls command executed on systemshell my $File_on_filer; $tahiti_shelf_log_path = "/mroot/etc/log/shelflog/asup/"; #Copying of files from Filer to Specified Log Directory. my $File_on_filer = $Filer_apiset->execute_raw_command( command => "ls $tahiti_shelf_log_path" ); chomp($File_on_filer); logcomment("File on filer >$File_on_filer<"); my $files = [$File_on_filer]; my @temp_array = @$files; @temp_array = split( "\t+|\n", $temp_array[0] ); foreach my $file (@temp_array) { $file = "/mroot/etc/log/shelflog/asup/" . "$file"; $Filer_apiset->execute_raw_command( command => "chmod 777 $file" ); logcomment("Copy Tahiti Shelf log : $file logs to $log_dir_2_copy directory"); try { $scp->copy_from_target_to_source( 'files_to_copy' => [$file], set_directory_on_source => "$log_dir_2_copy", ); } catch NATE::BaseException with { logcomment("$FILER_C : Could NOT copy file :$file"); }; } } sub nvme_pre_test { my (%opts) = validate( @_, { filer_mode => { type => SCALAR }, node_present => { type => ARRAYREF }, Test => { type => OBJECT }, change_state_to => { type => SCALAR, optional => 1, default => 'CLI' }, LOGDIR => { type => SCALAR, optional => 1, default => '' }, test_setup => { type => SCALAR, optional => 1, default => 'SAS' }, BOOT_TYPE => { type => SCALAR, optional => 1, default => 'A' }, Tahiti => { type => SCALAR, optional => 1, default => 'no' }, } ); my @Nodes; #handle_run_local_error(); if ( defined( @{ $opts{node_present} } ) ) { @Nodes = @{ $opts{node_present} }; } else { @Nodes = NACL::C::Node->find(); } Storage::Common_Lib::handle_run_local_error( Nodes => [@Nodes] ); my $Tahiti = $opts{Tahiti}; my $Filer_mode = $opts{filer_mode}; my $Test = $opts{Test}; my $change_to = $opts{change_state_to}; my $TEST_CONFIG = $opts{test_config}; my $test_setup = $opts{test_setup}; my $exit_test = 0; my @Filer_array = (); my $execution_state; my @disk_list; my $node_ref; my @disk_array; my %FILER_API = (); my @disk_array; my $timeout; my $Api_Set_Obj; my $disk_capacity_total = 0; my $LOGDIR = $opts{LOGDIR}; my $params = NATE::ParamSet->new( global => 1 ); my $EOE = $params->get( 'EOE', default => 'default' ); my $EMAIL = $params->get( 'EMAIL', default => 'y' ); my $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); my $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); if ( $opts{LOGDIR} ) { $LOGDIR = $opts{LOGDIR}; } else { $LOGDIR = $ENV{HOME}; } my $TEST_NAME; opendir( DIR_LOG31, $LOGDIR ); my @files_DIR_LOG31 = readdir(DIR_LOG31); foreach my $file (@files_DIR_LOG31) { if ( ( $file =~ /(.*_NADQ02.*)\.log/ ) && ( $file !~ /results|END_FILER_CONFIG/ ) ) { $TEST_NAME = $1; } } $node_ref = \@Nodes; foreach my $temp_node (@Nodes) { my $Filer = $temp_node->name(); push( @Filer_array, $Filer ); } # Check state of filer and Execute based on state logcomment("Nodes present - @Filer_array"); logcomment("Log loction - $LOGDIR"); logcomment("Apollo-Tahiti : $Tahiti"); my @chk_state = (); foreach my $Node (@Nodes) { my $Transit_obj; my $FILER_C = $Node->name(); $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); $Test->nlog("$FILER_C : Filer is in $filer_state "); push( @chk_state, $filer_state ); } my $len = @chk_state; if ( $len > 1 ) { if ( $chk_state[0] eq $chk_state[1] ) { $execution_state = $chk_state[0]; $Test->nlog("Execution state - $execution_state\n"); } else { $Test->nlog("States of filer is different, Boot filer in same mode"); } } else { $execution_state = $chk_state[0]; $Test->nlog("Execution state - $execution_state\n"); } if ( $execution_state eq 'MAINT' ) { foreach my $Node (@Nodes) { my $Filer = $Node->name(); my $host = host($Filer); my $API_Object = NACL::APISet->new( hostobj => $host, category => "Node", interface => "CLI", set => "Maintenance" ); $FILER_API{$Filer} = $API_Object; } } # Check filer State and change to required state $Test->nlog("##################################################################################"); $Test->nlog("##"); $Test->nlog("## Begining of PRE_TEST logs for Filer : @Filer_array"); $Test->nlog("##"); $Test->nlog("##################################################################################"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); if ( $execution_state eq 'CLI' ) { my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $API_obj->set_timeout( "connectrec-timeout" => 1200000 ); $API_obj->execute_raw_command( command => "set -rows 0" ); $API_obj->execute_raw_command( command => "system timeout modify 0" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $API_obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $API_obj->execute_raw_command( command => "set -showallfields false" ); $API_obj->execute_raw_command( command => "set -showseparators \" \" " ); $API_obj->execute_raw_command( command => "\013" ); my $prompts_answers = [ ".*Are you sure you want to stop firmware update qualification test*" => "yes" ]; $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk_fw_update -S\"", 'connectrec-match_table' => $prompts_answers ); $API_obj->execute_raw_command( 'command' => "\013" ); logcomment("Disk fw update stopped"); logcomment("Disable fw debug level "); $API_obj->execute_raw_command( 'command' => "run local dbg level diskfw=0" ); $API_obj->execute_raw_command( 'command' => "\013" ); $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk_qual clear -d all\"" ); $API_obj->execute_raw_command( 'command' => "\013" ); $API_obj->execute_raw_command( 'command' => "run local options shelf.fw.auto.update" ); logcomment("Disable Shelf FW and Drive FW update"); $API_obj->execute_raw_command( 'command' => "run local options shelf.fw.auto.update off" ); $API_obj->execute_raw_command( 'command' => "\013" ); $API_obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable off" ); $API_obj->bootargs_set( arg => "AUTOBOOT", value => "false" ); logcomment("AUTOBOOT set to False"); $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test; bootargs set disk-skip-fw-force? false\"" ); $API_obj->execute_raw_command( 'command' => "\013" ); $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test; bootargs set shelf-skip-fw-force? false\"" ); $API_obj->execute_raw_command( 'command' => "\013" ); #$API_obj->bootargs_set( arg => "disk-skip-fw-force?", value => "false" ); #$API_obj->bootargs_set( arg => "shelf-skip-fw-force?", value => "false" ); $API_obj->execute_raw_command( 'command' => "\013" ); } elsif ( $execution_state eq 'MAINT' ) { logcomment("DEBUG : clear previous disk_qual status "); my $Api_Set_Obj = $FILER_API{$FILER_C}; $Api_Set_Obj->execute_command( 'command' => 'disk_qual clear -d all' ); $Api_Set_Obj->execute_command( 'command' => "\013" ); } } $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 1 : Check SSH parameters and Enable SSH"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("DEBUG:$FILER_C - Checking SSH and enabling SSH"); if ( $execution_state eq 'CLI' ) { $Test->nlog("$FILER_C : Check security login, enable ssh option "); $Test->nlog("$FILER_C : Filer mode is : $Filer_mode"); enable_ssh( Node => $Node, Test => $Test, Mode => $Filer_mode ); logcomment("$FILER_C - End of Checking SSH and enabling SSH"); } else { logcomment("$FILER_C - SSH WILL NOT BE VALIDATED IN MAINTENANCE MODE TESTS"); } } # Check kernel version $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 2 : Check Kernel Version"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER_C - Check Version on filer"); if ( $execution_state eq 'CLI' ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); } else { $Api_Set_Obj = $FILER_API{$FILER_C}; } my $ver_output = $Api_Set_Obj->execute_command( 'command' => 'version' ); logcomment("Version on $FILER_C - $ver_output"); logcomment("$FILER_C - End of Checking Version on filer"); } # Check for disk availibility (pre test) $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 3 : Checking for disk availability"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $disk_list_result = nvme_get_all_disk( node_present => [$Node] ); my @dsk = keys %$disk_list_result; my @disk_info = sort(@dsk); logcomment("Drive Present : @disk_info"); my $disk_length = @disk_info; @pre_disk_count = @disk_info; if ( $disk_length > 0 ) { $Test->nlog( "@Filer_array - Disk attached - \n" . @disk_info ); } else { $Test->nlog("@Filer_array -No disk available for testing, exiting from rest of the test cases"); $exit_test = 1; logcomment("**FATAL** : @Filer_array - NO DISK available for testing, exiting from rest of the test cases"); logresult( "FATAL", "@Filer_array - NO DISK AVAILABLE FOR TESTING,exiting from rest of the test cases" ); } last; } #Un-owned drives and assign $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 4 : Check for any unowned or unassigned drives"); $Test->nlog("==================================================================================="); if ( $Tahiti =~ /yes/ ) { logcomment("Apollo-Tahiti config drive assignment is done in check_filer_setup"); logcomment("If user had skipped check_filer_setup, User has to manually assign and config setup"); } else { logcomment("Checking for un-owned drives"); nvme_drv_assign( Nodes => \@Nodes, assign => "yes" ); logcomment("Check for un-owned drives completed"); } #Set log sense $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 5 : Set log sense interval"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("DEBUG: $FILER_C - Set log sense interval"); if ( $execution_state eq 'CLI' ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); logcomment("DEBUG:$FILER_C -Set options disk.log_sense.interval for 40 minutes"); NACL::C::Options->option( command_interface => $Node, 'option-name' => 'disk.log_sense.interval', 'option-value' => '4' ); logcomment("DEBUG:FILER - Waiting for 6 minute"); sleep(360); NACL::C::Options->option( command_interface => $Node, 'option-name' => 'disk.log_sense.interval', 'option-value' => '40' ); logcomment("DEBUG:FILER - Waiting for a minute"); sleep(60); } else { $Api_Set_Obj = $FILER_API{$FILER_C}; logcomment("$FILER_C : NO NEED TO SET LOG SENSE INTERVAL TO 4 HRS FOR MAINTENANCE MODE"); } logcomment("$FILER_C - End of setting log sense interval"); } # Execution sysconfig -r $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 6 : Record sysconfig -r information of the filer"); $Test->nlog("==================================================================================="); if ( $execution_state eq 'CLI' ) { my @subtests; $pre_post_recon_copy = "pre"; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&check_recons_sickdisk, -runid, "check_recons_sickdisk_$FILER_C", -bg, "--", Node => $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); status_return( subtest_ref => [@subtests] ); } else { logcomment("FILER IS IN MAINTENANCE MODE, COMMAND \"sysconfig -r\" IS NOT SUPPORTED"); } # Storage show information of the filer $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 7 : Record storage show information of the filer"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("DEBUG:$FILER_C - Executing storage show"); if ( $execution_state eq 'CLI' ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_command( 'command' => 'storage show', 'timeout' => 600 ); } else { $Api_Set_Obj = $FILER_API{$FILER_C}; $Api_Set_Obj->execute_command( 'command' => 'storage show', 'timeout' => 600 ); } logcomment("$FILER_C - End of Executing storage show"); } # Record expander map of the adapter $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 8 : Record expander map of the adapter"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER_C - Record expander map of the adapter"); my $Host = host($FILER_C); if ( $test_setup ne 'FC' ) { Storage::Common_Lib::sasadmin_expander_map_cmode( node => $Node ); } else { Storage::Common_Lib::fcadmin_device_map( Node => $Node ); } logcomment("$FILER_C - End of Record expander map of the adapter"); } # Execute disk_qual t 12 for all disk $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 9 : Printing logsense data for all the disks"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER_C - As log page 40 is not supported, disk_qual -t 12 will not be executed"); #Burt No: 1144744 # logcomment("$FILER_C - Print logsense data for all drives"); # $SCSI_DISK_ERRORS->{$FILER_C} = Storage::Common_Lib::scsi_disk_errors( Node => $Node, Flag_Pre => 1 ); # logcomment("$FILER_C - End if printing logsense data for all drives"); } $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 10 : Execute get_phy_state"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); if ( $test_setup ne 'FC' ) { logcomment("$FILER_C - Execute sasadmin expander_phy_state"); $GET_PHY_STATE->{$FILER_C} = Storage::Common_Lib::get_phy_state( Node => $Node ); logcomment("$FILER_C - End of sasadmin expander_phy_state"); } else { logcomment("NO CHECKING FOR FC"); } } $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 11 : Execute storage show disk -a"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("DEBUG:$FILER_C - Execute storage show disk -a"); logcomment("PRE_TEST : $FILER_C - Checking for Dynamic Qualification of the drives"); $STO_DISK_SHOW->{$FILER_C} = Storage::Common_Lib::storage_show_disk_a_data( Node => $Node ); my @sto_disk_show_fields = keys ${$STO_DISK_SHOW}{$FILER_C}; my %dq_drives; foreach my $field (@sto_disk_show_fields) { if ( $field =~ /\w+\.\S+/ ) { if ( ${$STO_DISK_SHOW}{$FILER_C}{$field}{'Dynamically qualified'} =~ /[Yy][Ee][Ss]/ ) { $dq_drives{$field} = ${$STO_DISK_SHOW}{$FILER_C}{$field}{'Dynamically qualified'}; } } } if (%dq_drives) { logcomment("**WARNING** : PRE_TEST : $FILER_C - One or more drives have been determined to be dynamically qualified"); } else { logcomment("PRE_TEST : $FILER_C : There are no drives to be dynamically qualified"); } logcomment("$FILER_C - End of Execute storage show disk -a"); } $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 12 : Execute sysconfig -v"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("DEBUG:$FILER_C - Executing sysconfig -v"); if ( $execution_state eq 'CLI' ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->execute_command( 'command' => 'sysconfig -v', 'timeout' => 600 ); } else { $Api_Set_Obj = $FILER_API{$FILER_C}; $Api_Set_Obj->execute_command( 'command' => 'sysconfig -v', 'timeout' => 600 ); } } $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 13 : Get log Pages for Data Analysis "); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Api_Set_Obj_SS; my $drv_result = {}; if ( $execution_state eq 'CLI' ) { my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); logcomment("Check Current Privilage Level "); $Api_Set_Obj->execute_raw_command( 'command' => "set" ); logcomment("Set privilage level to diag"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set diag", 'connectrec-match_table' => $prompts_answers ); logcomment("Privilage level diag set"); $Api_Set_Obj->execute_raw_command( 'command' => "set" ); logcomment("Get password to login in system shell"); my $Passwd = $Host->default_password(); logcomment("Filer password : $Passwd"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj_SS = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Systemshell" ); } else { my $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); my $Passwd = $Host->default_password(); my $prompts_answers = [ ".*password*" => "$Passwd" ]; $API_Object->execute_raw_command( 'command' => "systemshell", 'connectrec-match_table' => $prompts_answers ); $API_Object->execute_raw_command( 'command' => "\013" ); $Api_Set_Obj_SS = $API_Object; } logcomment("System shell API created, getting log pages "); my $ss_disk_list = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol devlist" ); foreach my $line ( split( /\n/, $ss_disk_list ) ) { my ( $drv, $ns, $nsz ); if ( $line =~ /(\S+)\:\s+(\S+)/ ) { $drv = $1; $drv_result->{$drv}->{'PRDID'} = $2; } if ( $line =~ /(\S+)ns\d+/ ) { $drv = $1; if ( $line =~ /(\S+)\s+\((\S+)\)/ ) { $ns = $1; $nsz = $2; $drv_result->{$drv}->{'NAME_SP'} = $ns; $drv_result->{$drv}->{'NS_SIZE'} = $nsz; } } } my @nv_drv = keys(%$drv_result); logcomment("System Shell Drive(s) : @nv_drv"); foreach my $drv (@nv_drv) { logcomment("Recording controller infromation on $drv"); $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol identify $drv" ); logcomment("Recorded controller infromation on $drv"); } foreach my $drv (@nv_drv) { logcomment("Recording Error Information log page 1 data for drive $drv "); my $lgpg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol logpage -p 1 $drv " ); if ( $lgpg =~ /No error entries found/ ) { logcomment("NO LOG PAGE 1 ENTRIES FOUND for drive $drv"); } logcomment("Recorded Error Information log page data for drive $drv "); } foreach my $drv (@nv_drv){ logcomment("Recording SMART/Health log page 2 data for drive $drv "); my $smlg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol logpage -p 2 $drv " ); logcomment("Recorded SMART/Health log page data for drive $drv"); #burt1241170 foreach my $line ( split( /\r?\n/, $smlg ) ) { if ( $line =~ /Percentage used\:\s+(\d+)/ ) { my $per_used = $1; if ( $per_used < 2 ) { logcomment("Percent used for drive $drv is $per_used"); } else { logcomment("**WARNING** : PRE_TEST : $FILER_C - Percent used for drive $drv is $per_used, It should be less than 2"); } } } } foreach my $drv (@nv_drv) { logcomment("Recording Firmware log page 3 data for drive $drv "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol logpage -p 3 $drv " ); logcomment("Recorded Firmware log page data for drive $drv "); } foreach my $drv (@nv_drv) { logcomment("Recording 0xC0 log page data for drive $drv "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol logpage -p 0xC0 $drv " ); logcomment("Recorded Firmware log page data for drive $drv "); } logcomment("$FILER_C : Basic log pages Completed "); if ( $execution_state eq 'CLI' ) { logcomment("Check for SCM FLASH CACHE DRIVE and get all logpage"); my $flash_cache_drv = get_scm_flash_cache_drv( Node => $Node ); my @res_disks; @res_disks = keys $flash_cache_drv if ($flash_cache_drv); if (@res_disks) { logcomment("Flash Cache Drive present: @res_disks"); foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Get logpages for drive :$ctrl_id"); logcomment("Recording controller infromation on $ctrl_id"); $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol identify $ctrl_id" ); logcomment("Recorded controller infromation on $ctrl_id"); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording Error Information log page 1 data for drive $ctrl_id"); my $lgpg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 1 $ctrl_id" ); if ( $lgpg =~ /No error entries found/ ) { logcomment("NO LOG PAGE 1 ENTRIES FOUND for drive $ctrl_id"); } logcomment("Recorded Error Information log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording SMART/Health log page 2 data for drive $ctrl_id"); my $smlg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 2 $ctrl_id " ); logcomment("Recorded SMART/Health log page data for drive $ctrl_id "); #burt1241170 foreach my $line ( split( /\r?\n/, $smlg ) ) { if ( $line =~ /Percentage used\:\s+(\d+)/ ) { my $per_used = $1; if ( $per_used < 2 ) { logcomment("Percent used for drive $c_dr is $per_used"); } else { logcomment("**WARNING** : PRE_TEST :$FILER_C - Percent used for drive $c_dr is $per_used, it should be less than 2"); } } } } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording Firmware log page 3 data for drive $ctrl_id "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 3 $ctrl_id " ); logcomment("Recorded Firmware log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording 0xC0 log page data for drive $ctrl_id "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 0xC0 $ctrl_id " ); logcomment("Recorded Firmware log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording 0xC0 log page data for drive $ctrl_id "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 0xCA $ctrl_id " ); logcomment("Recorded Firmware log page data for drive $ctrl_id "); } logcomment("$FILER_C : Basic log pages Completed for Flash Cache NVMe Drive"); } } else { $Api_Set_Obj_SS->execute_raw_command( 'command' => "exit" ); } } $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 14 : Check for drive path on both nodes "); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); if ( $execution_state eq 'CLI' ) { my $Host = host($FILER_C); my $Api_Set_Obj_SS; logcomment("$FILER_C : Check storage failover status"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" ); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" ) if ( $out eq '' ); my @miss_drv; my @local_nodes = NACL::C::Node->find(); my $FILER_A = $local_nodes[0]->name(); my $FILER_B = $local_nodes[1]->name(); logcomment("FILER_A : $FILER_A and FILER_B : $FILER_B"); foreach my $line ( split /\n/, $out ) { my ( $drv, $path ); next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change|\[\S+\]/ ); if ( ( $line =~ /$FILER_A/ ) && ( $line =~ /$FILER_B/ ) ) { my ( $drv, $path ) = $line =~ /(\S+)\s+(\S+)/; logcomment("Drive $drv has path $path"); } else { my ( $drv, $path ) = $line =~ /\S+\:(\S+)\s+(\S+)/; logcomment("Drive $drv path is incomplete"); push( @miss_drv, $drv ); } } if (@miss_drv) { logcomment("!!!! DRIVE PATH MISSING : @miss_drv"); logcomment("**FATAL** : DRIVE PATH MISSING : @miss_drv"); logresult( "FATAL", "DRIVE PATH MISSING : @miss_drv" ); } sleep 30; } else { logcomment("FILER: $FILER_C IS IN MAINTENANCE MODE, COMMAND \"storage disk show\" IS NOT SUPPORTED"); } } $Test->nlog("==================================================================================="); $Test->nlog("==== PRE_TEST STEP 15 : Check port link state for all drives"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); if ( $execution_state eq 'CLI' ) { my @link_dis_drivs; my $Host = host($FILER_C); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); logcomment("$FILER_C : Check port link state for all drives"); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;run local storage show psm" ); foreach my $line ( split /\n/, $out ) { next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change/ ); if ( $line =~ /\[.*]/ ) { if ( $line =~ /DIS\/LNK|LNK|DIS/ ) { my ( $drv, $state, $lnk_lan, $max_lnk, $neg_lnk, $med_mx_lnk, $dev_mx_la, $Neg_max, $lan_wd ) = ( $line =~ /(\[.*\])\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ); logcomment("Drive ID : $drv State is $state !!!!!!!!! "); logcomment("Drive port link states are : Disk ID : $drv\n Port Max Link Speed:$lnk_lan\n Negotiated Max Link Speed:$max_lnk\n Medium Link Speed:$neg_lnk\n Device Max Link Speed:$med_mx_lnk\n Port Max Lane Width:$dev_mx_la\n Negotiated Max Lanewidth:$Neg_max\n Lanewidth:$lan_wd\n"); logcomment("Drive ID : $drv State is $state"); push( @link_dis_drivs, $drv ); } } } if (@link_dis_drivs) { logcomment("***FATAL*** : $FILER_C : Port State for following drives are DISABLED or OFFLINE, Drive ID(s) are: @link_dis_drivs"); logresult( "FATAL", "Port State for following drives are DISABLED or OFFLINE, Drive ID(s) are: @link_dis_drivs" ); } } else { logcomment("FILER: $FILER_C IS IN MAINTENANCE MODE, COMMAND \"storage show psm\" IS NOT SUPPORTED"); } } $Test->nlog("##################################################################################"); $Test->nlog("##"); $Test->nlog("## End of PRE_TEST logs for Filer : @Filer_array"); $Test->nlog("##"); $Test->nlog("##################################################################################"); } sub nvme_post_test { my (%opts) = validate( @_, { filer_mode => { type => SCALAR }, node_present => { type => ARRAYREF }, Test => { type => OBJECT }, change_state_to => { type => SCALAR, optional => 1, default => 'CLI' }, LOGDIR => { type => SCALAR, optional => 1, default => '' }, test_setup => { type => SCALAR, optional => 1, default => 'SAS' }, BOOT_TYPE => { type => SCALAR, optional => 1, default => 'A' }, Tahiti => { type => SCALAR, optional => 1, default => 'no' }, } ); my @Nodes; if ( defined( @{ $opts{node_present} } ) ) { @Nodes = @{ $opts{node_present} }; } else { @Nodes = NACL::C::Node->find(); } Storage::Common_Lib::handle_run_local_error( Nodes => [@Nodes] ); my $Tahiti = $opts{Tahiti}; my $Filer_mode = $opts{filer_mode}; my $Test = $opts{Test}; my $change_to = $opts{change_state_to}; my $TEST_CONFIG = $opts{test_config}; my $test_setup = $opts{test_setup}; my $exit_test = 0; my @Filer_array = (); my $execution_state; my @disk_list; my $node_ref; my @disk_array; my %FILER_API = (); my @disk_array; my $timeout; my $Api_Set_Obj; my $disk_capacity_total = 0; my $LOGDIR = $opts{LOGDIR}; my $params = NATE::ParamSet->new( global => 1 ); my $EOE = $params->get( 'EOE', default => 'default' ); my $EMAIL = $params->get( 'EMAIL', default => 'y' ); my $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); my $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); if ( $opts{LOGDIR} ) { $LOGDIR = $opts{LOGDIR}; } else { $LOGDIR = $ENV{HOME}; } my $TEST_NAME; opendir( DIR_LOG31, $LOGDIR ); my @files_DIR_LOG31 = readdir(DIR_LOG31); foreach my $file (@files_DIR_LOG31) { if ( ( $file =~ /(.*_NADQ02.*)\.log/ ) && ( $file !~ /results|END_FILER_CONFIG/ ) ) { $TEST_NAME = $1; } } $node_ref = \@Nodes; foreach my $temp_node (@Nodes) { my $Filer = $temp_node->name(); push( @Filer_array, $Filer ); } # Check state of filer and Execute based on state logcomment("Nodes present - @Filer_array"); logcomment("Log loction - $LOGDIR"); logcomment("Apollo-Tahiti - $Tahiti"); my @chk_state = (); # Check for state and call respective subroutines: foreach my $Node (@Nodes) { my $Transit_obj; my $FILER_C = $Node->name(); $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); $Test->nlog("$FILER_C : Filer is in $filer_state "); push( @chk_state, $filer_state ); } my $len = @chk_state; if ( $len > 1 ) { if ( $chk_state[0] eq $chk_state[1] ) { $execution_state = $chk_state[0]; $Test->nlog("Execution state - $execution_state\n"); } else { $Test->nlog("States of filer is different, Boot filer in same mode"); } } else { $execution_state = $chk_state[0]; $Test->nlog("Execution state - $execution_state\n"); } if ( $execution_state eq 'MAINT' ) { foreach my $Node (@Nodes) { my $Filer = $Node->name(); my $host = host($Filer); my $API_Object = NACL::APISet->new( hostobj => $host, category => "Node", interface => "CLI", set => "Maintenance" ); $FILER_API{$Filer} = $API_Object; } } # Check filer State and change to required state $Test->nlog("##################################################################################"); $Test->nlog("##"); $Test->nlog("## Begining of POST_TEST logs for Filer : @Filer_array"); $Test->nlog("##"); $Test->nlog("##################################################################################"); $Test->nlog("Checking boot state of filer, and booting in required mode"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); if ( $execution_state eq 'CLI' ) { my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $API_obj->set_timeout( "connectrec-timeout" => 1200000 ); $API_obj->execute_raw_command( command => "set -rows 0" ); $API_obj->execute_raw_command( command => "system timeout modify 0" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $API_obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $API_obj->execute_raw_command( command => "set -showallfields false" ); $API_obj->execute_raw_command( command => "set -showseparators \" \" " ); $API_obj->execute_raw_command( command => "\013" ); logcomment("Stop disk_qual"); $API_obj->execute_raw_command( command => "run local \"priv set -q test;disk_qual stop -d all\"" ); $API_obj->execute_raw_command( command => "\013" ); } elsif ( $execution_state eq 'MAINT' ) { logcomment("Stop disk_qual"); my $Api_Set_Obj = $FILER_API{$FILER_C}; $Api_Set_Obj->execute_command( 'command' => "disk_qual stop -d all" ); logcomment("disk_qual stopped"); } } # Check security login, ssh option $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 1 : Check SSH connection in post test"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER_C - Checking SSH parameters"); if ( $execution_state eq 'CLI' ) { try { $Test->nlog("$FILER_C : Check security login, enable ssh option "); $Test->nlog("$FILER_C : Filer mode is : $Filer_mode"); enable_ssh( Node => $Node, Test => $Test, Mode => $Filer_mode ); $Test->nlog("$FILER_C : Disable all storage debug messages"); my $host = host($FILER_C); disable_debug_msg( Node => $Node ); } catch NACL::APISet::Exceptions::MgwdCrashedException with { logcomment("Management gateway restarted in between, Check for any coredump"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*to page down, for next line, or 'q' to quit.*" => 'q' ]; } catch NATE::BaseException with { logcomment("Management gateway restarted "); }; } else { logcomment("FILER $FILER_C IS IN MAINTENANCE MODE, SSH is NOT USED IN MAINTENANCE MODE"); } logcomment("$FILER_C - End of Checking SSH"); } # Checking Kernel version $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 2 : Record version of Kernel used in Post test"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER_C - Checking Kernel version on filers"); if ( $execution_state eq 'CLI' ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); } else { $Api_Set_Obj = $FILER_API{$FILER_C}; } my $ver_output = $Api_Set_Obj->execute_command( 'command' => 'version' ); logcomment("Version on $FILER_C - $ver_output"); logcomment("End of Checking kernel version"); } #Un-owned drives and assign $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 3 : Check for any unowned or unassigned drives"); $Test->nlog("==================================================================================="); if ( $Tahiti =~ /yes/ ) { logcomment("Apollo-Tahiti config"); } else { logcomment("Checking for un-owned drives"); nvme_drv_assign( Nodes => \@Nodes ); logcomment("Check for un-owned drives completed"); } #check for disk availibility (pre test) $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 4 : Checking for disk availability"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $disk_list_result = nvme_get_all_disk( node_present => [$Node] ); my @dsk = keys %$disk_list_result; my @disk_info = sort(@dsk); logcomment("Drive Present : @disk_info"); @post_disk_count = @disk_info; if ( scalar @pre_disk_count != scalar @post_disk_count ) { my @Pre_drv = (); my @Post_drv = (); grep { if ( $_ =~ /\w+\.(\w+\.\w+)|(\w+\.\w+)/ ) { push( @Pre_drv, $1 ) } } @pre_disk_count; grep { if ( $_ =~ /\w+\.(\w+\.\w+)|(\w+\.\w+)/ ) { push( @Post_drv, $1 ) } } @post_disk_count; my @diff = difference( [ sort @Pre_drv ], [ sort @Post_drv ] ); push( @diff, difference( [ sort @Post_drv ], [ sort @Pre_drv ] ) ); logcomment( "**FATAL** :Disk device @diff was NOT found during post test( PRE = " . scalar @pre_disk_count . " disks and POST = " . scalar @post_disk_count . " disks )" ); logresult( 'FATAL', "DISK DEVICE @diff WAS NOT FOUND DURING POST TEST( PRE = " . scalar @pre_disk_count . " disks and POST = " . scalar @post_disk_count . " disks )" ); } else { logcomment( "Total Drive count from PRE = " . scalar @pre_disk_count . " disks and POST = " . scalar @post_disk_count . " disks are equal)" ); } last; } # Execution sysconfig -r $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 5 : Record sysconfig -r information of the filer"); $Test->nlog("==================================================================================="); if ( $execution_state eq 'CLI' ) { my @subtests; $pre_post_recon_copy = "post"; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&check_recons_sickdisk, -runid, "check_recons_sickdisk_$FILER_C", -bg, "--", Node => $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); status_return( subtest_ref => [@subtests] ); } else { logcomment("FILER IS IN MAINTENANCE MODE, COMMAND \"sysconfig -r\" is NOT SUPPORTED"); } # Storage show information of the filer $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 6 : Record storage show information of the filer"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("DEBUG:$FILER_C - Executing storage show"); if ( $execution_state eq 'CLI' ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_command( 'command' => 'storage show', 'timeout' => 600 ); } else { $Api_Set_Obj = $FILER_API{$FILER_C}; $Api_Set_Obj->execute_command( 'command' => 'storage show', 'timeout' => 600 ); } logcomment("End of $FILER_C - Executing storage show"); } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 7 : Set log sense interval"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("DEBUG: $FILER_C - Set log sense interval"); if ( $execution_state eq 'CLI' ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); logcomment("DEBUG:$FILER_C -Set options disk.log_sense.interval for 40 minutes"); NACL::C::Options->option( command_interface => $Node, 'option-name' => 'disk.log_sense.interval', 'option-value' => '4' ); logcomment("DEBUG:FILER - Waiting for 6 minute"); sleep(360); NACL::C::Options->option( command_interface => $Node, 'option-name' => 'disk.log_sense.interval', 'option-value' => '40' ); logcomment("DEBUG:FILER - Waiting for a minute"); sleep(60); } else { logcomment("$FILER_C : NO NEED TO SET LOG SENSE INTERVAL TO 4 HRS FOR MAINTENANCE MODE"); } logcomment("$FILER_C - End of setting log sense interval"); } # Execute disk_qual t 12 for all disk $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 8 : Printing logsense data for all the disks"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER_C - As log page 40 is not supported, disk_qual -t 12 will not be executed"); #Burt No: 1144744 # logcomment("$FILER_C - Print logsense data for all drives"); # $SCSI_DISK_ERRORS->{$FILER_C} = Storage::Common_Lib::scsi_disk_errors( Node => $Node, Flag_Pre => 1 ); # logcomment("$FILER_C - End if printing logsense data for all drives"); } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 9 : Execute get_phy_state"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); if ( $test_setup ne 'FC' ) { logcomment("$FILER_C - Execute sasadmin expander_phy_state"); $GET_PHY_STATE->{$FILER_C} = Storage::Common_Lib::get_phy_state( Node => $Node ); logcomment("$FILER_C - End of sasadmin expander_phy_state"); } else { logcomment("NO CHECKING FOR FC"); } } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 10 : Execute storage show disk -a"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("DEBUG:$FILER_C - Execute storage show disk -a"); logcomment("PRE_TEST : $FILER_C - Checking for Dynamic Qualification of the drives"); $STO_DISK_SHOW->{$FILER_C} = Storage::Common_Lib::storage_show_disk_a_data( Node => $Node ); my @sto_disk_show_fields = keys ${$STO_DISK_SHOW}{$FILER_C}; my %dq_drives; foreach my $field (@sto_disk_show_fields) { if ( $field =~ /\w+\.\S+/ ) { if ( ${$STO_DISK_SHOW}{$FILER_C}{$field}{'Dynamically qualified'} =~ /[Yy][Ee][Ss]/ ) { $dq_drives{$field} = ${$STO_DISK_SHOW}{$FILER_C}{$field}{'Dynamically qualified'}; } } } if (%dq_drives) { logcomment("**WARNING** : POST_TEST : $FILER_C - One or more drives have been determined to be dynamically qualified"); } else { logcomment("POST_TEST : $FILER_C : There are no drives to be dynamically qualified"); } logcomment("$FILER_C - End of Execute storage show disk -a"); } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 11 : Execute sysconfig -v"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("DEBUG:$FILER_C - Executing sysconfig -v"); if ( $execution_state eq 'CLI' ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->execute_command( 'command' => 'sysconfig -v', 'timeout' => 600 ); } else { $Api_Set_Obj = $FILER_API{$FILER_C}; $Api_Set_Obj->execute_command( 'command' => 'sysconfig -v', 'timeout' => 600 ); } } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 12 : Get log Pages for Data Analysis "); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Api_Set_Obj_SS; my $drv_result = {}; if ( $execution_state eq 'CLI' ) { my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); logcomment("Check Current Privilage Level "); $Api_Set_Obj->execute_raw_command( 'command' => "set" ); logcomment("Set privilage level to diag"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set diag", 'connectrec-match_table' => $prompts_answers ); logcomment("Privilage level diag set"); $Api_Set_Obj->execute_raw_command( 'command' => "set" ); logcomment("Get password to login in system shell"); my $Passwd = $Host->default_password(); logcomment("Filer password : $Passwd"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj_SS = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Systemshell" ); } else { my $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); my $Passwd = $Host->default_password(); my $prompts_answers = [ ".*password*" => "$Passwd" ]; $API_Object->execute_raw_command( 'command' => "systemshell", 'connectrec-match_table' => $prompts_answers ); $API_Object->execute_raw_command( 'command' => "\013" ); $Api_Set_Obj_SS = $API_Object; } logcomment("System shell API created, getting log pages "); my $ss_disk_list = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol devlist" ); my $drv_result = {}; foreach my $line ( split( /\n/, $ss_disk_list ) ) { my ( $drv, $ns, $nsz ); if ( $line =~ /(\S+)\:\s+(\S+)/ ) { $drv = $1; $drv_result->{$drv}->{'PRDID'} = $2; } if ( $line =~ /(\S+)ns\d+/ ) { $drv = $1; if ( $line =~ /(\S+)\s+\((\S+)\)/ ) { $ns = $1; $nsz = $2; $drv_result->{$drv}->{'NAME_SP'} = $ns; $drv_result->{$drv}->{'NS_SIZE'} = $nsz; } } } my @nv_drv = keys(%$drv_result); logcomment("System Shell Drive(s) : @nv_drv"); foreach my $drv (@nv_drv) { logcomment("Recording controller infromation on $drv"); $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol identify $drv" ); logcomment("Recorded controller infromation on $drv"); } foreach my $drv (@nv_drv) { logcomment("Recording Error Information log page 1 data for drive $drv "); my $lgpg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol logpage -p 1 $drv " ); if ( $lgpg =~ /No error entries found/ ) { logcomment("NO LOG PAGE 1 ENTRIES FOUND for drive $drv"); } logcomment("Recorded Error Information log page data for drive $drv "); } foreach my $drv (@nv_drv) { logcomment("Recording SMART/Health log page 2 data for drive $drv "); my $smlg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol logpage -p 2 $drv " ); logcomment("Recorded SMART/Health log page data for drive $drv "); #burt1241170 foreach my $line ( split( /\r?\n/, $smlg ) ) { if ( $line =~ /Percentage used\:\s+(\d+)/ ) { my $per_used = $1; if ( $per_used < 2 ) { logcomment("Percent used for drive $drv is $per_used"); } else { logcomment("**WARNING** : POST_TEST : $FILER_C - Percent used for drive $drv is $per_used, it should be less than 2"); } } } } foreach my $drv (@nv_drv) { logcomment("Recording Firmware log page 3 data for drive $drv "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol logpage -p 3 $drv " ); logcomment("Recorded Firmware log page data for drive $drv "); } foreach my $drv (@nv_drv) { logcomment("Recording 0xC0 log page data for drive $drv "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo peg_nvmecontrol logpage -p 0xC0 $drv " ); logcomment("Recorded Firmware log page data for drive $drv "); } logcomment("$FILER_C : Basic log pages Completed "); if ( $execution_state eq 'CLI' ) { logcomment("Check for SCM FLASH CACHE DRIVE and get all logpage"); my $flash_cache_drv = get_scm_flash_cache_drv( Node => $Node ); my @res_disks; @res_disks = keys $flash_cache_drv if ($flash_cache_drv); if (@res_disks) { logcomment("Flash Cache Drive present: @res_disks"); foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Get logpages for drive :$ctrl_id"); logcomment("Recording controller infromation on $ctrl_id"); $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol identify $ctrl_id" ); logcomment("Recorded controller infromation on $ctrl_id"); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording Error Information log page 1 data for drive $ctrl_id"); my $lgpg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 1 $ctrl_id" ); if ( $lgpg =~ /No error entries found/ ) { logcomment("NO LOG PAGE 1 ENTRIES FOUND for drive $ctrl_id"); } logcomment("Recorded Error Information log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording SMART/Health log page 2 data for drive $ctrl_id"); my $smlg = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 2 $ctrl_id " ); logcomment("Recorded SMART/Health log page data for drive $ctrl_id "); #burt1241170 foreach my $line ( split( /\r?\n/, $smlg ) ) { if ( $line =~ /Percentage used\:\s+(\d+)/ ) { my $per_used = $1; if ( $per_used < 2 ) { logcomment("Percent used for drive $c_dr is $per_used"); } else { logcomment("**WARNING** : POST_TEST : $FILER_C - Percent used for drive $c_dr is $per_used, it should be less than 2"); } } } } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording Firmware log page 3 data for drive $ctrl_id "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 3 $ctrl_id " ); logcomment("Recorded Firmware log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording 0xC0 log page data for drive $ctrl_id "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 0xC0 $ctrl_id " ); logcomment("Recorded Firmware log page data for drive $ctrl_id "); } foreach my $c_dr (@res_disks) { my ($ctrl_id) = $c_dr =~ /(.*)ns/; logcomment("Recording 0xC0 log page data for drive $ctrl_id "); my $fwl = $Api_Set_Obj_SS->execute_raw_command( 'command' => "sudo nvmecontrol logpage -p 0xCA $ctrl_id " ); logcomment("Recorded Firmware log page data for drive $ctrl_id "); } logcomment("$FILER_C : Basic log pages Completed for Flash Cache NVMe Drive"); } else { logcomment("No Flash Cache Drive found , Proceed with next steps "); } } else { logcomment("$FILER_C : Exit form system shell"); $Api_Set_Obj_SS->execute_raw_command( 'command' => "exit" ); } } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 13 : Copying the system logs to the filer"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $num; my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /MAINT/gi ) { logcomment("FILER IS IN $filer_state STATE. SYSTEM LOGS WILL NOT BE GENERATED"); last; } elsif ( $filer_state =~ /CLI/gi ) { my $params = NATE::ParamSet->new( global => 1 ); my $LOGDIR = $params->get( 'LOGDIR', default => undef ); my ($actual_log_dir) = $LOGDIR =~ /(\S+)\/HDD/; if ( $parse_id =~ /(\d+)\_\S+/ ) { #aditis: 861147 $num = $1; } my $log_dir = "$actual_log_dir\/SYSTEM_LOGS\/" . $FILER_C; #blessy: 861147 if ( !( -d "$log_dir" ) ) { system("sudo mkdir -p $log_dir"); } get_system_logs( Node => $Node, logdir => $log_dir ); logcomment("$FILER_C : System logs copied to the directory : $log_dir"); if ( $DRIVE_TYPE =~ /Tahiti/i ) { logcomment("$FILER_C : Copying TAHITI_SHELF_LOG logs to the directory $log_dir"); nvme_get_tahiti_shelf_log( Node => $Node, Destination => $log_dir ); logcomment("$FILER_C : TAHITI_SHELF_LOG are copied to the directory $log_dir"); } if ( $RUNID !~ /Client_IO|NVMe_Perf/ ) { logcomment("$FILER_C : Copying sk_trace logs to the directory $log_dir"); nvme_get_sk_trace_logs( Node => $Node, Destination => $log_dir, runtime => 300 ); logcomment("$FILER_C : Sk_trace logs are copied to the directory $log_dir"); } } sleep 20; } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 14 : Check for drive path on both nodes "); $Test->nlog("==================================================================================="); my @Nodes = NACL::C::Node->find(); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /CLI/gi ) { logcomment("$FILER_C : Check storage failover status"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" ); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;storage disk show -fields diskpathnames" ) if ( $out eq '' ); my @miss_drv; my @local_nodes = NACL::C::Node->find(); my $FILER_A = $local_nodes[0]->name(); my $FILER_B = $local_nodes[1]->name(); logcomment("FILER A : $FILER_A and FILER_B : $FILER_B"); foreach my $line ( split /\n/, $out ) { my ( $drv, $path ); next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change|\[\S+\]/ ); if ( ( $line =~ /$FILER_A/ ) && ( $line =~ /$FILER_B/ ) ) { my ( $drv, $path ) = $line =~ /(\S+)\s+(\S+)/; logcomment("Drive $drv has path $path"); } else { my ( $drv, $path ) = $line =~ /\S+\:(\S+)\s+(\S+)/; logcomment("Drive $drv path is incomplete"); push( @miss_drv, $drv ); } } if (@miss_drv) { logcomment("!!!! DRIVE PATH MISSING : @miss_drv"); logcomment("**FATAL** : DRIVE PATH MISSING : @miss_drv"); logresult( "FATAL", "DRIVE PATH MISSING : @miss_drv" ); } sleep 30; } else { logcomment("$FILER_C : is in $filer_state state, so storage disk show not supported"); } } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 15 : Check port link state for all drives"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my @link_dis_drivs; my $FILER_C = $Node->name(); if ( $execution_state eq 'CLI' ) { my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); logcomment("$FILER_C : Check port link state for all drives"); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "set -rows 0;run local storage show psm" ); foreach my $line ( split /\n/, $out ) { next if ( $line =~ /disk|^-|displayed|debug|entries|^$|session\.change/ ); if ( $line =~ /\[.*]/ ) { if ( $line =~ /DIS\/LNK|LNK|DIS/ ) { my ( $drv, $state, $lnk_lan, $max_lnk, $neg_lnk, $med_mx_lnk, $dev_mx_la, $Neg_max, $lan_wd ) = ( $line =~ /(\[.*\])\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ); logcomment("Drive ID : $drv State is $state !!!!!!!!! "); logcomment("Drive port link states are : Disk ID : $drv\n Port Max Link Speed:$lnk_lan\n Negotiated Max Link Speed:$max_lnk\n Medium Link Speed:$neg_lnk\n Device Max Link Speed:$med_mx_lnk\n Port Max Lane Width:$dev_mx_la\n Negotiated Max Lanewidth:$Neg_max\n Lanewidth:$lan_wd\n"); logcomment("Drive ID : $drv State is $state"); push( @link_dis_drivs, $drv ); } } } if (@link_dis_drivs) { logcomment("***FATAL*** : $FILER_C : Port State for following drives are DISABLED or OFFLINE, Drive ID(s) are: @link_dis_drivs"); logresult( "FATAL", "Port State for following drives are DISABLED or OFFLINE, Drive ID(s) are: @link_dis_drivs" ); } } else { logcomment("$FILER_C : is in MAINTENANCE state, so storage show psm not supported"); } } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 16 : Check for any drives"); $Test->nlog("==================================================================================="); my @subtests_f; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER_C : Checking for failed drives in the system"); nvme_report_failed_drives( Node => $Node, dump_fail => 1 ); } $Test->nlog("==================================================================================="); $Test->nlog("==== POST_TEST STEP 17 : Enable shelf and disk fw"); $Test->nlog("==================================================================================="); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); if ( $execution_state eq 'CLI' ) { my $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $API_obj->set_timeout( "connectrec-timeout" => 1200000 ); $API_obj->execute_raw_command( command => "set -rows 0" ); $API_obj->execute_raw_command( command => "system timeout modify 0" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $API_obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $API_obj->execute_raw_command( command => "set -showallfields false" ); $API_obj->execute_raw_command( command => "set -showseparators \" \" " ); $API_obj->execute_raw_command( command => "\013" ); my $prompts_answers = [ ".*Are you sure you want to stop firmware update qualification test*" => "yes" ]; $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk_fw_update -S\"", 'connectrec-match_table' => $prompts_answers ); $API_obj->execute_raw_command( 'command' => "\013" ); logcomment("Disk fw update stopped"); logcomment("Disable fw debug level "); $API_obj->execute_raw_command( 'command' => "run local dbg level diskfw=0" ); $API_obj->execute_raw_command( 'command' => "\013" ); $API_obj->execute_raw_command( 'command' => "\013" ); $API_obj->execute_raw_command( 'command' => "run local options shelf.fw.auto.update" ); logcomment("Disable Shelf FW and Drive FW update"); $API_obj->execute_raw_command( 'command' => "run local options shelf.fw.auto.update on" ); $API_obj->execute_raw_command( 'command' => "\013" ); $API_obj->execute_raw_command( command => "run local options raid.background_disk_fw_update.enable off" ); $API_obj->bootargs_set( arg => "AUTOBOOT", value => "false" ); logcomment("AUTOBOOT set to False"); $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test; bootargs set disk-skip-fw-force? false\"" ); $API_obj->execute_raw_command( 'command' => "\013" ); $API_obj->execute_raw_command( 'command' => "run local \"priv set -q test; bootargs set shelf-skip-fw-force? false\"" ); $API_obj->execute_raw_command( 'command' => "\013" ); #$API_obj->bootargs_set( arg => "disk-skip-fw-force?", value => "true" ); #$API_obj->bootargs_set( arg => "shelf-skip-fw-force?", value => "true" ); $API_obj->execute_raw_command( 'command' => "\013" ); } } $Test->nlog("##################################################################################"); $Test->nlog("##"); $Test->nlog("## End of POST_TEST logs for Filer : @Filer_array"); $Test->nlog("##"); $Test->nlog("##################################################################################"); } sub nvme_sto_agg_show_spare_dsk { my (%opts) = validate( @_, { Node => { type => OBJECT } } ); my $Node = $opts{Node}; my @spare_drav; my $dsk_out; my $filer_spare_rst = {}; my $API_obj; my $FILER_C = $Node->name(); #$API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $API_obj = $Node->get_7m_or_nodescope_apiset(); $API_obj->set_timeout( "connectrec-timeout" => 1200000 ); $API_obj->execute_raw_command( command => "set -rows 0" ); $API_obj->execute_raw_command( command => "system timeout modify 0" ); $API_obj->execute_raw_command( command => "\013" ); sleep 2; $API_obj->execute_raw_command( command => "\013" ); $API_obj->execute_raw_command( 'command' => "event config modify -console-log-level INFO -console off" ); $dsk_out = $API_obj->execute_raw_command( command => "storage aggregate show-spare-disks -original-owner $FILER_C" ); $dsk_out = $API_obj->execute_raw_command( command => "storage aggregate show-spare-disks -original-owner $FILER_C" ) if ( $dsk_out eq '' ); logcomment("Get Sapre drive for Filer : $FILER_C"); foreach my $line ( split( /\n/, $dsk_out ) ) { next if ( $line =~ /Disk|Origina|Pool|Root|Spar|-------|entries/ ); next if ( $line =~ /left behind|not zeroed/ ); #aa6-167:0n.12 SSD-NVM solid-state - block 3.29TB 0B 3.49TB zeroed if ( $line =~ /(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ) { my $drive = $1; $filer_spare_rst->{$drive}->{'disk'} = $drive; $filer_spare_rst->{$drive}->{'type'} = $2; $filer_spare_rst->{$drive}->{'class'} = $3; $filer_spare_rst->{$drive}->{'rpm'} = $4; $filer_spare_rst->{$drive}->{'chksum'} = $5; $filer_spare_rst->{$drive}->{'usebldata'} = $6; $filer_spare_rst->{$drive}->{'root_usebl'} = $7; $filer_spare_rst->{$drive}->{'size'} = $8; $filer_spare_rst->{$drive}->{'status'} = $9; } elsif ( $line =~ /(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ) { my $drive = $1; $filer_spare_rst->{$drive}->{'disk'} = $drive; $filer_spare_rst->{$drive}->{'type'} = $2; $filer_spare_rst->{$drive}->{'class'} = $3; $filer_spare_rst->{$drive}->{'rpm'} = $4; $filer_spare_rst->{$drive}->{'chksum'} = $5; $filer_spare_rst->{$drive}->{'usebldata'} = $6; $filer_spare_rst->{$drive}->{'size'} = $7; $filer_spare_rst->{$drive}->{'status'} = $8; } } return $filer_spare_rst; } sub nvme_dsk_qual_outpt_parsed { my (%opts) = validate( @_, { Node => { type => OBJECT }, Dsk_output => { type => SCALAR }, } ); my $Node = $opts{Node}; my $disk_qual_output = $opts{Dsk_output}; my $dsk_qual_result = {}; if ( $disk_qual_output =~ /\:Started|Finished/ ) { my ($drive) = ( $disk_qual_output =~ /\[(\S+)\]\:/ ); print "Drie : $drive\n"; foreach my $line ( split( /\n/, $disk_qual_output ) ) { if ( $line =~ /Disk\s+\[(\S+)\s+(\S+)\s+(\S+)\]\s+Serial\s+Number\s+\=\s+(\S+)/ ) { $dsk_qual_result->{$drive}->{'DISK'} = $drive; $dsk_qual_result->{$drive}->{'vendor'} = $1; $dsk_qual_result->{$drive}->{'model'} = $2; $dsk_qual_result->{$drive}->{'fw_rev'} = $3; $dsk_qual_result->{$drive}->{'srl_no'} = $4; } $dsk_qual_result->{$drive}->{'data_rate'} = $2 if ( $line =~ /\[(\S+)\]:\s+Data Rate\:\s+(\S+)/ ); $dsk_qual_result->{$drive}->{'iops'} = $2 if ( $line =~ /\[(\S+)\]:\s+IOPS\:\s+(\S+)/ ); $dsk_qual_result->{$drive}->{'total_time'} = $2 if ( $line =~ /\[(\S+)\]:\s+Total\s+time\:\s+(\S+)/ ); $dsk_qual_result->{$drive}->{'non_med_err'} = $2 if ( $line =~ /\[(\S+)\]:\s+Non-medium\s+Error\s+Count\:\s+(\S+)/ ); $dsk_qual_result->{$drive}->{'pwr_on_tm'} = $2 if ( $line =~ /\[(\S+)\]:\s+Power\s+on\s+time\s+\(minutes\)\:\s+(\S+)/ ); } } elsif ( ( $disk_qual_output =~ /\:\s+Passed/ ) || ( $disk_qual_output =~ /\:\s+Failed/ ) ) { my $dsk; foreach my $line ( split( /\n/, $disk_qual_output ) ) { if ( $line =~ /disk\,result/ ) { print " $line \n"; } if ( $line =~ /(\S+)\,passed|failed/ ) { print "Disk - $1\n"; $dsk = $1; my @array = split /,/, $line; $dsk_qual_result->{$dsk}->{'DISK'} = shift @array; $dsk_qual_result->{$dsk}->{'RESULT'} = shift @array; $dsk_qual_result->{$dsk}->{'seq'} = shift @array; $dsk_qual_result->{$dsk}->{'test'} = shift @array; $dsk_qual_result->{$dsk}->{'test_name'} = shift @array; $dsk_qual_result->{$dsk}->{'first'} = shift @array; $dsk_qual_result->{$dsk}->{'end'} = shift @array; $dsk_qual_result->{$dsk}->{'block'} = shift @array; $dsk_qual_result->{$dsk}->{'que'} = shift @array; $dsk_qual_result->{$dsk}->{'ave'} = shift @array; $dsk_qual_result->{$dsk}->{'min'} = shift @array; $dsk_qual_result->{$dsk}->{'max'} = shift @array; $dsk_qual_result->{$dsk}->{'ave_stop'} = shift @array; $dsk_qual_result->{$dsk}->{'min_stop'} = shift @array; $dsk_qual_result->{$dsk}->{'max_stop'} = shift @array; $dsk_qual_result->{$dsk}->{'t_msecs'} = shift @array; $dsk_qual_result->{$dsk}->{'t_ops'} = shift @array; $dsk_qual_result->{$dsk}->{'iops'} = shift @array; $dsk_qual_result->{$dsk}->{'rate'} = shift @array; $dsk_qual_result->{$dsk}->{'s_err'} = shift @array; $dsk_qual_result->{$dsk}->{'h_err'} = shift @array; $dsk_qual_result->{$dsk}->{'p_err'} = shift @array; $dsk_qual_result->{$dsk}->{'d_err'} = shift @array; $dsk_qual_result->{$dsk}->{'seek_t'} = shift @array; $dsk_qual_result->{$dsk}->{'Glist_s'} = shift @array; $dsk_qual_result->{$dsk}->{'Glist_e'} = shift @array; $dsk_qual_result->{$dsk}->{'time_s'} = shift @array; $dsk_qual_result->{$dsk}->{'time_e'} = shift @array; $dsk_qual_result->{$dsk}->{'prod'} = shift @array; $dsk_qual_result->{$dsk}->{'vendor'} = shift @array; $dsk_qual_result->{$dsk}->{'fw'} = shift @array; $dsk_qual_result->{$dsk}->{'sn'} = shift @array; } } } else { logcomment("Cannot parser disk_qual output"); } return $dsk_qual_result; } sub nvme_get_disk_owned { my (%opts) = validate( @_, { Node => { type => OBJECT }, } ); my $Node = $opts{Node}; my $FILER_C = $Node->name(); my $Host = host($FILER_C); my @drve_local; my $API_obj; my $dsk_out; my $flag = 0; my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); logcomment("Filer : $FILER_C is in $filer_state state"); if ( $filer_state =~ /UP|CLI/i ) { $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $API_obj->set_timeout( "connectrec-timeout" => 1200000 ); $API_obj->execute_raw_command( command => "set -rows 0;system timeout modify 0" ); $dsk_out = $API_obj->execute_raw_command( command => "run local disk show -o $FILER_C" ); } elsif ( $filer_state =~ /MAINT/i ) { my $API_obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); my $sys_id_a = Storage::Common_Lib::sysconfig_v_data( node => $Node, API_Obj => $API_obj ); my $sys_id = $sys_id_a->{'system_id'}; logcomment("FILER : $FILER_C : System ID : $sys_id"); $dsk_out = $API_obj->execute_raw_command( 'command' => "disk show -s $sys_id" ); } else { logcomment("**WARNING**: Cannot get filer state, Check filer is accessible and in proper state"); $flag = 1; } if ( $flag == 0 ) { foreach my $line ( split( /\n/, $dsk_out ) ) { next if ( $line =~ /---/ ); next if ( $line =~ /DISK|NOTE/ ); if ( $line =~ /(\w+\.\w+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)\s+(\S+)/ ) { push( @drve_local, $1 ); } elsif ( $line =~ /(\w+\.\w+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ) { push( @drve_local, $1 ); } } } logcomment("Drive Belongs to Filer : $FILER_C : @drve_local"); return @drve_local; } sub nvme_rename_fw_files { my (%opts) = validate( @_, { Node => { type => OBJECT }, rename_to => { type => SCALAR, optional => 1, default => "99" }, disk_fw_location => { type => SCALAR }, } ); my $Node = $opts{Node}; my $rename_to = $opts{rename_to}; my $fw_file_loc_dir = $opts{disk_fw_location}; my $FILER_C = $Node->name(); my $Home = $ENV{HOME}; logcomment("Get Existing firmware revision from available drives"); my $disk_list_result = nvme_get_all_disk( node_present => [$Node] ); my @dsk = keys %$disk_list_result; my @disk_info = sort(@dsk); my %model = (); my %rev = (); my %model_change = (); my %new_disk_fw = (); my $flag = 0; my $new_disk_fw_p = []; $new_disk_fw_p = \%new_disk_fw; my @NT_fw_dsk; foreach my $dsk (@disk_info) { my $model_name = $disk_list_result->{$dsk}->{'MODEL'}; my $rev_num = $disk_list_result->{$dsk}->{'REV'}; $model{$dsk} = $model_name; $rev{$dsk} = $rev_num; } logcomment("Check for firmware file under $fw_file_loc_dir"); my @disk_fw = `cd $fw_file_loc_dir;ls *.LOD`; if ( @disk_fw < 1 ) { logcomment("NO DISK FIRMWARE FILES ARE AVAILABLE"); return ( 0, $new_disk_fw_p ); } else { logcomment("Following Firmwares found in the directory $opts{disk_fw_location}:"); logcomment("@disk_fw"); } foreach my $dsk (@disk_info) { my $model_name = $model{$dsk}; my $rev_num = $rev{$dsk}; $model_name =~ s/^\s+|\s+$//g; my $fw = $model_name . "\.$rev_num" . "\.LOD"; logcomment("DEBUG: Disk : $dsk ::: Model : $model_name ::: Rev No : $rev_num"); if ( !( $new_disk_fw{$dsk} ) ) { $new_disk_fw{$dsk} = $fw; if ( $rev_num =~ /(NA|NQ)(\d\d)/ ) { my $ver_num = $2; if ( $ver_num < 50 ) { logcomment("DEBUG: Current drive FW rev is $rev_num, which is less than 50 so renaming as NQ49"); $model_change{$model_name} = $model_name . "\.NQ49" . "\.LOD"; } elsif ( $ver_num >= 50 ) { logcomment("DEBUG: Current drive FW rev is $rev_num, which is more than 49 so renaming as NQ99"); $model_change{$model_name} = $model_name . "\.NQ99" . "\.LOD"; } } elsif ( $rev_num =~ /NT/ ) { my $msg = "**WARNING** : Disk $dsk Contains the NT Revision Number - $fw"; push( @NT_fw_dsk, $msg ); $flag = 1; # NORUN if the NT files are present. } } } if ( $flag == 1 ) { grep { logcomment("$_") } @NT_fw_dsk; return ( 1, $new_disk_fw_p ); } my @model_chnage = values %model_change; my $firmware_backup = "BACK_UP_FIRMWARES_FILES_" . $FILER_C; logcomment("Creating directory $firmware_backup"); my $src_file = "$Home/NDATE/FIRMWARE/DISKFW/$firmware_backup"; if ( !( -d "$Home/NDATE/FIRMWARE/DISKFW/$firmware_backup" ) ) { logcomment("Directory $Home/NDATE/FIRMWARE/DISKFW/$firmware_backup does not exist"); system("sudo mkdir $Home/NDATE/FIRMWARE/DISKFW/$firmware_backup"); system("sudo chmod 777 $Home/NDATE/FIRMWARE/DISKFW/$firmware_backup"); } else { logcomment("Directory $Home/NDATE/FIRMWARE/DISKFW/$firmware_backup already exist"); system("sudo chmod 777 $Home/NDATE/FIRMWARE/DISKFW/$firmware_backup"); } logcomment("Modifying and copying files @model_chnage to $firmware_backup "); my $cnt_model = scalar @model_chnage; my $cnt = 0; foreach my $model (@model_chnage) { logcomment("COPYING the $model to the $src_file path"); if ( $model =~ /LOD$/ ) { if ( $model =~ /(\w+)\.\w+\.LOD/ ) { my $mod = $1; my $output = system("sudo cp -rf $fw_file_loc_dir/$mod* $src_file/$model"); if ( $output == 256 ) { logcomment("**WARNING** : No firmware file found with model number : $mod "); } } } } my $files_present = `ls $Home/NDATE/FIRMWARE/DISKFW/$firmware_backup/`; if ( !$files_present ) { return ( 2, $new_disk_fw_p ); } logcomment("After copying files present under $src_file are \n $files_present"); logcomment("Copying the FIRMWARE Files to the /etc/ path"); my @copied_firmwares = copy_FW_files_to_filer_directory( Node => $Node, Files_path => "$src_file", Files => [@model_chnage] ); @copied_firmwares = sort @copied_firmwares; my @common = intersection( [ sort @model_chnage ], [@copied_firmwares] ); my @diff = array_diff( [ sort @common ], [ sort @model_chnage ] ); if ( !@copied_firmwares ) { logcomment("Firmware files which are not copied :: @diff ") if (@diff); return ( 3, $new_disk_fw_p ); } else { logcomment("All necessary firmware files :: @common :: to /mroot/etc/disk_fw are copied"); } logcomment("Deleting folder $src_file"); system("sudo rm -rf $src_file"); if ( -d "$src_file" ) { logcomment("Error occured during the deletion of the folder $src_file"); } return ( 4, $new_disk_fw_p ); } sub nvme_get_agg_dtls_n_create { my (%opts) = validate( @_, { Node => { type => OBJECT }, disk_list => { type => ARRAYREF, optional => 1 }, aggr_name => { type => SCALAR, optional => 1, default => "aggr_def" }, disk_count => { type => SCALAR, optional => 1, default => undef }, addinal_agg => { type => SCALAR, optional => 1, default => "no" }, } ); my $Node = $opts{Node}; my @drive_to_use; my $drive_count = $opts{disk_count}; my $arrg_name = $opts{aggr_name}; my $addinal_create = $opts{addinal_agg}; my @non_root_ag; my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $num = 1; my $append_name = $Node->name(); my $agg; my $Api_Set_Obj; my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); logcomment("Filer : $FILER_C is in $filer_state state"); if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200000 ); $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;set -rows 0;system timeout modify 0\"" ); } elsif ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); logcomment("Filer is in Maintenance mode, Aggregate creation has not define, Create NEW Burt for this requirement"); return (1); } logcomment("Check for any non-root aggregate existance "); my ( $non_rt_status, @non_rt_agg ) = nvme_check_non_root_aggr( Node => $Node ); if ( defined $opts{aggr_name} ) { $agg = "aggr_" . "$append_name" . "_" . $opts{aggr_name}; } else { $agg = "aggr_" . "$append_name"; } if ( defined scalar @{ $opts{disk_list} } ) { @drive_to_use = @{ $opts{disk_list} }; logcomment("Drives passed : @drive_to_use"); } if ( $non_rt_status == 0 ) { logcomment("No non root aggregate is present, Proceed with creation of aggregate"); logcomment("User specified parameters :"); logcomment("Aggregate name : $agg"); logcomment("Drive Count : $drive_count ") if ( defined $drive_count ); logcomment("Drive selected based on user choice : @drive_to_use") if ( defined scalar @{ $opts{disk_list} } ); logcomment("Create additional aggregate if non-root is already present : $addinal_create"); if (@drive_to_use) { logcomment("Create aggregate using drives : @drive_to_use"); my $rt = nvme_create_aggregate( Node => $Node, aggr_name => $agg, drive_to_use => \@drive_to_use ); sleep 3; logcomment("DEBUG : returned : $rt"); } else { logcomment("Drives are not passed to create aggregate, random drives will be used for creation"); my $rt = nvme_create_aggregate( Node => $Node, aggr_name => $agg, drv_count => $drive_count ); sleep 3; } } elsif ( $non_rt_status == 1 ) { logcomment("Non-root Aggregate is already present "); logcomment("Aggregate are : @non_rt_agg"); if ( $addinal_create =~ /yes/ ) { logcomment("User requires additional aggregate, Creating additional aggregate"); if (@drive_to_use) { my $rt = nvme_create_aggregate( Node => $Node, aggr_name => $agg, drive_to_use => \@drive_to_use ); sleep 3; } else { my $rt = nvme_create_aggregate( Node => $Node, aggr_name => $agg, drv_count => $drive_count ); sleep 3; } } else { logcomment("User has specified additonal aggregate option as $addinal_create, Hence additional aggregate will not be created"); return @non_rt_agg; } } else { logcomment("Non-root aggregate return is not valid, Please check manually"); } logcomment("Check for aggregates"); sleep 4; $Api_Set_Obj->execute_raw_command( command => "" ); my $agg_stat = $Api_Set_Obj->execute_raw_command( command => "run local aggr status -r " ); logcomment("$agg_stat"); logcomment("Check in Node scope"); my $non_rt_agg = $Api_Set_Obj->execute_raw_command( 'command' => "aggr show -fields root,node" ); foreach my $line ( split( /\n/, $non_rt_agg ) ) { next if ( $line =~ /---/ ); if ( $line =~ /(\S+)\s+\S+\s+(true|false)/ ) { logcomment("Aggregate $1 root parameter is $2"); if ( $line =~ /(\S+)\s+(\S+)\s+false/ ) { my $ag = $1; my $fl = $2; logcomment("Aggregate belogin to $FILER_C is $ag"); push( @non_root_ag, $ag ) if ( $fl =~ /$FILER_C/ ); } } } logcomment("$FILER_C : Non root aggregate(s) are :@non_root_ag"); return @non_root_ag; } sub nvme_create_aggregate { my (%opts) = validate( @_, { Node => { type => OBJECT }, aggr_name => { type => SCALAR }, drv_count => { type => SCALAR, optional => 1, default => undef }, drive_to_use => { type => ARRAYREF, optional => 1, default => undef }, } ); my $Node = $opts{Node}; my $FILER_C = $Node->name(); my $aggr_name = $opts{aggr_name}; my $drv_cont = $opts{drv_count}; my @drive_touse; if ( defined scalar @{ $opts{drive_to_use} } ) { @drive_touse = @{ $opts{drive_to_use} }; @drive_touse = shuffle @drive_touse; logcomment("SHUFFLING DRIVES TO PICK DRIVES IN RANDOM ORDER : < @drive_touse > TO CREATE AGGREGATE"); } $aggr_name =~ s/-/_/g; logcomment("DEBUG : Aggregate name : $aggr_name"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; if ($drv_cont) { logcomment("Required Aggregate Drive Count : $drv_cont"); my $spare_list = nvme_sto_agg_show_spare_dsk( Node => $Node ); my @spare_drvs = keys %$spare_list; my $spare_count = @spare_drvs; logcomment("$FILER_C : Total spare are : $spare_count and Spare Drive are : @spare_drvs"); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate create -aggregate $aggr_name -diskcount $drv_cont -node $FILER_C", 'connectrec-match_table' => $prompts_answers ); if ( $out =~ /Error|command failed/ ) { logcomment("Aggregation creation failed"); return 1; } else { return 0; } } elsif (@drive_touse) { logcomment("NVMe : Drives @drive_touse will be used to create aggregate "); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; my $out = $Api_Set_Obj->execute_raw_command( 'command' => "storage aggregate create -aggregate $aggr_name -disklist @drive_touse -node $FILER_C", 'connectrec-match_table' => $prompts_answers ); if ( $out =~ /Error|command failed/ ) { logcomment("Aggregation creation failed"); return 1; } else { return 0; } } else { logcomment("Drive count or list of drives are not passed, cannot create non-root aggregate"); return 1; } } #################################################################################################### # NVME - Get All nnon root volumes #Retrun : Array ofnon root volumes # burt: #################################################################################################### sub nvme_check_non_root_aggr { my (%opts) = validate( @_, { Node => { type => OBJECT } } ); my $Node = $opts{Node}; my $FILER_C = $Node->name(); my @non_rot_aggr; my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); logcomment("Check for non-root aggregates"); my $non_rt_agg = $Api_Set_Obj->execute_raw_command( 'command' => "aggr show -fields root,node" ); foreach my $line ( split( /\n/, $non_rt_agg ) ) { next if ( $line =~ /---/ ); if ( $line =~ /(\S+)\s+\S+\s+(true|false)/gm ) { logcomment("Aggregate $1 root parameter is $2"); if ( $line =~ /(\S+)\s+(\S+)\s+false/ ) { my $ag = $1; my $fl = $2; logcomment("Aggregate belogin to $FILER_C is $ag"); push( @non_rot_aggr, $ag ) if ( $fl =~ /$FILER_C/ ); } } } if (@non_rot_aggr) { logcomment("$FILER_C : Non root aggregate(s) are : @non_rot_aggr"); logcomment("Return all non root aggregate names belongs to $FILER_C"); return ( 1, @non_rot_aggr ); } else { logcomment("Non-Root aggregate cannot be found for filer $FILER_C"); return ( 0, @non_rot_aggr ); } } #################################################################################################### # NVME - Drive firmware to lower version, downgrade firmware from /FIRMWARE/DISKFW/LOW # Return : Test Status ( 1 - Pass | 0 - Failed ) # Burt : # Author : arunak #################################################################################################### sub nvme_disk_fw_update_low_or_high { my (%opts) = validate( @_, { Node => { type => OBJECT }, FW_DIR_LOC => { type => SCALAR }, FORCE_UPDTE => { type => SCALAR, optional => 1, default => 'yes' }, } ); my $Node = $opts{Node}; my $disk_dir = $opts{FW_DIR_LOC}; my $force_update = $opts{FORCE_UPDTE}; my $Home = $ENV{HOME}; my %firmware_hash = (); my @firmware_list = (); my $test_status = 0; logcomment("MAP Firmware, Copy Firmware from $disk_dir and force update : $force_update"); # my $disk_dir = "$Home/NDATE/FIRMWARE/DISKFW/LOW"; my $firmware_list = nvme_map_fw_dsk_rev_model( Node => $Node, disk_fw_location => $disk_dir, ); %firmware_hash = %$firmware_list; my %hash; my ( $key, $value ); while ( ( $key, $value ) = each %firmware_hash ) { push( @{ $hash{$value} }, $key ); } my @drive_list = grep { (/\S/) } keys(%firmware_hash); logcomment("The list of drives on which lower firmware will download : @drive_list "); @firmware_list = unique( values(%firmware_hash) ); logcomment("The list of lower firmware files present : @firmware_list "); if ( !@firmware_list ) { logcomment("**FAIL**: NO LOWER FIRMWARE FILES ARE AVAILABLE FOR FILER "); $test_status = 1; return $test_status; } logcomment("Copying the FW files to filer directory"); my $FILER_C = $Node->name(); logcomment( "Lower firmware files to be copied are " . scalar(@firmware_list) . " - @firmware_list " ); my @firmwares; foreach (@firmware_list) { my $val = $disk_dir . "/" . $_; push( @firmwares, $val ); } logcomment("Calling Common lib subroutine copy_fw"); my @fw_files = copy_FW_files_to_filer_directory( Node => $Node, Files => \@firmwares ); if ( !@fw_files ) { logcomment("**FAIL**: $FILER_C - DISK FIRMWARE FILES ARE NOT FOUND IN /etc/disk_fw"); $test_status = 1; # to mark as sub routine fail return $test_status; } logcomment("Firmware files copied, Proceeding with update based on force update"); logcomment("DEBUG : FORCE UPDATE : $force_update"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" ); $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=1" ); my $time_out = 1000; my @drv_loaded_pas; my @fw_dwl_drives; if ( $force_update =~ /yes/i ) { logcomment("$FILER_C :Firmware option foce update selected : $force_update"); foreach my $value ( keys %hash ) { $Api_Set_Obj->set_timeout( "connectrec-timeout" => $time_out ); my @arr_t = split( /\./, $value ); my $prod_id = $arr_t[0]; logcomment("DEBUG: Executing disk_fw_update -F $value $prod_id"); my @prompts = ( 'Are you sure you want to' => 'y', ); my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"disk_fw_update -F $value -t $prod_id\"", "connectrec-match_table" => \@prompts ); logcomment("Wait for few seconds"); sleep(60); #logcomment("OUTPUT : $output "); foreach my $line ( split( /\n/, $output ) ) { #STARTING download request - 1 requests pending to shelf 0 -- disk 0n.8 if ( $line =~ /STARTING download request/ ) { if ( $line =~ /\s+disk\s+(\S+)/ ) { logcomment("Starting Firmware download on drive : $1"); chomp($1); push( @fw_dwl_drives, $1 ); } logcomment("Started Firmware download on drives @fw_dwl_drives"); } if ( $line =~ /Firmware is up-to-date/ ) { logcomment("Firmware is up-to-date on all eligible disks"); } } logcomment("Check console messages"); my $c_out = $Api_Set_Obj->execute_raw_command( 'command' => " " ); logcomment("CONSOLE: $c_out"); my $drv; push( @drv_loaded_pas, $drv ); } logcomment("Firmware downloaded to @fw_dwl_drives"); } else { logcomment("Valid Fimrware files copied from $disk_dir to filer $FILER_C"); } my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); sleep 2; $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=0" ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); sleep 2; if ( $force_update =~ /yes/i ) { return @fw_dwl_drives; } else { return $test_status; } } #################################################################################################### # NVME - Map/Match Firmware with disk , revision model. from given location/directory # Return : Hash of drive with firmware # Burt : # Author : arunak #################################################################################################### sub nvme_map_fw_dsk_rev_model { my (%opts) = validate( @_, { Node => { type => OBJECT }, disk_fw_location => { type => SCALAR }, timeout => { type => SCALAR, default => 1200, optional => 1 }, } ); my $Node = $opts{Node}; my $disk_fw_path = $opts{disk_fw_location}; my $sudo = "/usr/bin/sudo"; my $Mode = $Node->mode(); my %new_disk_fw = (); my $new_disk_fw_p = []; $new_disk_fw_p = \%new_disk_fw; my %model = (); my %rev = (); my @disk_fw = `cd $opts{disk_fw_location};ls *.LOD`; my @failed_array; logcomment("Following Firmwares found in the directory $opts{disk_fw_location}:"); logcomment("@disk_fw"); # This function validates the same firmwares with same product id's if ( @disk_fw < 1 ) { logcomment("NO DISK FIRMWARE FILES ARE AVAILABLE"); return ($new_disk_fw_p); } my $disk_list_result = nvme_get_all_disk( node_present => [$Node] ); my @dsk = keys %$disk_list_result; my @disk_info = sort(@dsk); foreach my $dsk (@disk_info) { my $model_name = $disk_list_result->{$dsk}->{'MODEL'}; my $rev_num = $disk_list_result->{$dsk}->{'REV'}; my $drive_type = $disk_list_result->{$dsk}->{'Type'}; my $flag = 0; foreach my $fw (@disk_fw) { $fw =~ s/^\s+|\s+$//g; $model_name =~ s/^\s+|\s+$//g; if ( $fw =~ /^$model_name\.\S+\.LOD$/ ) { logcomment(" Disk : $dsk ::: Model : $model_name ::: Rve No : $rev_num"); $flag = 1; if ( !( $new_disk_fw{$dsk} ) ) { $new_disk_fw{$dsk} = $fw; } } ## if ( $fw =~ /$model_name/ )a } if ( $flag == 0 ) { push( @failed_array, "No firmware file found for the disk $dsk with model number : $model_name and version $rev_num" ); } } grep { logcomment("$_") } @failed_array; return ($new_disk_fw_p); } #################################################################################################### # NVME - Eye Catcher to be called in all NVMe-Scripts # Return : N/A # Burt : # Author : arunak #################################################################################################### sub nvme_eye_catcher { my (%opts) = validate( @_, { Test => { type => OBJECT }, string => { type => SCALAR, default => undef, optional => 1 }, } ); my $Test = $opts{Test}; my $string = $opts{string}; $Test->nlog("====================================================================================================="); $Test->nlog("=== NVMe = $string"); $Test->nlog("====================================================================================================="); } #################################################################################################### # NVME - List of all drive with all Fields from storage show disk command ( CLI and MAINT ) # Return : Hash of all drive details - # Burt : # Author : arunak #################################################################################################### sub nvme_get_all_disk { my (%opts) = validate( @_, { node_present => { type => ARRAYREF }, } ); my @Nodes = @{ $opts{node_present} }; my $dsk_result = {}; foreach my $node (@Nodes) { my $Api_Set_Obj; my $FILER_C = $node->name(); my $Host = host($FILER_C); logcomment("Get all NVMe drives attached"); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); my $output; if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;storage show disk -T -x\"" ); } elsif ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "storage show disk -T -x" ); } my @lines = split( /\n/, $output ); my $n = 0; while ( $n <= $#lines ) { last if ( $lines[$n] =~ /^-----/ ); $n++; } $n++; foreach my $line ( @lines[ $n .. $#lines ] ) { my ( $disk, $shlf, $bay, $serial, $vendor, $model, $rev, $type ) = ( $line =~ /(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ); logcomment("DEBUG : $line "); if ( ( $line =~ /\d\w\.\d\d/ ) || ( $line =~ /\d\w\.\d\d\.\d\d/ ) || ( $line =~ /\d\w\.\S+/ ) || ( $line =~ /\d\w\.\d\d\.\S+/ ) ) { $dsk_result->{$disk} = { 'DISK' => $disk, 'SHELF' => $shlf, 'BAY' => $bay, 'SERIAL' => $serial, 'VENDOR' => $vendor, 'MODEL' => $model, 'REV' => $rev, 'TYPE' => $type, }; } } last; } return $dsk_result; } #################################################################################################### # NVME - List of all drive with all Fields from disk_list command ( CLI and MAINT ) # Return : Hash of all drive details # Burt : # Author : arunak #################################################################################################### sub nvme_list_only_nvme { my (%opts) = validate( @_, { node_present => { type => ARRAYREF }, } ); my @Nodes = @{ $opts{node_present} }; my $dsk_result = {}; foreach my $node (@Nodes) { my $Api_Set_Obj; my $FILER_C = $node->name(); my $Host = host($FILER_C); logcomment("Get all NVMe drives attached"); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); my $output; if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $node->get_7m_or_nodescope_apiset( connid => 'console' ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk_list\"" ); } elsif ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 ); $output = $Api_Set_Obj->execute_raw_command( command => "disk_list" ); } my @lines = split( /\n/, $output ); my $n = 0; while ( $n <= $#lines ) { last if ( $lines[$n] =~ /^-----/ ); $n++; } $n++; foreach my $line ( @lines[ $n .. $#lines ] ) { # my ( $disk, $ven, $prodid, $rev, $serialno, $hw, $blocks, $bps, $dq ) = ($line =~ /(\S+)\s+(.*)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/); my ( $disk, $chan, $ven, $prodid, $rev, $serialno, $hw, $blocks, $bps, $dq ) = ( $line =~ /(\S+)\s+(.*)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ); if ( ( $line =~ /\d\w\.\d\d/ ) || ( $line =~ /\d\w\.\d\d\.\d\d/ ) || ( $line =~ /\d\w\.\S+/ ) || ( $line =~ /\d\w\.\d\d\.\S+/ ) ) { $dsk_result->{$disk} = { 'DISK' => $disk, 'VENDOR' => $ven, 'PRODUCT_ID' => $prodid, 'REV' => $rev, 'SERIAL' => $serialno, 'HW' => $hw, 'BLOCKS' => $blocks, 'BPS' => $bps, 'DQ' => $dq }; } } last; } return $dsk_result; } #################################################################################################### # NVME - Checks system health ( CLI and MAINT ) # Return : Hash ref # Burt : 1159588 # Author : khosur #################################################################################################### sub check_system_health { my (%opts) = validate( @_, { Nodes => { type => ARRAYREF }, } ); my @Nodes = @{ $opts{Nodes} }; my $Transit_obj; my $Api_Set_Obj; my %APISet_Objs; my %test_result; my $filer_state; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $host = host($FILER_C); ##Burt 1159588 - re-opened - check added for loader. $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("$FILER_C : Filer is in $filer_state"); if ( $filer_state =~ /LOADER|FIRMWARE/ ) { logcomment("FILER : $FILER_C is in $filer_state state, Booting in maintenance mode"); nvme_boot_to_maint( Node => $Node ); logcomment("FILER : $FILER_C booted in MAINT mode"); } logcomment("DEBUG : Check filer state again, before creating API"); $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer is in $filer_state"); if ( $filer_state eq 'MAINT' ) { $Api_Set_Obj = NACL::APISet->new( hostobj => $host, category => "Node", interface => "CLI", set => "Maintenance", connid => "console" ); } elsif ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); } $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 ); $APISet_Objs{$FILER_C} = $Api_Set_Obj; } #1.Check for failed drives – FATAL if failed drives are present. ( Do not proceed ) my @subtests; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER_C : Checking for failed drives in the system"); push( @subtests, subtest( \&nvme_report_failed_drives, -runid, "nvme_report_failed_drives_" . "$FILER_C", -bg, "--", Node => $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); #2.Check for any un-owned drives If present – Assign drives equally, and return 1. logcomment("Checking for un-owned drives If present Assigning drives equally"); my $return; foreach my $Node (@Nodes) { $return = nvme_chk_unowned_drv( Filer_State => $filer_state, API_Obj => $APISet_Objs{ $Node->name() } ); last if ($return); } if ($return) { nvme_drv_assign( Nodes => \@Nodes, force => 0, assign => 'yes' ); logcomment("Drives are assigned equally"); $test_result{'Drive_Assign'} = $return; # return $return; } #3.Print System Path – Single / Multi ( No need to check system path ) logcomment("Checking System Path"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Transit_obj = NACL::Transit->new( name => $FILER_C ); $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /MAINT/i ) { my $sys_path = Storage::Common_Lib::sysconfig_v_data( node => $Node, API_Obj => $APISet_Objs{$FILER_C} ); my $system_path = $sys_path->{'path'}; logcomment("FILER : $FILER_C : System Path : $system_path"); } elsif ( $filer_state =~ /UP|CLI/i ) { my $response_obj = $APISet_Objs{$FILER_C}->sysconfig( "verbose" => 1 ); my $parsed_output = $response_obj->get_parsed_output(); logcomment("$FILER_C : System Path : $parsed_output->[0]->{'system_storage_configuration'}"); } } #4.Check for any drive are zeroing. ( Print Warning and proceed ) foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Transit_obj = NACL::Transit->new( name => $FILER_C ); $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /UP|CLI/i ) { logcomment("$FILER_C : Checking any drives are zeroing"); my @Zeroing_Spares = nvme_get_local_spares( API_con => $APISet_Objs{$FILER_C}, Filer => $FILER_C, handle => 2 ); if (@Zeroing_Spares) { my $dsks = _print_array( \@Zeroing_Spares ); logcomment( "**WARNING** : $FILER_C - Total number of zeroing/not zeroed drives are " . scalar @Zeroing_Spares . " and the drives are :\n" . $dsks . " " ); } else { logcomment("$FILER_C : No zeroing spares found"); } } } #5.Check any Stale or offline aggregate, if root is offline or stale, return 1 . #if non-root is present, Delete aggregate and zero spares. foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Transit_obj = NACL::Transit->new( name => $FILER_C ); $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /UP|CLI/i ) { logcomment("$FILER_C : Checking any Stale or offline aggregates are present"); my $return = nvme_del_nonroot_stale_aggr_and_start_zeroing( node => $Node, API_Obj => $APISet_Objs{$FILER_C} ); if ( $return == 1 ) { logcomment("$FILER_C : Stale root aggregate is present"); $test_result{$FILER_C}{'Stale_Root_Aggr'} = $return; } } if ( $filer_state =~ /MAINT/i ) { logcomment("$FILER_C : Checking root aggregates are present"); my $aggr_output = $APISet_Objs{$FILER_C}->execute_raw_command( 'command' => "aggr status -r" ); if ( $aggr_output =~ /No root aggregate found/ig || $aggr_output =~ /You must specify a root aggregate/ig || $aggr_output =~ /No usable root volume was found!/ig || $aggr_output =~ /restricted|destroyed|frozen/ig || $aggr_output =~ /wafl\.volinfo\.fsinfo\.error/ig ) { logcomment("$FILER_C : No root aggregate is present"); $test_result{$FILER_C}{'No_root_aggr'} = 1; } } } return \%test_result; } #################################################################################################### # NVME - Checks stale root and non root aggregate. Deletes stale data aggregate if present # Return : 0 : If stale root aggregate not present # 1 : If stale root aggregate is present # Burt : # Author : khosur #################################################################################################### sub nvme_del_nonroot_stale_aggr_and_start_zeroing () { my (%opts) = validate( @_, { node => { type => OBJECT }, API_Obj => { type => OBJECT }, } ); my $API_Set_Obj = $opts{API_Obj}; my $Node = $opts{node}; my $FILER_C = $Node->name(); ## Checking non root aggr my @rootaggr; my $aggr_output = $API_Set_Obj->execute_raw_command( 'command' => "run local aggr status" ); foreach my $var ( split( /\n/, $aggr_output ) ) { if ( $var =~ /\s*Aggr\s*State/ ) { next; } if ( $var =~ /\s*(\S+)\s*\S+\s*\S+/i ) { my $agr = $1; if ( $agr =~ /aggr/ ) { push( @rootaggr, $agr ) if ( $agr eq /aggr0/i ); } } } if (@rootaggr) { logcomment("Stale root aggregate is present : @rootaggr"); return 1; } ## Delete non-root stale aggr and start spares zeroing aggregate_volume_delete( Node => $Node ); ## zeroing spares my @spare_disk = nvme_get_local_spares( API_con => $API_Set_Obj, Filer => $FILER_C, handle => 1 ); if ( scalar @spare_disk ) { my $dsks = _print_array( \@spare_disk ); logcomment( "Filer : $FILER_C : Number of zeored spare drives are " . scalar @spare_disk . " and the spares are:\n" . $dsks ); } return 0; } #################################################################################################### # NVME - Reports failed disk # Return : FATAL if failed disks present # Burt : # Author : khosur #################################################################################################### sub nvme_report_failed_drives { my (%opts) = validate( @_, { Node => { type => OBJECT }, dump_fail => { type => SCALAR, optional => 1, default => 0 }, } ); my $Node = $opts{Node}; my $dump_fail = $opts{dump_fail}; logcomment("DEBUG : Dump failures - $dump_fail"); my $FILER_C = $Node->name(); my ( $failed, $failed_u, $sysconfig_flg ) = ( 0, 0, 0 ); my @broken; my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /CLI|UP/ ) { my @broken_disks_sysconfig = (); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 ); my $response_obj = $Api_Set_Obj->sysconfig( "verbose" => 1 ); my $sysconfig_out = $response_obj->get_raw_output(); chomp( my @sysconf_lines = split( /\r?\n/, $sysconfig_out ) ); my ( $drv, $model, $fw, $size, $bytes_sec, $state, $adapter ); foreach my $line (@sysconf_lines) { if ( $line =~ /\s*(\d+)\s*\:\s*\w*\s*(\w*)\s*(\w*)\s*(\d*\.\w*)\s*\((.*)\)\s*\((.*)\)/i ) { ( $drv, $model, $fw, $size, $bytes_sec, $state ) = $line =~ /\s*(\d+)\s*\:\s*\w*\s*(\w*)\s*(\w*)\s*(\d*\.\w*)\s*\((.*)\)\s*\((.*)\)/i; logcomment("drv : $drv Model : $model FW : $fw Size : $size Bytes_sec : $bytes_sec State : $state"); my $drive = "0n" . "." . $drv if ( $state =~ /Failed/i ); logcomment("Failed drive is : $drive"); push( @broken_disks_sysconfig, $drive ) if ($drive); } } $sysconfig_flg = 1 if ( !( scalar @broken_disks_sysconfig ) ); ## Assigning the broken disks to array @broken if ( scalar(@broken_disks_sysconfig) ) { @broken = @broken_disks_sysconfig; } ## return nothing if there are no failed drives if ($sysconfig_flg) { logcomment("$FILER_C : NO UNKNOWN BROKEN DISKS found for the filer"); return; } } elsif ( $filer_state =~ /MAINT/ ) { logcomment("Checking for failed disk - Maintenance mode"); my $result = disk_show_v( Node => $Node ); my @res_disks = keys $result; foreach (@res_disks) { if ( $result->{$_}->{POOL} =~ /FAILED/ ) { push( @broken, $_ ); } } } logcomment("Broken/failed disks are : @broken") if ( scalar @broken ); ## If broken disks present read Failure information bytes and send mail if (@broken) { if ( $filer_state =~ /CLI|UP/ ) { my $output; my @broken_dsk; my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 ); $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local disk show -v", 'timeout' => 600 ); foreach my $disk ( split( "\n", $output ) ) { if ( $disk =~ /^(\d\S\.\d+)\s.*.*(FAILED|BROKEN)/ig ) { push( @broken_dsk, $1 ); } } my @disk_list = @broken_dsk; @broken_dsk = (); foreach my $dsk (@disk_list) { if ( $dsk =~ /(P\d+)$/ ) { next; } push( @broken_dsk, $dsk ); } @broken = (); @broken = @broken_dsk; my $dsks = _print_array( \@broken ); if ( scalar @broken ) { logcomment( "**FATAL** : $FILER_C : Number of FAILED disks are " . scalar @broken . " and the disks are: " . $dsks ); } foreach my $disk (@broken) { $output = $Api_Set_Obj->execute_raw_command( 'command' => "system node run -node $FILER_C -command \" priv set -q test; scsi dumpfailures -s $disk\"", 'timeout' => 600 ); logcomment("Failure information bytes read from disk $disk: $output"); } } elsif ( $filer_state =~ /MAINT/ ) { my $Host = host($FILER_C); my $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 ); foreach my $disk (@broken) { my $output = $Api_Set_Obj->execute_raw_command( 'command' => "scsi dumpfailures $disk", 'timeout' => 600 ); logcomment("Failure information bytes read from disk $disk: $output"); } } if ( $dump_fail == 1 ) { logcomment("Dump failure flag is enabled, Dump drive failure logs"); foreach my $drv (@broken) { logcomment("Dump failure info for drive : $drv"); logcomment("Filer State is : $filer_state"); dump_drv_failure_info( Node => $Node, filer_state => $filer_state, drive => $drv ); logcomment("Completed dumping failure information for drive $drv"); } } my $mail_subject = "INFO : FAILED drives are present in the filer $FILER_C"; my $mail_body = "################################################################################################# ## ## FAILED drives are present in the filer $FILER_C ## ## Number of failed drives are " . scalar @broken . " and the drives are @broken" . " ## ## Stopping the test suite execution : $STEST_FILE ## #################################################################################################"; if ( $EMAIL !~ /d/i ) { my %mail_params = ( HTML => $mail_body, Text => $mail_body, From => $MAIL_FROM, To => $MAIL_TO, Subject => $mail_subject, ); $mail_params{HTML} = "
$mail_params{Text}<\\HTML>";
            my $mail = HTML::Mail->new(%mail_params);
            $mail->send() or logcomment("CONFIG : Trouble sending mail");
        } else {
            logcomment("Email NOT sent because email set to debug by user.");
        }
        logcomment("**FATAL** :FAILED DRIVES are present in the filer $FILER_C, so killing the test suite : $STEST_FILE");
        return logresult( 'FATAL', "FAILED DRIVES ARE PRESENT IN THE FILER $FILER_C, so killing the test suite : $STEST_FILE" );
    }
}

####################################################################################################
# NVME - Checks any un-owned drive present in the setup.
# Return : Returns 1 if any un-owned drive is present
# Burt :
# Author : khosur
####################################################################################################
sub nvme_chk_unowned_drv {
    my (%opts) = validate(
        @_,
        {
            Filer_State => { type => SCALAR },
            API_Obj     => { type => OBJECT },
        }
    );

    my $filer_state = $opts{Filer_State};
    my $Api_Set_Obj = $opts{API_Obj};

    my @unowned;
    my $unowned = 0;
    logcomment("DEBUG:Node is in the Filer state $filer_state");
    my $output;

    if ( $filer_state =~ /UP|CLI/i ) {
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 );
        $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk show -n\"" );
    } elsif ( $filer_state =~ /MAINT/i ) {
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 );
        $output = $Api_Set_Obj->execute_raw_command( command => "disk show -n" );
    }
    foreach my $line ( split( /\n/, $output ) ) {
        if ( $line =~ /Not Owned|Unowned/ ) {
            if ( $line =~ /(\S+)\s+\S+\s+\S+\s+\S+/ ) {
                logcomment("DEBUG DRIVE : $1");
                push( @unowned, $1 );
                $unowned = 1;
            }
        }
    }

    if ($unowned) {
        return 1;
    } else {
        return 0;
    }
}
1;