#!/usr/software/bin/perl # $Id: //depot/prod/test/Rsierranevada/storage/hdd/NADQ_SEA/CMODE/Check_filer_setup.thpl#18 $ # Copyright (c) 2005 Network Appliance, Inc. # All rights reserved # ## @summary Mode ## check_filer_setup.thpl ## ## @description ## This script will take care of creating the cluster on C-Mode Filer's. ## ## @status Automated ## @auther arjun.lavu@netapp.com ## @burt ## @change YY/MM/DD from author: Description of the change ############################################################################# ######################################### ### Library functions ########################################## # Compiler directives. use strict; use Storage::Common_Lib; use Storage::NVMe_Common_Lib; use Storage::Tahiti_Common_Lib; use POSIX; use Net::Ping::External qw(ping); my $ping_host = Net::Ping->new(); sub kill_process(;$); my $params = NATE::ParamSet->new( global => 1 ); my $FILER = $params->get( 'FILER', default => undef ); my $Filer_mode = $params->get( 'Filer_mode', default => 'F' ); my $EMAIL = $params->get( 'EMAIL', default => undef ); my $MAIL_TO = $params->get( 'MAIL_TO', default => undef ); my $MAIL_FROM = $params->get( 'MAIL_FROM', default => undef ); my $EOE = $params->get( 'EOE', default => undef ); my $TEST_CONFIG = $params->get( 'TEST_CONFIG', default => undef ); my $BOOT_MODE = $params->get( 'BOOT_MODE', default => undef ); my $TEST_SUITE_NAME = $params->get( 'TEST_SUITE_NAME', default => undef ); my $FILER_TYPE = $params->get( 'FILER_TYPE', default => 'SN' ); my $Primary_Node = $params->get( 'Primary_Node', default => undef ); my $KERNEL_TYPE = $params->get( 'KERNEL_TYPE', default => undef ); my $GROUP_NO = $params->get( 'GROUP_NO', default => undef ); my $LOGDIR = $params->get( 'LOGDIR', default => undef ); my $BOOT_TYPE = $params->get( 'BOOT_TYPE', default => undef ); my $KERNEL_SEL = $params->get( 'KERNEL', default => undef ); my $CHECK_FILER_SETUP = $params->get( 'CHECK_FILER_SETUP', default => 'Y' ); my $FIPS_SELECTED = $params->get( 'DRIVE_TYPE', default => 'NO' ); my $DRIVE_TYP_SLDT = $params->get( 'DRIVE_TYPE', default => 'NO' ); my $SKIP_CHK_FIL_SET = $params->get( 'SKIP_CHK_FIL_SET', default => 'NO' ); ######################################### ### Initialization/declaration ######################################### my $TC_name = "check_filer_setup"; my $Home = $ENV{HOME}; my $NVMe = 0; my $Tahiti = 0; my $Saber_tahiti = 0; my $SCM = 0; #checking the partner id declaration my %sysconfig_sysID; my %flag_set_patner_id; # End log parameters my $main_log_file = "$LOGDIR/$TC_name" . "\.log"; ##Delete .ssh file system("rm -rf $Home/.ssh/known_hosts"); my ( $fail_cnt, $fatal_cnt, @results, @subtests, $Mode, $Filer_names, $f_state, @FILER_MAINT, $Original_State, $time_out, @Nodes, @FILER_ARR, %FILER_ARR, %nodes_filer, $Transit_obj, $K_type_filer ); my $Cluster = 0; my $test_status = 0; my ( $email_subject, $email_body ); my @Testcases; ################################################################################# ## Available main test are : ## 1) NVMe_setup -- For NVMe setup with drives NVM-SSD ## 2) Cluster --- For normal setup with drive SSD/HDD ################################################################################# ## NVMe Drive type check. if ( $DRIVE_TYP_SLDT =~ /NVMe/ ) { $NVMe = 1; logcomment("User has selected NVMe Drive type, Proceeding with NVMe check setup steps"); @Testcases = ( NVMe_setup => "Check setup which has NVMe drives" ); } elsif ( $DRIVE_TYP_SLDT =~ /Tahiti/ ) { $Tahiti = 1; logcomment("User has selected Tahiti shelf Type, Tahiti shelf must be connected to Apollo"); @Testcases = ( Tahiti_Test => "Check Tahiti system" ); } elsif ( $DRIVE_TYP_SLDT =~ /Saber_tahiti/ ) { $Saber_tahiti = 1; logcomment("User has selected Saber Tahiti"); @Testcases = ( Tahiti_Test => "Check Saber system" ); } elsif ( $DRIVE_TYP_SLDT =~ /SCM/ ) { $SCM = 1; logcomment("User has selected SCM / OCZ drive test"); @Testcases = ( SCM_Test => "SCM / OCZ drive test" ); } else { logcomment("User has selected $DRIVE_TYP_SLDT drive type, Proceeding with normal check filer setup steps"); @Testcases = ( Cluster => "Find all nodes in the cluster" ); } my $Home = $ENV{HOME}; my @Filers; if ( $FILER =~ /\,/ ) { @Filers = split( /\,/, $FILER ); } else { push( @Filers, $FILER ); } logcomment("Number of filers @Filers"); foreach my $name (@Filers) { #------------------------------------------------------------ ## Checking if filer info for the filers are available at home direcory NDATE folder #------------------------------------------------------------ logcomment("Checking if filer info for the filers are available at home direcory NDATE folder"); my $filer = "$name" . "_C"; if ( -e "$Home/NDATE/FILER_INFO/$filer" ) { system("sudo cp -rf $Home/NDATE/FILER_INFO/$filer $Home/NDATE/FILER_INFO/$name"); } else { logcomment("**FATAL** : NO CONFIGURATION FILE PRESENT FOR THE FILER $name IN THE DIRECTORY $Home/NDATE/FILER_INFO/$filer"); logresult( 'FATAL', "No configuration file present for the filer $name in the directory $Home/NDATE/FILER_INFO/$filer" ); #kill_process($err); } } &main(); sub main { # Debug break point $DB::single = 2; # Create Test Case Driver object $Test = new TCD( -testcases => [@Testcases] ); if ( $Test->error ) { $Test->log( $Test->errmsg ); return $TCD::FAIL; } # Performs method callbacks $Test->run_test(); if ( $Test->error ) { $Test->log( $Test->errmsg ); return $TCD::FAIL; } exit(0); } ## end sub main ########## INIT ############################################ # init automatically called before executing tests ############################################################# sub init() { $Test->description("check_filer_setup : Finding Nodes"); logcomment("..init code.."); $Filer_names = $Test->get_param("FILER"); @FILER_ARR = split( /,/, $Filer_names ); logcomment("Filers present are $Filer_names"); #------------------------------------------------------------ ## INIT : Step 1 - Display NDATE and PNATE versions #------------------------------------------------------------ my $str = "INIT : STEP 1 : Display NDATE and PNATE versions"; $str = "NVMe - STEP 1 : Display NDATE and PNATE versions" if ( $NVMe == 1 ); $str = "Tahiti - STEP 1 : Display NDATE and PNATE version" if ( $Tahiti == 1 ); $str = "Saber_Tahiti - STEP 1 : Display NDATE and PNATE version" if ( $Saber_tahiti == 1 ); $str = "SCM/OCZ - STEP 1 : Display NDATE and PNATE version" if ( $SCM == 1 ); eye_catcher( Test => $Test, string => $str, ); pnate_versions_display(); #------------------------------------------------------------ ## INIT : Step 2 - Check if filer connection closed by foreign host or not #------------------------------------------------------------ my $str = "INIT : STEP 2 : Check if filer connection closed by foreign host or not"; $str = "NVMe - STEP 2 : Check if filer connection closed by foreign host or not" if ( $NVMe == 1 ); $str = "Tahiti - STEP 2 : Check if filer connection closed by foreign host or not" if ( $Tahiti == 1 ); $str = "Saber_Tahiti - STEP 2 : Check if filer connection closed by foreign host or not" if ( $Saber_tahiti == 1 ); $str = "SCM/OCZ - STEP 2 : Check if filer connection closed by foreign host or not" if ( $SCM == 1 ); eye_catcher( Test => $Test, string => $str, ); my @Nodes_temp = NACL::C::Node->find(); foreach my $Node (@Nodes_temp) { my $Filer = $Node->name(); $nodes_filer{$Filer} = [$Node]; } @Nodes = map { @{$_} } grep { /\S/ } values(%nodes_filer); $Mode = $Nodes[0]->mode(); logcomment("Mode of the Filers $Mode"); check_filer_connection( Filer_name => \@FILER_ARR ); if ( ( $NVMe == 0 ) && ( $Tahiti == 0 ) && ( $SCM == 0 ) && ( $Saber_tahiti == 0 ) ) { #------------------------------------------------------------ ## INIT : Step 3 - Check if the kernal version is supported on the filer platform #------------------------------------------------------------ eye_catcher( Test => $Test, string => "INIT : STEP 3 : Check if the kernal version is supported on the filer platform", ); #Burt 1033916 nmamatha my @subtests; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&check_valid_platform, -runid, "check_valid_platform_$FILER_C", -bg, "--", Node => $Node, Test => $Test ) ); } Subtest::wait_finish( subtest => [@subtests] ); #------------------------------------------------------------ ## INIT : Step 4 - Check if both filers are in the same state #------------------------------------------------------------ eye_catcher( Test => $Test, string => "INIT : STEP 4 : Check if both filers are in the same state", ); # my $cluster = 0; my $FILERA = $Nodes[0]->name(); my $FILERB = $Nodes[1]->name(); my $Transit_obj_A = NACL::Transit->new( name => $FILERA ); my $Transit_obj_B = NACL::Transit->new( name => $FILERB ); my $filer_state_A = $Transit_obj_A->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILERA is in $filer_state_A state"); my $filer_state_B = $Transit_obj_B->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILERB is in $filer_state_B state"); if ( $filer_state_A !~ /$filer_state_B/ ) { logcomment("Both filers are in DIFFERENT state, booting to Maintenance"); $Transit_obj_A->change_state( to => "MAINT" ); $Transit_obj_B->change_state( to => "MAINT" ); } else { logcomment("Both filers are in same state"); } foreach my $fil (@Nodes) { my $FILER_C = $fil->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); if ( $filer_state =~ /DBLADECLI/i ) { logcomment("$FILER_C is in $filer_state, changing to CLI"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); $Server->console( cmd => "exit" ); } } logcomment("As user has selected -b=7, check_filer setup skips the check for cluster") if ( $CHECK_FILER_SETUP eq 'N' ); #------------------------------------------------------------ ## INIT : Step 5 - Setting up cluster and checking if node names are valid #------------------------------------------------------------ if ( $CHECK_FILER_SETUP eq 'Y' ) { eye_catcher( Test => $Test, string => "INIT : STEP 5 : Setting up cluster and checking if node names are valid", ); my $filerA = $Nodes[0]->name(); my $filerB = $Nodes[1]->name(); my $Transit_obj = NACL::Transit->new( name => $filerA ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); my $host = Tharn::host($filerA); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $filerA ); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $cluster_set = 0; if ( $filer_state =~ /UP|CLI/i ) { nodename_interchange_checkup(); logcomment("Node Names Interchange checkup completed successfully"); logcomment("$filerA is in $filer_state, checking cluster health"); eval { $Server->console( cmd => "set test", additional_prompts_timeout => 3600, ignore => $ignore_strings, additional_prompts => { ".*want to continue.*" => "y", "::*>" => "FINAL_PROMPT", "$prompt_fw" => "FINAL_PROMPT" } ); }; if ($@) { logcomment("Inside eval"); } my $output = $Server->console( cmd => "cluster show", additional_prompts_timeout => 3600, ignore => $ignore_strings, ".*::*>.*" => "FINAL_PROMPT", "$prompt_fw" => "FINAL_PROMPT" ); if ( ( $output =~ /$filerA/ ) && ( $output =~ /$filerB/ ) ) { logcomment("Cluster is already enabled"); } elsif ( $output =~ /is not a recognized command/i ) { logcomment("CLUSTER IS NOT ENABLED. NOW ,CLUSTER IS GOING TO CREATE"); $cluster_set = 1; } else { logcomment("CLUSTER IS NOT ENABLED. NOW ,CLUSTER IS GOING TO CREATE"); $cluster_set = 1; } } else { logcomment("Filer is not in CLI to check cluster health"); } if ( ( $cluster_set == 1 ) && ( ( $GROUP_NO != 1 ) || ( $GROUP_NO != 2 ) || ( $GROUP_NO != 7 ) ) ) { ##For booting the filer with option 4 and creating the root aggregate, root volume and cluster. setup_cmode_filers( node_present => \@Nodes, Test => $Test, ); logcomment("Booting completed"); nodename_interchange_checkup(); } #------------------------------------------------------------ ## INIT : Step 5a - Check for normal and encrypted drives in the filer #------------------------------------------------------------ eye_catcher( Test => $Test, string => "INIT : STEP 5a : Check for normal and encrypted drives in the filer", ); my @subtests; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&check_encrypt_disk, -runid, "check_encrypt_disk_$FILER_C", -bg, "--", $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); } logcomment("Node Names Interchange checkup completed successfully"); if ( $Mode =~ /7Mode/i ) { logcomment("**FATAL** : FILER CONFIG IS IN 7MODE,SO CANNOT PROCEED FURTHER"); logresult( 'FATAL', "Filer config is in 7Mode, So cannot proceed furthur" ); #kill_process($err); } # pnate_versions_display(Test => $Test) logcomment("TEST_CONFIG value is $TEST_CONFIG"); } elsif ( ( $NVMe == 1 ) || ( $Tahiti == 1 ) || ( $Saber_tahiti == 1 ) ) { eye_catcher( Test => $Test, string => "NVME Check STEP 1 : Check filer state and accesibility of filer" ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); nvme_eye_catcher( Test => $Test, string => "CEHCK NVME PRINT" ); logcomment("Check State of filer"); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state(); logcomment("FILER : $FILER_C state is $filer_state"); if ( $filer_state =~ /MAINT/ ) { logcomment("Filer is in Maintenance mode, Proceed checking drive list"); } elsif ( $filer_state =~ /BOOT_MENU/ ) { logcomment("Filer is in boot menu, change to maintenace "); } elsif ( $filer_state =~ /CLI/ ) { logcomment("Filer is in file system mode "); logcomment("New logic "); logcomment("Check ssh connection"); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $host = host($FILER_C); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); my $out = $Server->console( cmd => "system timeout modify -timeout 0 ;rows 0;net int show", additional_prompts_timeout => 3600 ); if ( $out =~ /not a recognized command|table is currently empty/ ) { logcomment("Filer : $FILER_C is in CLI, but not in CLUSTER MODE, Booting to maintenance mode"); $Transit_obj->change_state( to => "MAINT" ); my $Original_State = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILER_C is in $Original_State"); } else { logcomment("Cluster configured"); my $prompts_answers = [ ".*continue?" => 'y' ]; my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); sleep 2; } sleep 1; } elsif ( $filer_state =~ /USERNAME/ ) { logcomment("Filer is in login state, Change to CLI "); $Transit_obj->change_state( to => "CLI" ); logcomment("Filer is in CLI"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*continue?" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); sleep 2; } else { logcomment("Filer state cannot be checled"); } } } elsif ( $SCM == 1 ) { logcomment("SCM/OCZ drive is selected"); logcomment("Basic check of SCM will be performed"); } else { logcomment("Drive Typr or NDATE type is not selected "); } return $TCD::PASS; } ## end sub init() ########## SETUP ##################################### # setup automatically called before executing tests ####################################################### sub setup() { $Test->description("check_filer_setup : Set-up test bed"); if ( ( $NVMe == 0 ) && ( $Tahiti == 0 ) && ( $SCM == 0 ) && ( $Saber_tahiti == 0 ) ) { logcomment("As user has selected -b=7, successive checks will be skipped.") if ( $CHECK_FILER_SETUP eq 'N' ); if ( $CHECK_FILER_SETUP eq 'Y' ) { my @subtests; my $cluster = 0; #------------------------------------------------------------ ## SETUP : Step 1 - Check for filer connections, System partner Id #------------------------------------------------------------ eye_catcher( Test => $Test, string => "SETUP : STEP 1 : Check for filer connections, System partner Id", ); if ( ( $GROUP_NO == 1 ) || ( $GROUP_NO == 2 ) || ( $GROUP_NO == 7 ) ) { #Burt 1041403 nmamatha logcomment("As the group selected by user is $GROUP_NO which is for MAINTENANCE mode scripts, NOT checking for system_partner_id"); } else { logcomment("As the group selected by user is $GROUP_NO which is for CLI mode scripts, checking for system_partner_id"); check_system_partner_id(@Nodes); } #------------------------------------------------------------ ## SETUP : Step 2 - Check for kernel type and switch based on the required type of kernel #------------------------------------------------------------ eye_catcher( Test => $Test, string => "SETUP : STEP 2 : Check for kernel type and switch based on the required type of kernel", ); my @subtests_2; foreach (@Nodes) { my $FILER_C = $_->name(); push( @subtests_2, subtest( \&check_install_kernel_type, -runid, "check_install_kernel_type_" . "$FILER_C", -bg, "--", $_, ) ); } Subtest::wait_finish( subtest => [@subtests_2] ); #------------------------------------------------------------ ## SETUP : Step 3 - Check if kernel selected and kernel on filer are same #------------------------------------------------------------ eye_catcher( Test => $Test, string => "SETUP : STEP 3 : Check if kernel selected and kernel on filer are same", ); { logcomment("Selected Kernel version is $KERNEL_SEL"); logcomment("Checking on Filer, before proceeding with other steps"); foreach my $fil (@Nodes) { my $FILER_C = $fil->name(); my $Host = host($FILER_C); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state"); my $node_version; if ( $filer_state =~ /UP|CLI/i ) { my $Api_Set_Obj = $fil->get_7m_or_nodescope_apiset(); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); $node_version = $Api_Set_Obj->execute_raw_command( 'command' => 'run local version' ); } elsif ( $filer_state =~ /MAINT/i ) { my $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $API_Object->set_timeout( "connectrec-timeout" => 1200 ); $node_version = $API_Object->execute_raw_command( command => "version" ); } else { logcomment("VERSION COULD NOT BE FOUND"); } my $user_sel_kern; $user_sel_kern = "LongBoard" if ( $KERNEL_SEL =~ /LB/ ); $user_sel_kern = "FullSteam" if ( $KERNEL_SEL =~ /FS/ ); logcomment("User Selected ONTAP version - $user_sel_kern"); if ( ( $user_sel_kern =~ /LongBoard/ ) && ( $node_version =~ /Longboard|R8\.4|disktest_lb0|RlongboardN|8_4_[0-9]/ ) ) { logcomment("Selected ONTAP is same as on filer"); } elsif ( ( $user_sel_kern =~ /FullSteam/ ) && ( $node_version =~ /FullSteam|RfullsteamN/ ) ) { logcomment("Selected ONTAP is same as on filer"); } else { logcomment("Selected ONTAP - $user_sel_kern is not available on Filer $FILER_C"); logcomment("**WARNING** : Selected ONTAP - $user_sel_kern is not available on Filer $FILER_C"); #logresult ("FATAL", "Selected ONTAP - $user_sel_kern is not available on Filer $FILER_C"); } } } #------------------------------------------------------------ ## SETUP : Step 4 - Check if drives are equally assigned on both nodes Burt#1003912 #------------------------------------------------------------ eye_catcher( Test => $Test, string => "SETUP : STEP 4 : Check if drives are equally assigned on both nodes", ); { logcomment("Checking for drives presence on filers"); logcomment("If drives are not assigned equally, NDATE will remove and assign drives respectively"); disk_remv_assign_ownership( Node_ref => \@Nodes ); logcomment("Remove assign of drives completed, Proceedign with next steps"); } #------------------------------------------------------------ ## SETUP : Step 5 - Partition setup and remove partition if selected group is 5 #------------------------------------------------------------ if ( ( $GROUP_NO == 5 ) || ( $GROUP_NO == 7 ) || ( $GROUP_NO =~ /NADQ02_SSD_Durability/ ) || ( $GROUP_NO =~ /NADQ02_ERT_/ ) || ( $GROUP_NO == 6 ) ) { eye_catcher( Test => $Test, string => "SETUP : STEP 5 : Partition setup and remove partition if selected group is 5", ); logcomment("Selected Group is $GROUP_NO, which requires unpartition drives"); logcomment("Checking type of system - PARTITION or NON-PARTITION"); check_partition_remove_evn_parti_var(); } else { logcomment("Check for partition, If unpartition, Take out environment variable and partition drives"); my $partition_enabled_2 = 0; foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filers is in $filer_state state"); my $API_Object; my $drive_data; if ( $filer_state =~ /CLI|USER|pass/i ) { $Transit_obj->change_state( to => "CLI" ); $API_Object = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $API_Object->set_timeout( "connectrec-timeout" => 6000 ); $drive_data = $API_Object->execute_raw_command( 'command' => "run local disk show -v" ); } else { if ( $filer_state !~ /MAINT/ ) { $Transit_obj->change_state( to => "MAINT" ); } my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /MAINT/ ) { $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); } $API_Object->set_timeout( "connectrec-timeout" => 6000 ); $drive_data = $API_Object->execute_raw_command( 'command' => "disk show -v" ); } logcomment("**DEBUG**: API_OBJECT and Drive data collected"); my @data = split( /\n/, $drive_data ); my @disk_part; foreach my $line (@data) { if ( $line =~ /(\S+)\s+\S+\s*\(\d+/ ) { my $disk = $1; if ( $disk =~ /P\d+$/ ) { push( @disk_part, $disk ); $partition_enabled_2 = 1; } } } if ( $partition_enabled_2 == 1 ) { last; } } # ENd of foreach Node if ( ( $partition_enabled_2 == 0 ) && ( $FIPS_SELECTED =~ /SSD$/i ) ) { my @re_subtest; foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $filer_name = $Node->name(); logcomment( "Calling change_state " . scalar( localtime() ) ); push( @re_subtest, subtest( \&boot_filer_in_req_state, -runid, "enb_parti_sdown_node_" . "$filer_name", -bg, "--", filer_name => $filer_name, change_state_to => 'FIRMWARE' ) ); } Subtest::wait_finish( subtest => [@re_subtest] ); logcomment("Booted Filers in FIRMWARE"); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $FILER = $Node->name(); my $host = host($FILER); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; my $Transit_obj = NACL::Transit->new( name => $FILER ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("FILER is in LOADER prompt, disabling auto partition env variables"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); $Server->console( cmd => 'unsetenv root-uses-shared-disks? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'unsetenv allow-ssd-partitions? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'unsetenv root-uses-shared-ssds? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'saveenv', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); logcomment("Set all required environment variables to enable autopartition"); } else { logcomment("CANNOT SET ENVIRONMENT VARIABLE , AS FILER IS NOT IN LOADER PROMPT"); } } #end of foreach node, setting env variable logcomment("Set-up Filer, Re-initilize filer"); setup_cmode_filers( node_present => \@Nodes, Test => $Test ); logcomment("unpartitioned setup and re-initilization completed"); } else { logcomment("Parttion not enabled or valid for this system"); } #End of partitonenable=0 } } #end of if($CHECK_FILER_SETUP eq 'Y') } elsif ( ( $NVMe == 1 ) || ( $Tahiti == 1 ) || ( $Saber_tahiti == 1 ) ) { eye_catcher( Test => $Test, string => "NVMe - SETUP Code" ); ##Modified for Burt-1233883 khosur if ( $NVMe == 1 ) { my $partition_check = 0; my $Api_Set_Obj; my %APISet_Objs; my $filer_state; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer state is $filer_state"); if ( $filer_state =~ /FIRMWARE/ ) { $Transit_obj->change_state( to => "MAINT" ); logcomment("Filer state is $filer_state now"); } } foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $host = host($FILER_C); $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("$FILER_C : Filer is in $filer_state"); if ( $filer_state eq 'MAINT' ) { $Api_Set_Obj = NACL::APISet->new( hostobj => $host, category => "Node", interface => "CLI", set => "Maintenance", connid => "console" ); } elsif ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset(); } $Api_Set_Obj->set_timeout( "connectrec-timeout" => 7200 ); $APISet_Objs{$FILER_C} = $Api_Set_Obj; } logcomment("Checking for un-owned drives If present Assigning drives equally"); my $return; my $boot_to_maint_flg = 0; my $partition = 0; my %filer_states; foreach my $Node (@Nodes) { $return = nvme_chk_unowned_drv( Filer_State => $filer_state, API_Obj => $APISet_Objs{ $Node->name() } ); last if ($return); } if ($return) { foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); $filer_states{$FILER_C} = $filer_state; if ( $filer_state =~ /CLI/ || $filer_state =~ /FILRMWARE/ ) { $Transit_obj->change_state( to => "MAINT" ); $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer state is $filer_state"); $boot_to_maint_flg = 1; } else { logcomment("Filer state is $filer_state already"); } } logcomment("Checking whether system is partitioned"); my $output = $APISet_Objs{ $Nodes[0]->name }->execute_raw_command( 'command' => "disk show -n" ); my @lines = split( /\n/, $output ); # skip to the start of table contents my $n = 0; while ( $n <= $#lines ) { last if ( $lines[$n] =~ /^-----/ ); $n++; } $n++; foreach my $line ( @lines[ $n .. $#lines ] ) { if ( ( $line =~ /Not Owned/g ) ) { my ($disk) = ( $line =~ /(\S+)\s+Not Owned\s+(.*)/ ); if ( $disk =~ /P\d+$/ ) { $partition = 1; logcomment("Setup is partitioned"); last; } } } if ($partition) { logcomment("Setup is partitioned. So assigning partitions"); nvme_drv_assign( Nodes => \@Nodes, force => 1, assign => 'no' ); assign_partitions(); } else { nvme_drv_assign( Nodes => \@Nodes, force => 1, assign => 'yes' ); } } ## Checking again if there are any un-owned drives are present foreach my $Node (@Nodes) { $return = nvme_chk_unowned_drv( Filer_State => $filer_state, API_Obj => $APISet_Objs{ $Node->name() } ); last if ($return); } if ($return) { logcomment("**FAIL** : Drives are still NOT assigned properly. Please check"); } if ($boot_to_maint_flg) { foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Transit_obj->change_state( to => $filer_states{$FILER_C} ) if ( $filer_states{$FILER_C} !~ /FILRMWARE/ ); logcomment("$FILER_C Change to $filer_states{$FILER_C}"); } } } } elsif ( ( $SCM == 1 ) ) { eye_catcher( Test => $Test, string => "SCM/OCZ - Setup Code .... " ); } else { logcomment("No drive or NDATE type selected"); } return $TCD::PASS; } #------------------------------------------------------------ ##SMC/OCZ #------------------------------------------------------------ sub SCM_Test { my $test_status = 0; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Perform basic checks"); logcomment("Check filer setup basic checks completed"); } return $TCD::PASS; } #------------------------------------------------------------ ## Tahiti #------------------------------------------------------------ sub Tahiti_Test { $Test->description("check_filer_setup : Tahiti Mode "); my $filer_ker_ver = {}; my $status = 0; my $system_health = 1; #Change to 0 . my $return = 0; my $return_saber = 0; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Perform basic checks"); my $version = basic_check_n_setup( Node => $Node ); logcomment("$FILER_C : $version"); $filer_ker_ver->{$FILER_C}->{'KERNEL'} = $version; logcomment("Check filer setup basic checks completed"); } if ( $Saber_tahiti == 1 ) { logcomment("Checking for port status and state for saber system"); $return = nvme_saber_check_port( Nodes => \@Nodes ); if ( $return == 1 ) { $status = 1; logcomment("**FATAL** : Port are DISABLE/OFFLINE "); logresult( 'FATAL', "Port are DISABLE/OFFLINE" ); } else { logcomment("Ports are Enable/Online for system"); } } else { logcomment("System is not a Saber system, Skipping check for port state and status"); } logcomment("Check user selected group"); logcomment("USER SELECTED GROUP : $GROUP_NO"); logcomment("User has selected -b=7 , Check filer setup will not be executed") if ( $CHECK_FILER_SETUP eq 'N' ); if ( $CHECK_FILER_SETUP eq 'Y' ) { #kernel check logcomment("User selected $GROUP_NO group "); if ( ( $GROUP_NO == 2 ) || ( $GROUP_NO == 4 ) || ( $GROUP_NO == 7 ) || ( $GROUP_NO == 8 ) || ( $GROUP_NO == 9 ) || ( $GROUP_NO == 10 ) ) { logcomment("User selected $GROUP_NO group, which requires NON_DEBUG Kernel "); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Check Filer state and Set Test privilage and execute basic setup commands"); my $cu_ver = $filer_ker_ver->{$FILER_C}->{'KERNEL'}; if ( $cu_ver =~ /<1d>/ ) { logcomment("$FILER_C : Kernel is DEBUG, $GROUP_NO Group requires NON_DEBUG Kernel"); $status = 1; logcomment("**FATAL** : $FILER_C : Install NON_DEBUG KERNEL Manually or use NDATE opiton 9 -> option 5 -> option 3 and provide non-debug kernel"); logresult( 'FATAL', "$FILER_C : Install NON_DEBUG KERNEL Manually or use NDATE opiton 9 -> option 5 -> option 3 and provide non-debug kernel" ); } else { logcomment("$FILER_C : has NON_DEBUG Kernel, Proceeding with next steps"); } } } elsif ( ( $GROUP_NO == 3 ) || ( $GROUP_NO == 5 ) || ( $GROUP_NO == 6 ) || ( $GROUP_NO == 1 ) ) { logcomment("User selected $GROUP_NO group, which requires DEBUG Kernel "); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Check Filer state and Set Test privilage and execute basic setup commands"); my $cu_ver = $filer_ker_ver->{$FILER_C}->{'KERNEL'}; if ( $cu_ver =~ /<1d>/ ) { logcomment("$FILER_C : has DEBUG Kernel, Proceeding with next steps"); } else { logcomment("$FILER_C : Kernel is NON_DEBUG, $GROUP_NO Group requires DEBUG Kernel"); logcomment("**FATAL** : $FILER_C : Install DEBUG KERNEL Manually or use NDATE opiton 9 -> option 5 -> option 3 and provide debug kernel"); $status = 1; logresult( 'FATAL', "$FILER_C : Install DEBUG KERNEL Manually or use NDATE opiton 9 -> option 5 -> option 3 and provide debug kernel" ); } } } #Setup System - based on group if ( ( $GROUP_NO == 3 ) || ( $GROUP_NO == 2 ) || ( $GROUP_NO == 4 ) || ( $GROUP_NO == 8 ) || ( $GROUP_NO == 9 ) || ( $GROUP_NO == 1 ) ) { logcomment("User selected group $GROUP_NO, expects system to be PARTITIONED"); my @tahiti_drv; my @internal_drv; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Transit_obj->set( 'timeout' => 7200 ); $Transit_obj->set( 'change_state_timeout' => 7200 ); my $FILER_C_STATE = $Transit_obj->get_state( 'timeout' => 7200, 'get_state_timeout' => 7200 ); logcomment("Filer : $FILER_C is in $FILER_C_STATE"); my $dsk_out; if ( $FILER_C_STATE =~ /CLI/ ) { logcomment("Filer is in CLI/File system mode"); my $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' ); $API_Obj->set_timeout( "connectrec-timeout" => 8887200 ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $API_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); logcomment("Privilage level test set"); $API_Obj->execute_raw_command( 'command' => "set -rows 0" ); logcomment("Get drive list"); $dsk_out = $API_Obj->execute_raw_command( 'command' => "run local disk_list" ); } elsif ( $FILER_C_STATE =~ /MAINT/ ) { logcomment("Filer is in Maintenance mode"); my $Host = host($FILER_C); my $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance", ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 8887200 ); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); $dsk_out = $Api_Set_Obj->execute_raw_command( 'command' => "disk_list" ); } foreach my $line ( split( /\n/, $dsk_out ) ) { if ( $line =~ /(\S+\.\S+)\s+/ ) { my $drv = $1; if ( $drv =~ /\S+\.\S+\.\S+\.\S+/ ) { push( @tahiti_drv, $drv ); } else { push( @internal_drv, $drv ); } } } if (@internal_drv) { logcomment("Check for internal drive slot"); logcomment("Internal drives : @internal_drv"); my $slot_staus = tahiti_check_drv_slots( APOLLO_DRVS => \@internal_drv ); logcomment("DEBUG: Drive Slot Status : $slot_staus"); } my $dsks = _print_array( \@tahiti_drv ); logcomment("Tahiti Drives : $dsks"); last; } #my $system_health = tahiti_system_health_check(Nodes => \@Nodes, part_flag => $partition); $system_health = 1; #Burt to check health, hardcode to 1 until developed. if ( $system_health == 1 ) { my $part = 1; my $return = tahiti_boot_main_unpart_assign_parts( NODES => \@Nodes, partition => $part ); if ($return) { logcomment("**FATAL** : Partions are NOT assigned properly. Please check"); $status = 1; logresult( 'FATAL', "Partions are NOT assigned properly. Please check" ); } } else { logcomment("System Appears to be healthy, Proceeding with next steps"); } } elsif ( ( $GROUP_NO == 5 ) || ( $GROUP_NO == 6 ) || ( $GROUP_NO == 7 ) || ( $GROUP_NO == 10 ) ) { logcomment("User selected group $GROUP_NO, expects system to be UN-PARTITIONED"); logcomment("System health check : $system_health"); if ( $system_health == 1 ) { my $part = 0; my $return = tahiti_boot_main_unpart_assign_parts( NODES => \@Nodes, partition => $part ); if ($return) { logcomment("**FATAL** : Partions are NOT assigned properly. Please check"); $status = 1; logresult( 'FATAL', "Partions are NOT assigned properly. Please check" ); } } else { logcomment("System Appears to be healthy, Proceeding with next steps"); } } else { logcomment("Invalid Group Selected"); logcomment("**FATAL** : Invalid Group Selected"); $status = 1; logresult( 'FATAL', "Invalid Group Selected" ); } #Boot as required by group if ( ( $GROUP_NO == 2 ) || ( $GROUP_NO == 7 ) || ( $GROUP_NO == 1 ) ) { logcomment("User selected group $GROUP_NO, Which required system to be in MAINTENANCE MODE"); logcomment("Checking State of Filers"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Transit_obj->set( 'timeout' => 7200 ); $Transit_obj->set( 'change_state_timeout' => 7200 ); my $FILER_C_STATE = $Transit_obj->get_state( 'timeout' => 7200, 'get_state_timeout' => 7200 ); logcomment("Filer : $FILER_C is in $FILER_C_STATE"); if ( $FILER_C_STATE =~ /MAINT/ ) { logcomment("Filer is in Maintenance mode"); } else { $Transit_obj->change_state( to => "CLI" ); logcomment("Filer $FILER_C changed to Maintenance mode"); } } } elsif ( ( $GROUP_NO == 3 ) || ( $GROUP_NO == 4 ) || ( $GROUP_NO == 5 ) || ( $GROUP_NO == 6 ) || ( $GROUP_NO == 8 ) || ( $GROUP_NO == 9 ) || ( $GROUP_NO == 10 ) ) { logcomment("User selected group $GROUP_NO, Which required system to be in FILE SYSTEM MODE "); if ( $system_health == 1 ) { if ( $Saber_tahiti == 1 ) { logcomment("Setup is saber, un-partition all drives before wipe_config"); my $part = 0; my $return = tahiti_boot_main_unpart_assign_parts( NODES => \@Nodes, partition => $part ); } logcomment("Checking State of Filers and re-initilizing"); my @maint_subtest; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Transit_obj->set( 'timeout' => 7200 ); $Transit_obj->set( 'change_state_timeout' => 7200 ); my $FILER_C_STATE = $Transit_obj->get_state( 'timeout' => 7200, 'get_state_timeout' => 7200 ); logcomment("Filer : $FILER_C is in $FILER_C_STATE"); if ( $FILER_C_STATE =~ /CLI|UP/ ) { logcomment("Filer is in FILE-SYSTEM MODE"); } else { logcomment("Change state to LOADER"); $Transit_obj->change_state( to => "FIRMWARE" ); logcomment("Filer :$FILER_C changed to LOADER"); logcomment("Re-initilize filer, create cluster and join cluster "); my @maint_subtest; my $env_data = "NA"; push( @maint_subtest, subtest( \&nvme_clean_config, -runid, "clean_config_" . "$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) ); logcomment("$FILER_C : Wipe config started "); } } Subtest::wait_finish( subtest => [@maint_subtest] ); logcomment("Re-initilize / wipe config process completed"); logcomment("Create cluster and Join"); create_join_cluster( node_present => [@Nodes], nvme_setup => "yes" ); logcomment("NVMe-Tahiti System setup completed"); } else { logcomment("System Appears to be healthy, Proceeding with next steps"); } } else { logcomment("Invalid Group Selected"); logcomment("**FATAL** : Invalid Group Selected"); $status = 1; logresult( 'FATAL', "Invalid Group Selected" ); } } else { logcomment("User has selected -b=7, check filer setup will not execute"); logcomment("Make sure setup has required kernel, partitioned or un-partitioned based on group selected"); } if ( $status == 1 ) { return $TCD::FAIL; } else { return $TCD::PASS; } } #------------------------------------------------------------ ## NVME System Setup part #------------------------------------------------------------ sub NVMe_setup { my $test_status = 0; my $setup_cmode_check = 0; my $partition_check = 0; my $flag = 0; logcomment("User has selected -b=7 , Check filer setup will not be executed") if ( $CHECK_FILER_SETUP eq 'N' ); logcomment("User Select test which has to be executed from LOADER, Skip check_filer_setup") if ( $SKIP_CHK_FIL_SET eq 'YES' ); $CHECK_FILER_SETUP = "N" if ( $SKIP_CHK_FIL_SET eq 'YES' ); if ( $CHECK_FILER_SETUP eq 'Y' ) { eye_catcher( Test => $Test, string => "Check Basic setup configuration and boot according to group selected" ); my $partition_check = 0; my $flag = 0; eye_catcher( Test => $Test, string => "Checking the system health" ); my $return_health = check_system_health( Nodes => \@Nodes ); #Burt No: 1159588 #Check Kernel and switch kernel based on group ##Burt No 1159589 - kiranp1 eye_catcher( Test => $Test, string => "Check kernel on both nodes, and group selected by user" ); my @subtests_2; foreach (@Nodes) { my $FILER_C = $_->name(); push( @subtests_2, subtest( \&check_install_kernel_type, -runid, "check_install_kernel_type_" . "$FILER_C", -bg, "--", $_, ) ); } Subtest::wait_finish( subtest => [@subtests_2] ); my $status = status_return( subtest_ref => [@subtests_2] ); logcomment("Test status is $status"); if ( $status ne 0 ) { return $TCD::FAIL; } ##Burt No:1159591, Burt No: 1159590 eye_catcher( Test => $Test, string => "Booting filer into required mode and Partition/Unpartition based on group selected" ); #if the filer is partitioned setting the flag 'partition_check' = 1 foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $disk_hash = disk_show_v( Node => $Node ); foreach ( keys %{$disk_hash} ) { if ( $_ =~ /P\d+$/ ) { $partition_check = 1; } } } logcomment("User Selected Group : $GROUP_NO "); if ( ( $GROUP_NO == 1 ) || ( $GROUP_NO == 2 ) || ( $GROUP_NO == 7 ) ) { if ( $partition_check == 0 ) { logcomment("Selected group is:$GROUP_NO System is un-partition, checking for system health"); my $return_val = check_system_health( Nodes => \@Nodes ); if ( $return_val->{ $Nodes[0] }->{'Drive_Assign'} ) { logcomment("Drives Assigned"); } } else { #If partition - call remove_partition and nvme_drv_assign logcomment("Selected group is:$GROUP_NO System is partitioned,Un-partitioning the system."); logcomment("Removing partitions "); remove_partition( Noderef => [@Nodes] ); logcomment("Partitioned removal complete"); nvme_drv_assign( Nodes => \@Nodes, assign => "yes" ); } #boot in Maintenance mode foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state(); logcomment("Filer : $FILER_C state is $filer_state"); if ( $filer_state =~ /MAINT/ ) { logcomment("Filer : $FILER_C is in $filer_state state,proceeding with further execution"); next; } else { logcomment("Filer: $FILER_C is not in maintainance mode. Changing State to maintenance"); nvme_boot_to_maint( Node => $Node ); } } } elsif ( ( $GROUP_NO == 10 ) || ( $GROUP_NO == 5 ) || ( $GROUP_NO == 6 ) ) { my $flag2 = 0; my @re_subtest; if ( $partition_check == 0 ) { logcomment("Selected group is:$GROUP_NO System is un-partition,checking for system health."); my @re_subtests; #check for system health and state. my $return_val = check_system_health( Nodes => \@Nodes ); if ( $return_val->{ $Nodes[0] }->{'Drive_Assign'} || $return_val->{ $Nodes[1]->name() }->{'Drive_Assign'} ) { logcomment("Drives Assigned"); } if ( $return_val->{ $Nodes[0]->name() }->{'Stale_Root_Aggr'} || $return_val->{ $Nodes[1]->name() }->{'Stale_Root_Aggr'} || $return_val->{ $Nodes[1]->name() }->{'No_root_aggr'} || $return_val->{ $Nodes[0]->name() }->{'No_root_aggr'} ) { logcomment("Root Aggr is not in proper state."); nvme_cli_unaprt(@Nodes); } else { foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state(); if ( $filer_state =~ /CLI/ ) { logcomment("Filer : $FILER_C is in $filer_state state,proceeding with further execution"); next; } else { logcomment("Filer: $FILER_C is not in CLI mode. Changing State to CLI"); $flag = 1; push( @re_subtest, subtest( \&nvme_boot_in_req_mode, -runid, "boot_to_CLI_$FILER_C", -bg, "--", Trans => $Transit_obj, boot_to => 'CLI', ) ); } } Subtest::wait_finish( subtest => [@re_subtest] ); logcomment("Filer are in CLI mode, check for cluster or system health"); my $health; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $timeout = "1900"; my $prompt_fw; my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; # Retreive valid prompts from NATE my $host = Tharn::host($FILER_C); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("PROMTS : $prompt_fw and @$prompt_array"); my $outptu = $Server->console( cmd => "set -rows 0;cluster statistics show", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "::>" => "FINAL_PROMPT", "login" => "admin", ".*Password.*" => "netapp1!", "::*>" => "FINAL_PROMPT", "\:\:\*\>" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", ".*::\*>.*" => "FINAL_PROMPT", "$prompt_fw" => "FINAL_PROMPT", ".*::>.*" => "FINAL_PROMPT", } ); if ( $outptu =~ /degraded|This table is currently empty/ ) { logcomment("$FILER_C : System health is degraded, Re-initilize the filer"); $health = "degraded"; } else { logcomment("$FILER_C : System health is OK, Proceed with next steps"); } $Server->console( cmd => "set test", additional_prompts_timeout => 3600, ignore => $ignore_strings, additional_prompts => { ".*Do you want to continue.*" => "y", "\.*\:\:\*\>\.*" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", ".*\:\:\*\>.*" => "FINAL_PROMPT", } ); } if ( $health =~ /degraded/ ) { foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); $Transit_obj->change_state( to => "FIRMWARE" ); logcomment("Filer halted"); } logcomment("Filer health is degaraded, re-initilize filer and setup cluster"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my @subtests; my $env_data = "Enter path"; push( @subtests, subtest( \&nvme_clean_config, -runid, "clean_config_node_degrad_$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) ); } Subtest::wait_finish( subtest => [@subtests] ); logcomment("Creating cluster using nodes available"); create_join_cluster( node_present => [@Nodes] ); logcomment("Completed creating and joining cluster "); } else { logcomment("System health is ok"); } } } else { logcomment("Selected group is:$GROUP_NO System is partioned, Un-partitioning the system"); nvme_unparition_cli(@Nodes); $flag = 1; } } elsif ( ( $GROUP_NO == 3 ) || ( $GROUP_NO == 8 ) || ( $GROUP_NO == 4 ) || ( $GROUP_NO == 9 ) || ( $GROUP_NO =~ /STRESS_TESTING/ ) ) { my $flag2 = 0; my @re_subtest; if ( $partition_check == 1 ) { logcomment("Selected group is: $GROUP_NO System is partition,checking for system health"); #check for system health and state. my $return_val = check_system_health( Nodes => \@Nodes ); if ( $return_val->{ $Nodes[0] }->{'Drive_Assign'} || $return_val->{ $Nodes[1]->name() }->{'Drive_Assign'} ) { logcomment("Drives Assigned"); } if ( $return_val->{ $Nodes[0]->name() }->{'Stale_Root_Aggr'} || $return_val->{ $Nodes[0]->name() }->{'Stale_Root_Aggr'} || $return_val->{ $Nodes[1]->name() }->{'No_root_aggr'} || $return_val->{ $Nodes[0]->name() }->{'No_root_aggr'} ) { logcomment("Root Aggr is not in proper state"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my @subtests; my $env_data = "Enter path"; push( @subtests, subtest( \&nvme_clean_config, -runid, "clean_config_node_$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) ); } Subtest::wait_finish( subtest => [@subtests] ); logcomment("Creating cluster using nodes available"); create_join_cluster( node_present => [@Nodes] ); logcomment("Completed creating and joining cluster "); } else { foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state(); if ( $filer_state =~ /CLI/ ) { logcomment("Filer : $FILER_C is in $filer_state state,proceeding with further execution"); next; } else { logcomment("Filer: $FILER_C is not in CLI mode. Changing State to CLI"); $flag2 = 1; push( @re_subtest, subtest( \&nvme_boot_in_req_mode, -runid, "boot_to_CLI_$FILER_C", -bg, "--", Trans => $Transit_obj, boot_to => 'CLI', ) ); } } Subtest::wait_finish( subtest => [@re_subtest] ); } } else { logcomment("Selected group is:$GROUP_NO System is Un-partioned, so partitioning the system"); check_partition_set_to_partition(@Nodes); $flag = 1; } } else { logcomment("User selection for GROUP cannot be captured, Please re-initiate ndate and select group available from the list"); } #check State , it should be in CLI if ( $flag == 1 ) { foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /CLI/ ) { logcomment("Filer : $FILER_C state is $filer_state"); } else { logcomment("Filer : $FILER_C is not in CLI mode, manually boot filer in CLI mode and restart the test "); logresult( 'FATAL', "$FILER_C is not in CLI mode, manually boot filer in CLI mode and restart the test" ); $test_status = 1; } } } #check for partition . my $partition = 0; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $disk_hash = disk_show_v( Node => $Node ); foreach ( keys %{$disk_hash} ) { if ( $_ =~ /P\d+$/ ) { $partition = 1; } } if ( ( $GROUP_NO == 1 ) || ( $GROUP_NO == 2 ) || ( $GROUP_NO == 5 ) || ( $GROUP_NO == 6 ) || ( $GROUP_NO == 7 ) || ( $GROUP_NO == 10 ) ) { if ( $partition == 0 ) { logcomment("Selected Group - $GROUP_NO requires drives to be un-partitioned, Drives are un-partioned"); } else { logcomment("Drives are partitioned"); $test_status = 1; logcomment("**FATAL** Group $GROUP_NO requires drives to be un-partitioned, Re-initilize filer with boot args as false to create partitioned drives"); logresult( 'FATAL', "Group $GROUP_NO requires drives to be un-partitioned, Re-initilize filer with boot args as false to create partitioned drives" ); } } if ( ( $GROUP_NO == 3 ) || ( $GROUP_NO == 4 ) || ( $GROUP_NO == 8 ) || ( $GROUP_NO == 9 ) || ( $GROUP_NO =~ /STRESS_TESTING/ ) ) { if ( $partition == 1 ) { logcomment("Selected Group - $GROUP_NO requires drives to be partitioned, Drives are partioned"); } else { logcomment("Drives are un-partitioned"); $test_status = 1; logcomment("**FATAL** : Group $GROUP_NO requires drives to be partitioned, Re-initilize filer with boot args as true to create partitioned drives"); logresult( 'FATAL', "Group $GROUP_NO requires drives to be partitioned, Re-initilize filer with boot args as true to create partitioned drives" ); } } last; } } else { logcomment("Check filer setup will NOT be executed, As user had selected -b =7"); logcomment("**WARNING** : Check filer setup will NOT be executed, As user had selected -b =7"); } if ( $test_status == 0 ) { return $TCD::PASS; } else { return $TCD::FAIL; } } #------------------------------------------------------------ ## Check and create cluster - normal check filer setup #------------------------------------------------------------ sub Cluster { if ( $CHECK_FILER_SETUP eq 'Y' ) { if ( $BOOT_TYPE =~ /^N$/ ) { logcomment("Check_filer_setup WILL NOT PERFORM TO CHECK THE CLUSTER AS -b=6 IS SELECTED"); } #------------------------------------------------------------ ## CLUSTER_SETUP : Step 1 - Booting filer in Maintenance as -b=5 is selected #------------------------------------------------------------ if ( ( ( $Filer_mode eq 'M' ) || ( $BOOT_MODE == '5' ) ) && ( $BOOT_MODE != '4d' ) ) { eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 1 : Booting filer in Maintenance as -b=5 is selected", ); if ( $BOOT_MODE == '5' ) { logcomment("Check_filer_setup will boot the filer in Maintenance mode as -b=5 is selected"); } $Test->description("Check_filer_setup : Find all nodes in the Maintenance"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Original_State = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILER_C is in $Original_State\n"); if ( $Original_State !~ /MAINT/ ) { $f_state = 1; push( @FILER_MAINT, $FILER_C ); } } if ( $f_state == 1 ) { foreach my $filer (@FILER_MAINT) { push( @subtests, subtest( \&boot_it_good_Maint, -runid, "boot_it_good_Maint_" . "$filer", -bg, "--", Test => $Test, Filer => $filer ) ); } Subtest::wait_finish( subtest => [@subtests] ); } #------------------------------------------------------------ ## CLUSTER_SETUP : Step 1a - Check disk owner #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 1a : Check disk owner", ); # remove_partition(Noderef => [@Nodes]); check_disk_owner( Noderef => [@Nodes] ); } ## end if ( $Filer_mode eq 'M' ) #------------------------------------------------------------ ## CLUSTER_SETUP : Step 1 - Booting filer in CLI as -b=1 is selected #------------------------------------------------------------ elsif ( ( ( $Filer_mode eq 'F' ) || ( $BOOT_MODE == '1' ) ) && ( $BOOT_MODE != '4d' ) ) { eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 1 : Booting filer in CLI as -b=1 is selected", ); if ( $BOOT_MODE == '1' ) { logcomment("Check_filer_setup will boot the filer in CLI mode as -b=1 is selected"); } $Test->description("Check_filer_setup : Find all nodes in the Cluster"); #------------------------------------------------------------ ## CLUSTER_SETUP : Step 1a - Checking for valid node names #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 1a : Checking for valid node names", ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); try { $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $Original_State_temp = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILER_C is in $Original_State_temp"); if ( $Original_State_temp =~ /USERNAME/ ) { $Transit_obj->change_state( to => "CLI" ); logcomment("$FILER_C Change to CLI"); } my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); eval { $Server->console( cmd => "system node rename -node local -newname $FILER_C" ); }; if ($@) { if ( $@ =~ /Error/i ) { logcomment("Valid node name exist"); } else { logcomment("VALID NODE NAME DOES NOT EXISTS, CHANGED NODE NAME TO $FILER_C"); } } } catch NACL::Transit::Exceptions::Timeout with { my $timeout = "1900"; my $prompt_fw; my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; # Retreive valid prompts from NATE my $host = Tharn::host($FILER_C); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; $Server->console( cmd => "node rename -newname $FILER_C", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "::>" => "FINAL_PROMPT", "login" => "admin", ".*Password.*" => "netapp1!", } ); logcomment("Changed node name"); }; } #------------------------------------------------------------ ## CLUSTER_SETUP : Step 1b - Checking for state of filers and booting it to required states #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 1b : Checking for state of filers and booting it to required states", ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); $Transit_obj = NACL::Transit->new( name => $FILER_C ); $Original_State = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILER_C is in $Original_State"); if ( ( $Original_State =~ /USERNAME|CLI|PASS/ ) || ( $Original_State !~ /CLI/ ) ) { if ( $Original_State =~ /USERNAME|CLI|PASS/i ) { logcomment("$FILER_C Change to CLI") if ( $Original_State !~ /CLI/ ); $Transit_obj->change_state( to => "CLI" ); logcomment("Filer $FILER_C is in $Original_State"); handle_run_local_error( Node => $Node ); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*continue?" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); sleep 2; my $aggr_output = $Api_Set_Obj->execute_command( command => "aggr status -r" ); if ( $aggr_output =~ /uninit/ ) { logcomment("$FILER_C :: Filer is booted in 24/7, So re-initializing the filer to create new file-system"); $Cluster = 1; } } elsif ( ( $Original_State =~ /FIRMWARE/ ) || ( $Original_State !~ /CLI/ ) ) { $Transit_obj->change_state( to => "MAINT" ); $Original_State = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILER_C is in $Original_State"); my $host = host($FILER_C); my $Api_Set_Obj = NACL::APISet->new( hostobj => $host, category => "Node", interface => "CLI", set => "Maintenance" ); my $filer_disks = disk_show_drive_maint( API_Obj => $Api_Set_Obj, Filer => $FILER_C ); my @filer_diskA = @{ $filer_disks->{'driveA'} }; my @filer_diskB = @{ $filer_disks->{'driveB'} }; my $diskA = @filer_diskA; my $diskB = @filer_diskB; if ( ( $diskA == 0 ) || ( $diskB == 0 ) ) { $Cluster = 1; } try { my $aggr_output = $Api_Set_Obj->execute_raw_command( command => "aggr status -r" ); if ( ( $aggr_output =~ /aggr0/ ) && ( $aggr_output !~ /No root aggregate/ ) ) { logcomment("Root aggregate exist on Filer $FILER_C"); if ( $aggr_output =~ /\(MIRROR\)/ ) { logcomment("**WARNING** : Mirrored aggregates found in the filer : $FILER_C"); # $Cluster = 1; } if ( $aggr_output =~ /uninit/ ) { logcomment("$FILER_C :: Filer is booted in 24/7, creating the new file-system"); $Cluster = 1; } if ( $aggr_output =~ /restricted/ ) { logcomment("**WARNING** : Restricted aggregates found in the filer : $FILER_C"); $Cluster = 1; } } else { logcomment("ROOT AGGREGATE IS NOT PRESENT ON FILER - $FILER_C"); $Cluster = 1; } } catch NACL::APISet::Exceptions::CouldNotResyncConnectionException with { logcomment("Expected: Caught a CouldNotResyncConnection Exception"); try { my $prompts_answers = [ ".*Aggregate set.*" => '1' ]; my $aggr_output = $Api_Set_Obj->execute_raw_command( 'command' => "aggr status -r", 'connectrec-match_table' => $prompts_answers ); if ( ( $aggr_output =~ /aggr0/ ) && ( $aggr_output !~ /No root aggregate/ ) ) { logcomment("Root aggregate exist on Filer $FILER_C"); if ( $aggr_output =~ /\(MIRROR\)/ ) { logcomment("**WARNING** : Mirrored aggregates found in the filer : $FILER_C"); # $Cluster = 1; } if ( $aggr_output =~ /uninit/ ) { logcomment("$FILER_C :: Filer is booted in 24/7, creating the new file-system"); $Cluster = 1; } if ( $aggr_output =~ /restricted/ ) { logcomment("**WARNING** : Restricted aggregates found in the filer : $FILER_C"); $Cluster = 1; } } else { logcomment("ROOT AGGREGATE IS NOT PRESENT ON FILER - $FILER_C"); $Cluster = 1; } } catch NATE::BaseException with { logcomment("ROOT AGGREGATE IS NOT PRESENT ON FILER - $FILER_C"); $Cluster = 1; }; }; } } } ####Change the state of filer to CLI if ( $Cluster == 0 ) { { my $current; my $FILER_C; my $partner; my %nodes_filer; my @temp_nodes = NACL::C::Node->find(); # Find Nodes/Filers used in the test, Based on FILER param. foreach my $Node (@temp_nodes) { $FILER_C = $Node->name(); $nodes_filer{$FILER_C} = $Node; } @temp_nodes = values(%nodes_filer); foreach my $Node (@temp_nodes) { my $FILER = $Node->name(); if ( $FILER =~ /^$FILER_C$/ ) { $current = $Node; } else { $partner = $Node; } } giveback_takeover( Node => $current, Partner => $partner ); } foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&boot_filer_in_req_state, -runid, "boot_to_CLI_" . "$FILER_C", -bg, "--", filer_name => $FILER_C, Test => $Test, change_state_to => "CLI" ) ); } Subtest::wait_finish( subtest => [@subtests] ); } nodename_interchange_checkup(); logcomment("Node Names Interchange checkup completed successfully"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $Original_State = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILER_C is in $Original_State"); if ( ( $Original_State =~ /CLI/ ) && ( $Cluster == 0 ) ) { my $timeout = "1900"; my $prompt_fw; my $Server = new Server( no_cli => 1 ); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; # Retreive valid prompts from NATE my $host = Tharn::host($FILER_C); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; $Server->set_execution_server( -Server => $FILER_C ); eval { $Server->console( cmd => "cluster show", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "login" => "admin", ".*Password.*" => "netapp1!", ".*Error:.*" => "FATAL_PROMPT", ".*is not a recognized command.*" => "FATAL_PROMPT", "\:\:\>" => "FATAL_PROMPT", "\:\:\.*\>" => "FATAL_PROMPT" } ); }; if ($@) { logcomment("Inside Eval"); if ( $@ =~ /Error/ ) { $Cluster = 1; logcomment("CLUSTER IS NOT ENABLED ON $FILER_C. NOW ,CLUSTER IS GOING TO CREATE"); } } eval { $Server->console( cmd => "set test", additional_prompts_timeout => 3600, additional_prompts => { ".*Do you want to continue.*" => "y", ignore => $ignore_strings, "\:\:\.*\>" => "FATAL_PROMPT", "\:\:\>" => "FATAL_PROMPT", ".*Error:.*" => "FATAL_PROMPT", ".*is not a recognized command.*" => "FATAL_PROMPT", "$prompt_fw" => "FINAL_PROMPT" } ); }; if ($@) { logcomment("Inside Eval"); } my $net_int_out = $Server->console( cmd => "net int show", additional_prompts_timeout => 3600 ); if ( $net_int_out !~ /mgmt1/i ) { logcomment("MANAGEMENT PORT IS NOT CONFIGURED, CLUSTER WILL BE RE-CREATED AGAIN"); $Cluster = 1; last; } } } #------------------------------------------------------------ ## CLUSTER_SETUP : Step 1c - Checking for root aggregates #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 1c : Checking if root aggregates are present", ); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $Original_State = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILER_C is in $Original_State"); if ( ( $Original_State =~ /CLI/ ) && ( $Cluster != 1 ) ) { my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); ##TEST 1: my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( command => "system timeout modify -timeout 0 ; rows 0" ); logcomment("Checking for root aggregate"); my $aggr_output = $Api_Set_Obj->execute_raw_command( command => "aggr show -fields Aggregate" ); if ( ( $aggr_output !~ /aggr0/ ) || ( $aggr_output =~ /Error/ ) ) { $Cluster = 1; logcomment("aggr_output = $aggr_output"); } else { ##TEST 1a: my $aggr_status = $Api_Set_Obj->execute_raw_command( command => "aggr show" ); if ( ( $aggr_status =~ /offline/ ) || ( $aggr_status !~ /online/ ) ) { $Cluster = 1; logcomment("aggr_status = $aggr_status"); } } ##TEST 2: logcomment("Checking for root aggregate and any failed drives"); my $vol_output = $Api_Set_Obj->execute_raw_command( command => "vol show -fields Aggregate" ); if ( ( $vol_output !~ /vol0/ ) || ( $vol_output =~ /Error/ ) ) { $Cluster = 1; logcomment("vol_output = $vol_output"); } else { ##TEST 2a: logcomment("Checking for root volume status"); my $vol_status = $Api_Set_Obj->execute_raw_command( command => "vol show" ); if ( ( $vol_status =~ /offline/ ) || ( $vol_status !~ /online/ ) ) { $Cluster = 1; logcomment("vol_status = $vol_status"); } } ##TEST 3: my $output = $Api_Set_Obj->execute_raw_command( command => "cluster show -fields health -node $FILER_C" ); my $output_net = $Api_Set_Obj->execute_raw_command( command => "net int show" ); if ( ( $output =~ /$FILER_C/ ) && ( $output_net =~ /$FILER_C/ ) ) { if ( ( $output =~ /true/ ) && ( $output !~ /false/ ) ) { logcomment("Cluster is already enable on $FILER_C."); } else { logcomment("\nCLUSTER IS NOT ENABLE ON $FILER_C."); $Cluster = 1; } } } if ( $Cluster == 1 ) { last; } } ## Checking whether SFO state for FILERA and FILERB if ( $Cluster == 0 ) { foreach my $Node (@Nodes) { $Node->refresh_command_interface(); try { my @resp = NACL::CS::Cluster->fetch( command_interface => $Node ); my $length = @resp; if ( $length < 2 ) { $Cluster = 1; logcomment('**WARNING** : FILERS ARE NOT IN CLUSTER'); last; } } catch NACL::STask::Exceptions::UnexpectedSFOState with { logcomment('**WARNING** :: NO ELEMENTS FOUND IN CLUSTER SHOW COMMAND'); $Cluster = 1; } catch NACL::Exceptions::NoElementsFound with { logcomment('**WARNING** : NO ELEMENTS FOUND IN CLUSTER SHOW COMMAND'); $Cluster = 1; }; } # try{ # NACL::STask::StorageFailover->check_state( # command_interface => $Nodes[0], # node => $Nodes[0], # partner => $Nodes[1], # node_state => "up", # partner_state => "up", # ); # }catch NACL::STask::Exceptions::UnexpectedSFOState with{ # logcomment('**FATAL** : filers are not in cluster'); # logresult('FATAL',"filers are not in cluster"); # $Cluster=1; # }; # try{ # NACL::STask::StorageFailover->check_state( # command_interface => $Nodes[1], # node => $Nodes[1], # partner => $Nodes[0], # node_state => "up", # partner_state => "up", # ); # }catch NACL::STask::Exceptions::UnexpectedSFOState with{ # logcomment('**FATAL** : filers are not in cluster'); # logresult('FATAL',"filers are not in cluster"); # $Cluster=1; # }; } logcomment("Cluster = $Cluster"); if ( $Cluster == 1 ) { ##For booting the filer with option 4 and creating the root aggregate, root volume and cluster. setup_cmode_filers( node_present => \@Nodes, Test => $Test, ); logcomment("Booting completed"); } foreach my $Node (@Nodes) { $Node->refresh_command_interface(); $time_out = get_max_timeout( node_present => \@Nodes, Test => $Test, ); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); # $Api_Set_Obj->execute_raw_command( command => "system timeout modify -timeout $time_out" ); } } elsif ( $BOOT_MODE == '4d' ) { logcomment("Check_filer_setup will boot the filer and create the cluster as -b=4d is selected"); setup_cmode_filers( node_present => \@Nodes, Test => $Test, ); logcomment("CMode setup completed"); foreach my $Node (@Nodes) { handle_run_local_error( Node => $Node ); $time_out = get_max_timeout( node_present => \@Nodes, Test => $Test, ); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( command => "system timeout modify -timeout 0 ; rows 0" ); } } ## end elsif ###Check for disk attached handle_run_local_error(); { ##Create new nodes, if there is any run local error occurred then nodes must be rebooted/new node objects my ( %nodes_filer, $Nodes ); my @Nodes_temp = NACL::C::Node->find(); foreach my $Node (@Nodes_temp) { my $FILER_C = $Node->name(); $nodes_filer{$FILER_C} = $Node; } @Nodes = values(%nodes_filer); } #------------------------------------------------------------ ## CLUSTER_SETUP : Step 2 - Checking for available disks and list any failed drives if present #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 2 : Checking for available disks and list any failed drives if present", ); my @disk_list = list_of_disk( Node => $Nodes[0] ); logcomment( "List of disk available are " . scalar @disk_list . " and the drives are : @disk_list" ); if ( ( $GROUP_NO == 1 ) || ( $GROUP_NO == 2 ) || ( $GROUP_NO == 7 ) ) { #Burt 1041403 nmamatha logcomment("DEBUG : As the group selected by user is $GROUP_NO which is for MAINTENANCE mode scripts,NOT checking for config state "); } else { logcomment("DEBUG: As the group selected by user is $GROUP_NO which is for CLI mode scripts, checking for config state "); get_config_status( Nodes => [@Nodes] ); } my $state = fail_if_diff_drive_foramt( Node => $Nodes[0] ); if ( $state =~ /fail/i ) { logresult( 'FATAL', "Please remove the unsupported drives from the system and try again" ); } { my @subtests; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&report_failed_drives, -runid, "report_failed_drives" . "$FILER_C", -bg, "--", Node => $Node, ) ); } logcomment("Subtest @subtests"); Subtest::wait_finish( subtest => [@subtests] ); } #------------------------------------------------------------ ## CLUSTER_SETUP : Step 3 - Check for aggregate/volume status if they are online/offline #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 3 : Check for aggregate/volume status if they are online/offline", ); grep { check_aggr_vol_offline( Node => $_ ) } @Nodes; #------------------------------------------------------------ ## CLUSTER_SETUP : Step 4 - Check for failed drives and unfail them #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 4 : Check for failed drives and unfail them", ); unfail_disk_zero_spares( Nodes => [@Nodes] ); logcomment("Checking for any unowned disks before execution of test"); disk_remv_assign_ownership( Node_ref => [@Nodes] ); #------------------------------------------------------------ ## CLUSTER_SETUP : Step 5 - Checking for filer version #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 5 : Checking for filer version", ); foreach my $Node (@Nodes) { my $node_version; my $FILER_C = $Node->name(); if ( $Filer_mode eq 'F' ) { my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( 'command' => "rows 0" ); $Api_Set_Obj->execute_raw_command( 'command' => "system timeout modify -timeout 0" ); $node_version = $Api_Set_Obj->execute_raw_command( 'command' => "version" ); } else { my $host = host($FILER_C); my $Api_Set_Obj = NACL::APISet->new( hostobj => $host, category => "Node", interface => "CLI", set => "Maintenance" ); $node_version = $Api_Set_Obj->execute_raw_command( command => "version" ); } logcomment("Node Version : $node_version"); if ( $node_version ne '' ) { logcomment("FILER - $FILER_C version :$node_version"); } else { logcomment("$FILER_C : NODE VERSION NOT FOUND ON THE FILER"); # $test_status = 1; # $email_subject = "$TC_name - $FILER_C : Test FAILED: FAIL"; # $email_body = "Version command failed for filer $FILER_C.\nLog Location : $LOGDIR\n"; # send_message( # mail_subject => $email_subject, # mail_body => $email_body, # MAIL_FROM => $MAIL_FROM, # MAIL_TO => $MAIL_TO, # EOE => $EOE, # EMAIL => $EMAIL # ); # logresult( 'FAIL', "Version command failed for filer $FILER_C" ); } } #------------------------------------------------------------ ## CLUSTER_SETUP : Step 6 - Disable background/foreground firmware and stop hammer #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 6 : Disable background/foreground firmware and stop hammer", ); disable_fg_bg_firmware_hammer( Nodes => [@Nodes] ); #------------------------------------------------------------ ## CLUSTER_SETUP : Step 7 - Check for the disks and reconstruct or copy sick disk data if present #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 7 : Check for the disks and reconstruct or copy sick disk data if present", ); filer_disk_create(); #------------------------------------------------------------ ## CLUSTER_SETUP : Step 8 - Record power cycle count and phy change count for the filer #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 8 : Record power cycle count and phy change count for the filer", ); foreach (@Nodes) { get_phy_state( Node => $_ ); } if ( $KERNEL_TYPE !~ /NON_DEBUG/ ) { my $FILER_MINI = {}; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("$FILER_C - Check for mini-disks"); my $Mode = $Node->mode(); my $Host = host($FILER_C); my $transit_obj = NACL::Transit->new( name => $Host ); my $filer_state = $transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); my ( @mini_disks, $Api_Set_Obj ); if ( $filer_state =~ /UP|CLI/i ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); } elsif ( $filer_state =~ /MAINT/i ) { my $console = Tharn::connect("$FILER_C#console"); # Set the properties for the Connectrec used for creating the APIset $console->{prompts} = [qr/\*>/]; $console->{max_idle} = 600; $console->{on_idle} = sub { $console->execute(""); }; $Api_Set_Obj = NACL::APISet->new( connection => $console, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); } my $res; try { $res = $Api_Set_Obj->mini_disk_list(); } catch NACL::APISet::Exceptions::FailedAndNotFoundException with { logcomment("FIler : $FILER_C : mini_disk LIST COMMAND IS NOT WORKING ON THIS PARTICULAR KERNEL"); }; if ( !$res ) { logcomment("FIler : $FILER_C : mini_disk LIST COMMAND IS NOT WORKING ON THIS PARTICULAR KERNEL"); # return undef; } my @disks_obj = @{ $res->get_parsed_output() }; foreach (@disks_obj) { if ( $_->{type} ne "NORMAL disk" ) { push( @mini_disks, $_->{disk} ); $FILER_MINI->{$FILER_C} = [@mini_disks] if ( scalar @mini_disks ); } } logcomment("$FILER_C - End of checking for mini-disks"); } foreach ( keys %{$FILER_MINI} ) { logcomment( "Filer : Number of Mini_disks present on the filer $_ are " . scalar @{ $FILER_MINI->{$_} } . " and the drives are @{$FILER_MINI->{$_}}" ); my $email_subject = "$_ : Mini_disks present on the filer"; my $email_body = "Number of Mini_disks present on the filer $_ are " . scalar @{ $FILER_MINI->{$_} } . " and the drives are @{$FILER_MINI->{$_}}.\nLog Location : $LOGDIR\n"; send_message( mail_subject => $email_subject, mail_body => $email_body, MAIL_FROM => $MAIL_FROM, MAIL_TO => $MAIL_TO, EOE => $EOE, EMAIL => $EMAIL ); } } #------------------------------------------------------------ ## CLUSTER_SETUP : Step 9 - Check if high availability feature is configured #------------------------------------------------------------ eye_catcher( Test => $Test, string => "CLUSTER_SETUP : STEP 9 : Checking if HA is configured", ); grep { check_path_ha( Node => $_ ) } @Nodes; #********************************************************* #checking for spare drives enough to run Group 3 4 5 scripts #*********************************************************** foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); if ( ( $GROUP_NO == 3 ) || ( $GROUP_NO == 4 ) || ( $GROUP_NO == 5 ) || ( $GROUP_NO == 6 ) ) { logcomment("$FILER_C : Selected Group is $GROUP_NO, which requires minimum one non root aggregate"); logcomment("Checking spare drives "); my @Spare_disk = spare_count_cmode( Node => $Node, run_local => "yes" ); logcomment( "DEBUG: $FILER_C:Total spare drives are : " . scalar @Spare_disk . " and the spare drives are :: @Spare_disk" ) if (@Spare_disk); my (%part_spares); my ( $num_spares, $kernel ); my @aggregates, my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); my $API_obj; my $check_flag = 0; if ( $filer_state =~ /MAINT/gi ) { my $Host = host($FILER_C); my @aggr_pres; $API_obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $API_obj->set_timeout( "connectrec-timeout" => 1200 ); $kernel = $API_obj->execute_raw_command( 'command' => "version" ); logcomment("Checking for aggregate"); my $prompts_answers = [ ".*Aggregate set.*" => '1' ]; my $output = $API_obj->execute_raw_command( 'command' => "aggr status", 'connectrec-match_table' => $prompts_answers ); if ( $output =~ /No root aggregate|Please supply a number/ig ) { logcomment("NO ROOT VOLUME EXIST ON THE FILER $FILER_C"); next; } else { foreach my $var ( split( /\n/, $output ) ) { if ( $var =~ /aggr\s+/ ) { if ( $var =~ /(\S+)\s+/ ) { my $ag = $1; if ( $ag =~ /aggr/ ) { logcomment("Aggregate found - $ag"); push( @aggr_pres, $ag ); } } } } } if (@aggr_pres) { logcomment("Aggregte present - @aggr_pres"); } else { logcomment("NO NON-ROOT AGGREGATE IS PRESENT, CHECKING FOR SPARES"); $check_flag = 1; } } elsif ( $filer_state =~ /(CLI|UP)/gi ) { $API_obj = $Node->get_7m_or_nodescope_apiset(); $API_obj->set_timeout( "connectrec-timeout" => 1200 ); $kernel = $API_obj->execute_raw_command( 'command' => "run local version" ); try { my @tot_aggr = NACL::CS::StorageAggregate->fetch( command_interface => $Node, filter => { node => $FILER_C, 'root' => 'false' } ); foreach my $aggr (@tot_aggr) { push( @aggregates, $aggr->aggregate ); } logcomment("$FILER_C :: Non Root aggregates are - @aggregates"); } catch NACL::Exceptions::NoElementsFound with { logcomment("$FILER_C : NON ROOT AGGREGATES ARE NOT FOUND IN THE FILER"); $check_flag = 1; }; } if ( $check_flag == 1 ) { my $num_phy_drives; my @numbr_drives = (); my @total_no_drives = (); if ( ( $FIPS_SELECTED =~ /SSD$/i ) && ( $filer_state =~ /(CLI|UP)/i ) ) { @numbr_drives = (); @total_no_drives = (); my $partition_enabled = 0; my $disk_show_v = $API_obj->execute_command( 'command' => "disk show -v" ); my @lines = split( /\n/, $disk_show_v ); my $n = 0; while ( $n <= $#lines ) { last if ( $lines[$n] =~ /^-----/ ); $n++; } $n++; foreach my $line ( @lines[ $n .. $#lines ] ) { if ( $line =~ /(\S+)\s+\S+\s*\(\d+/ ) { my $disk = $1; push( @numbr_drives, $disk ); } } grep { if ( $_ !~ /P\d+/i ) { push( @total_no_drives, $_ ) } } @numbr_drives; foreach (@numbr_drives) { if ( $_ =~ /P\d+$/ ) { $partition_enabled = 1; logcomment("DEBUG : Setup is PARTITIONED"); last; } } my $part_string; if ( $partition_enabled == 1 ) { $part_string = "PARTITIONED"; $num_phy_drives = 8; } else { $part_string = "UNPARTITIONED"; $num_phy_drives = 14; } logcomment("$FILER_C : DRIVE_TYPE : $FIPS_SELECTED | $part_string | Minimum physical drives required : $num_phy_drives"); } else { if ( $kernel !~ /<1d>/ ) { #logcomment("Non debug kernel will require at least 5 disks for raid_dp"); $num_spares = 5; } else { $num_spares = 3; } logcomment("DEBUG : $FILER_C : Minimum number of spare drives required to create aggregate are $num_spares. "); } #$num_spares = $num_spares + 2; #commented by aditis my @normal_spares; foreach (@Spare_disk) { if ( $_ =~ /(P\d+)$/ ) { push( @{ $part_spares{$1} }, $_ ); } else { push( @normal_spares, $_ ); } } my @partitions = keys %part_spares; my $flag = 0; my $spare = @normal_spares; my $phydrive_cnt = @total_no_drives; if ( ( $FIPS_SELECTED =~ /SSD/i ) && ( $filer_state =~ /(CLI|UP)/i ) ) { if (@total_no_drives) { logcomment( "$FILER_C : Total number of physical disks present : " . scalar @total_no_drives ); if ( $phydrive_cnt < $num_phy_drives ) { $flag = 1; } else { $flag = 0; } } } else { if (@normal_spares) { if ( $spare < $num_spares ) { $flag = 1; } else { $flag = 0; logcomment("$FILER_C: $spare Spare drives are present. "); } } foreach (@partitions) { logcomment( "$FILER_C : Number of drives present under the partition $_ : are " . scalar @{ $part_spares{$_} } . " and the drives are @{$part_spares{$_}}" ); if ( @{ $part_spares{$_} } >= ($num_spares) ) { $flag = 0; logcomment( "$FILER_C:" . scalar @{ $part_spares{$_} } . " Spare drives are present. " ); last; } else { $flag = 1; } } } if ( $flag == 1 ) { $test_status = 1; if ( ( $FIPS_SELECTED =~ /SSD/i ) && ( $filer_state =~ /(CLI|UP)/i ) ) { logcomment( "**FATAL** : $FILER_C : Total number of physical drives present : " . scalar @total_no_drives . " which is less than the required count $num_phy_drives" ); logresult( 'FATAL', "$FILER_C : Total number of physical drives present : " . scalar @total_no_drives . " which is less than the required count $num_phy_drives" ); } else { logcomment("**FATAL** : $FILER_C : SPARE DRIVES PRESENT IS LESS TO EXECUTE GROUP $GROUP_NO SCRIPTS"); logresult( 'FATAL', "$FILER_C : Spare drives present is less to execute GROUP $GROUP_NO scripts" ); } } } } } logcomment("Cluster setup and health check up completed, Checking for user selected FIPS option"); if ( $FIPS_SELECTED =~ /FIPS/i ) { foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); logcomment("Check $FILER_C is SED supported"); my $out = $Api_Set_Obj->execute_raw_command( 'command' => "storage encryption disk show-status" ); foreach my $line ( split /\n/, $out ) { if ( $line =~ /false/ ) { logcomment("FILER $FILER_C is not SED supported"); logcomment(" **FATAL** : FILER $FILER_C is not SED supported"); $test_status = 1; logresult( 'FATAL', "FILER $FILER_C is not SED supported" ); } } } my $retn_chk = 0; logcomment("Enabling and Setting On-Board Key Manager on @Nodes"); $retn_chk = setup_okm( NODES => \@Nodes ); if ( $retn_chk == 1 ) { logcomment("**FAIL** : OKM is not configured"); $test_status = 1; } else { logcomment("OKM is configured"); } } else { logcomment("User selected non-FIPS drive mode for test execution"); } #Check for FIPS and setup okm if ( $test_status == 0 ) { return $TCD::PASS; } else { return $TCD::FAIL; #kill_process(); } } return $TCD::PASS; } sub cleanup() { $Test->description("Cleanup"); # $/=undef; # logcomment("Opening file $main_log_file"); # open(FH,"$main_log_file")||kill_process(); # my $full_data = ; # close FH; # $/="\n"; # if($full_data=~/FATAL ERROR on|FAIL: Testcase Complete:(.+)failed/s){ # logcomment("Error occured in check_filer_setup, Exiting from the Test-suite"); # kill_process(); # } return $TCD::PASS; } sub kill_process(;$) { my $error = shift; my ($actual_log_dir); if ( $main_log_file =~ /(\S+)HDD/ ) { $actual_log_dir = $1; } else { $actual_log_dir = "$main_log_file ../../../"; } logcomment("Copying log file $main_log_file to location $actual_log_dir"); system("sudo cp -rf $main_log_file $actual_log_dir"); my $process_ID = $$; $email_subject = "FATAL FAIL : $TEST_SUITE_NAME - is stopped"; $email_body = "################################################################################################# ## ## Error occured in filer-setup ## ## LOGDIR :: $LOGDIR ## ## Exiting from the testsuite - $TEST_SUITE_NAME ## #################################################################################################"; if ( defined $error ) { $email_body = $email_body . "\n################################################################################################# ##Cause for failure :: $error ## ################################################################################################# "; } send_message( mail_subject => $email_subject, mail_body => $email_body, MAIL_FROM => $MAIL_FROM, MAIL_TO => $MAIL_TO, EOE => $EOE, EMAIL => $EMAIL ); logcomment("Process-ID is :: $$"); my $process = `ps -ef|grep $$`; foreach ( split( /\n/, $process ) ) { if ( $_ =~ /NDATE/ ) { my ($PPID) = $_ =~ /^\S+\s*\S+\s*(\S+)/; logcomment("Parent process-id is $PPID, Killing the parent Process-ID"); system("sudo kill -9 $PPID"); sleep 60; } } } sub check_install_kernel_type { my $Node = shift; my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Api_Set_Obj; my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); my $filer_orgin_state = $filer_state; my $secondary_var; my $recheck_flag = 0; my ( $version, $filer_version ); if ( $filer_state =~ /CLI|USER|pass/i ) { $Transit_obj->change_state( to => "CLI" ); $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( command => "system timeout modify -timeout 0 ; rows 0" ); $version = $Api_Set_Obj->execute_raw_command( 'command' => "version" ); sleep 2; $filer_version = $Api_Set_Obj->execute_command( 'command' => "version" ); } else { if ( $filer_state !~ /MAINT/ ) { $Transit_obj->change_state( to => "MAINT" ); } $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $version = $Api_Set_Obj->execute_raw_command( command => "version" ); $filer_version = $version; } logcomment("Version of filer $FILER_C is :: $version"); if ( $version !~ /Longboard|8_4|Fullste|R8|dev|lb|Kalyaniblack/i ) { logcomment("**WARNING** : KERNEL VERSION IS NEITHER Kalyaniblack OR LONGBOARD"); # logresult('FATAL',"Kernel version is not Fullsteam, so exiting from the test suite"); #kill_process("$err"); } if ( $filer_version =~ /7Mode/i ) { logcomment("**FATAL** : Filer is in 7Mode, please install CMode kernel and re-run the test"); logresult( 'FATAL', "Filer is in 7Mode, please install CMode kernel and re-run the test" ); } if ( defined $GROUP_NO ) { if ( $filer_version =~ /<1d>/i ) { logcomment("Kernel type - DEBUG kernel"); $K_type_filer = 'DEBUG'; } else { logcomment("Kernel type - NON-DEBUG kernel"); $K_type_filer = 'NON_DEBUG'; } logcomment("Kernel type choosed for the test is :: $KERNEL_TYPE"); if ( $KERNEL_TYPE =~ /^$K_type_filer$/ ) { logcomment("KERNEL types are matching from the filer and the test-suite"); } else { logcomment("KERNEL TYPE ARE NOT MATCHING WITH KERNEL INSTALLED ON FILER"); logcomment("User selected Group is $GROUP_NO with $KERNEL_TYPE type"); logcomment("Kernel type on the selected filer is $K_type_filer"); logcomment("NDATE will install required kernel type from default location"); sleep 2; logcomment("Required image for execution of selected test is $KERNEL_TYPE"); my $switch = 0; if ( $switch == 0 ) { $secondary_var = switch_img_n_check_ver( $Node, $KERNEL_TYPE ); logcomment("DEBUG : Returned value - $secondary_var "); if ( $secondary_var == 1 ) { logcomment("**FATAL** :$FILER_C : Installed Kernel type is still $K_type_filer kernel, so exiting test execution"); return logresult( "INFO", msg => 1 ); } } else { my $install_image; if ( $KERNEL_TYPE =~ /NON_DEBUG/ ) { open( FH, "$Home/NDATE/FILER_INFO/NON_DEBUG" ); my $image_file = ; close FH; chomp($image_file); $install_image = $image_file; } else { open( FH, "$Home/NDATE/FILER_INFO/DEBUG" ); my $image_file = ; close FH; chomp($image_file); $install_image = $image_file; } logcomment("Installing - $install_image"); my @subtests; foreach (@Nodes) { my $FILER = $_->name(); logcomment("Filer : $FILER installing new Image on the filer"); push( @subtests, subtest( \&compact_flash, -runid, "Install_Image_" . "$FILER", -bg, "--", Node => $_, tar_file => $install_image, ) ); } logcomment("Subtest @subtests"); Subtest::wait_finish( subtest => [@subtests] ); sleep 5; } #End of switch if condition recheck: logcomment("Check state of filer after switching/installing image"); logcomment("DEBUG: Original state filer was - $filer_orgin_state"); logcomment("DEBUG: Changing state to $filer_orgin_state"); try { $Transit_obj->change_state( to => $filer_orgin_state ); } catch NACL::Transit::Exceptions::RetriesExceeded with { logcomment("EXPECTED: Caught a RetriesExceeded exception"); } catch NACL::Transit::Exceptions::TransitException with { logcomment("COULD NOT TRANSIT TO $filer_orgin_state STATE"); }; my $filer_state_2 = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer state after switching or installing image is - $filer_state_2"); if ( $filer_state_2 =~ /CLI|USER|pass/i ) { $Transit_obj->change_state( to => "CLI" ); $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); } elsif ( $filer_state_2 =~ /MAINT/ ) { $Transit_obj->change_state( to => "MAINT" ); $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); } else { logcomment("FILERS IS IN UNKNOWN STATE - $filer_state_2"); } logcomment("Verifying Installed image"); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $Api_Set_Obj->execute_raw_command( command => "system timeout modify -timeout 0 ; rows 0" ); my $version = $Api_Set_Obj->execute_raw_command( 'command' => "version" ); sleep 2; my $filer_version = $Api_Set_Obj->execute_command( 'command' => "version" ); if ( $KERNEL_TYPE =~ /NON_DEBUG/ ) { if ( $filer_version !~ /<1d>/i ) { logcomment("Installed Kernel is NON_DEBUG, Proceeding with next steps"); } else { logcomment("Installed Kernel is DEBUG, Reloading and Setting SECONDARY VARIABLE again"); my $ret = re_set_n_check_image( $Node, $secondary_var ); logcomment("DEBUG: ret : $ret"); $recheck_flag++ if ( $ret == 1 ); logcomment("DEBUG: Recheck kernel version - $recheck_flag"); if ( $recheck_flag < 3 && $ret ne 0 ) { goto recheck; } else { if ( $recheck_flag == 3 ) { logcomment("**FATAL** :$FILER_C : Installed Kernel type is still Debug kernel after 3 retries, so exiting test execution"); return logresult( "INFO", msg => 1 ); } else { logcomment("**FATAL** :$FILER_C: Installed Kernel type is still Debug kernel, exiting test execution"); return logresult( "INFO", msg => 1 ); } } } } elsif ( $KERNEL_TYPE =~ /DEBUG/ ) { if ( $filer_version =~ /<1d>/i ) { logcomment("Installed Kernel is DEBUG, Proceeding with next steps"); } else { logcomment("Installed Kernel is DEBUG, Reloading and Setting SECONDARY VARIABLE again"); my $ret = re_set_n_check_image( $Node, $secondary_var ); logcomment("DEBUG: ret : $ret"); $recheck_flag++ if ( $ret == 1 ); logcomment("DEBUG: Recheck kernel version - $recheck_flag"); if ( $recheck_flag < 3 && $ret ne 0 ) { goto recheck; } else { if ( $recheck_flag == 3 ) { logcomment("**FATAL** : $FILER_C : Installed Kernel type is still Non_Debug kernel after 3 retries, so exiting test execution"); return logresult( "INFO", msg => 1 ); } else { logcomment("**FATAL** : $FILER_C : Installed Kernel type is still Non_Debug kernel, so exiting test execution"); return logresult( "INFO", msg => 1 ); } } } } else { logcomment("**FATAL** :: KERNEL TYPES ARE NOT MATCHING FROM THE FILER AND THE TEST-SUITE"); logresult( 'FATAL', "KERNEL types are not matching from the filer and the test-suite" ); } } ##do the below steps in subtest #check_for kernel type debug or non-debug #read kernel type from test suite #if not matching load the kernel form location to the NDATE location(sub compact_flash) } } sub check_filer_connections { return if ( $BOOT_MODE == '4d' ); my $Node = shift; my $FILER_C = $Node->name(); #filer_disk_create(); my $Host = host($FILER_C); my $filer_con = connect("$FILER_C#console"); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state; handle_prompts( Node => $Node ); try { $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); } catch NACL::Transit::Exceptions::Timeout with { logcomment("**FATAL** : SOMETHING WRONG OCCURED WITH THE FILER, SO KILLING THE TEST-SUITE"); logresult( 'FATAL', "Something wrong occurred with the filer,So killing the test-suite" ); #kill_process("$err"); }; logcomment("Filer :: $FILER_C in $filer_state mode "); if ( $filer_state =~ /FIRMWARE|HALTED|BOOT_MENU|CFE_BANNER|LOADER_BANNERFIRMWARE/ ) { logcomment("Booting the Filer in MAINTENANCE mode"); $Transit_obj->set( 'change_state_timeout' => 900 ); my $timeout_flag = 0; try { $Transit_obj->change_state( to => "MAINT" ); } catch NACL::Transit::Exceptions::RetriesExceeded with { logcomment("Expected: Caught a RetriesExceeded exception"); $timeout_flag = 1; } catch NACL::Transit::Exceptions::TransitException with { $timeout_flag = 1; }; if ($timeout_flag) { logcomment("**FATAL** : Time-out occurred while changing state to MAINTENANCE, \n 1> Login to filer through service process\n 2> Bring the filer to LOADER and set appropriate boot arguments\n 3> execute the boot_ontap command and give 'wipeconfig' command on boot_menu"); logresult( 'FATAL', "Time-out occurred while changing state to MAINTENANCE, \n 1> Login to filer through service process\n 2> Bring the filer to LOADER and set appropriate boot arguments\n 3> execute the boot_ontap command and give 'wipeconfig' command on boot_menu" ); # kill_process("$err"); } } logcomment("Checking the connections of the filer $FILER_C"); my ( $Node_MGMT_Ip, $Node_MGMT_GTY, $Cluste_IP, $Cluster_GTY, $Cluster_MGMT_IP, $Cluster_MGMT_GTY ) = ( $Host->{MGMT_IP}, $Host->{MGMT_GATEWAY}, $Host->{CLUSTER_IP1}, $Host->{CLUSTER_GATEWAY1}, $Host->{CLUSTER_MGMT_IP}, $Host->{CLUSTER_MGMT_GATEWAY} ); logcomment( " $FILER_C : Node Management port - " . $Host->{MGMT_PORT} . ", Cluster Management port - " . $Host->{CLUSTER_MGMT_PORT} . ", Cluster Port - " . $Host->{CLUSTER_PORT1} ); my $console_Ip = $Host->{conn}->{console}->{connhost}; logcomment("Console Ip = $console_Ip"); if ( $filer_state =~ /CLI|USERNAME|pass/i ) { $Transit_obj->change_state( to => "CLI" ); logcomment("Set privilage to Test mode"); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); logcomment("$FILER_C : Privilage set to test"); if ( ping( 'hostname' => $console_Ip ) ) { logcomment("Console Ip $console_Ip is accessible"); } else { logcomment("CONSOLE IP $console_Ip IS NOT ACCESSIBLE!!!"); } if ( ping( 'hostname' => $Node_MGMT_Ip ) ) { logcomment("Node Management Ip $Node_MGMT_Ip is accessible"); } else { logcomment("NODE MANAGEMENT IP $Node_MGMT_Ip IS NOT ACCESSIBLE!!!"); } if ( ping( 'hostname' => $Node_MGMT_GTY ) ) { logcomment("Node Management gateway Ip $Node_MGMT_GTY is accessible"); } else { logcomment("NODE MANAGEMENT GATEWAY IP $Node_MGMT_GTY IS NOT ACCESSIBLE!!!"); } if ( ping( 'hostname' => $Cluste_IP ) ) { logcomment("Cluster Ip $Cluste_IP is accessible"); } else { logcomment("CLUSTER IP $Cluste_IP IS NOT ACCESSIBLE!!!"); } if ( ping( 'hostname' => $Cluster_GTY ) ) { logcomment("Cluster gateway Ip $Cluster_GTY is accessible"); } else { logcomment("CLUSTER GATEWAY IP $Cluster_GTY IS NOT ACCESSIBLE!!!"); } if ( ping( 'hostname' => $Cluster_MGMT_IP ) ) { logcomment("Cluster Management Ip $Cluster_MGMT_IP is accessible"); } else { logcomment("CLUSTER MANAGEMENT IP $Cluster_MGMT_IP IS NOT ACCESSIBLE!!!"); } if ( ping( 'hostname' => $Cluster_MGMT_GTY ) ) { logcomment("Node Management gateway Ip $Cluster_MGMT_GTY is accessible"); } else { logcomment("NODE MANAGEMENT GATEWAY IP $Cluster_MGMT_GTY IS NOT ACCESSIBLE!!!"); } my $filer_con = connect("$FILER_C#console"); try { say( $filer_con, "system node rename -node local -newname $FILER_C", "::>|::\*>|\*>", -timeout, 60, -ok ); sleep 5; say( $filer_con, "system timeout modify -timeout 0 ; rows 0", "::>|::\*>|\*>", -timeout, 60, -ok ); sleep 10; } catch NATE::BaseException with { logcomment("Base exception in modifying filer time-out and rows"); }; } else { if ( ping( 'hostname' => $console_Ip ) ) { logcomment("Console Ip $console_Ip is accessible"); } else { logcomment("CONSOLE IP $console_Ip IS NOT ACCESSIBLE!!!"); } } } sub check_system_partner_id { #Burt 1031945 my @Nodes = @_; my $system_id_flag = 0; #check 1 : sysconfig -v fetching the self and patner id. foreach my $Node (@Nodes) { check_filer_connections($Node); my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Api_Set_Obj; my $system_id; my $partne_id; my $system_id_1; my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); my $output; logcomment("**DEBUG**:Filer are in state : $filer_state"); if ( $filer_state =~ /CLI|USER|pass/i ) { $Transit_obj->change_state( to => "CLI" ); $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); $output = $Api_Set_Obj->execute_command( 'command' => 'sysconfig -v' ); } else { if ( $filer_state !~ /MAINT/ ) { $Transit_obj->change_state( to => "MAINT" ); } $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); $output = $Api_Set_Obj->execute_raw_command( 'command' => 'sysconfig -v' ); } if ( $output =~ /System\s*ID\s*:\s*(\S+)/ ) { $system_id = $1; logcomment("Model-id :$system_id"); } if ( $output =~ /partner\s*ID\s*:\s*(\S+)/ ) { $partne_id = $1; logcomment("Partner-id : $1"); if ( $output =~ /partner\s*ID\s*:\s*\S+\s*\((\S+)\)/ ) { my $partner_name = $1; } } if ( !$partne_id ) { $flag_set_patner_id{$FILER_C} = 1; } else { logcomment("$FILER_C : System ID - $system_id , Partner_ID - $partne_id"); } if ( !$partne_id ) { logcomment("FILER $FILER_C IS NOT DISPLAYING THE SYSTEM PARTNER ID. EXECUTE STORAGE SHOW INITIATORS TO GET THE PARTNER ID"); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /MAINT/i ) { $output = $Api_Set_Obj->execute_raw_command( 'command' => 'storage show initiators' ); } else { $output = $Api_Set_Obj->execute_raw_command( 'command' => 'run local storage show initiators' ); } my @result = split( '\n', $output ); foreach my $line (@result) { if ( $line =~ /(\S+)\s*(\d+)\s*\(self/i ) { $system_id = $2; } elsif ( $line =~ /(\S+)\s*(\d+)/ ) { #print"line=$line\n"; if ( $line !~ "$FILER_C" ) { $partne_id = $2; } } } $system_id_flag = 1 if ( !$partne_id ); } logcomment("$FILER_C : System ID - $system_id , Partner_ID - $partne_id") if ($partne_id); $sysconfig_sysID{'self'}{$FILER_C} = $system_id; $sysconfig_sysID{'patner'}{$FILER_C} = $partne_id; } if ( $system_id_flag == 1 ) { #Burt 1031945 Assigning the Drives if partner id is not displaying. logcomment("**DEBUG** : Sysid and partner id is not displayed So assigning the one Drives to each Node"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Api_Set_Obj; my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); my $return = disk_show_n( Node => $Node ); if ( keys %{$return} ) { my @disk_part; my @container_disk; push( @disk_part, keys %{$return} ); grep { if ( $_ !~ /p\d+/i ) { push( @container_disk, $_ ) } } @disk_part; if ( $filer_state =~ /CLI|UP/i ) { $Transit_obj->change_state( to => "CLI" ); $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); $Api_Set_Obj->execute_raw_command( 'command' => "run local \"priv set -q test; disk assign $container_disk[0] -o $FILER_C\"" ); } else { if ( $filer_state !~ /MAINT/ ) { $Transit_obj->change_state( to => "MAINT" ); } $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); $Api_Set_Obj->execute_raw_command( 'command' => "disk assign $container_disk[0] -o $FILER_C" ); } $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); } } } else { logcomment("**DEBUG** : Drives are assigned to the Filer"); } my @subtests; foreach (@Nodes) { my $FILER_C = $_->name(); if ( $flag_set_patner_id{$FILER_C} == 1 ) { push( @subtests, subtest( \&set_partner_id, -runid, "set_partner_id_" . "$FILER_C", -bg, "--", $_, ) ); } } Subtest::wait_finish( subtest => [@subtests] ); } sub check_partner_id { #Burt 1031945 my $Node = shift; my $system_id; my $partne_id; my $FILER_C = $Node->name(); if ( $flag_set_patner_id{$FILER_C} == 1 ) { my $Host = host($FILER_C); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $output; my $Api_Set_Obj; logcomment("$FILER_C: Re-Executing STORAGE SHOW INITIATORS TO GET THE PARTNER ID"); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /MAINT/i ) { $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); $output = $Api_Set_Obj->execute_raw_command( 'command' => 'storage show initiators' ); } else { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); $output = $Api_Set_Obj->execute_raw_command( 'command' => 'run local storage show initiators' ); } my @result = split( '\n', $output ); foreach my $line (@result) { if ( $line =~ /(\S+)\s*(\d+)\s*\(self/i ) { print "Filer name - $1\n"; print "System_id - $2\n"; $system_id = $2; } elsif ( $line =~ /(\S+)\s*(\d+)/ ) { if ( $line !~ "$FILER_C" ) { print "Partner Filer name - $1\n"; print "Partner System_id - $2\n"; $partne_id = $2; } } } } return ( $system_id, $partne_id ); } sub set_partner_id { #Burt 1031945 my $Node = shift; my ( $system_id, $partne_id ); my $filer_state; my $Transit_obj; my $Api_Set_Obj; my $FILER_C = $Node->name(); $Transit_obj = NACL::Transit->new( name => $FILER_C ); $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); my ( $system_id, $partne_id ) = check_partner_id($Node); if ( !$partne_id ) { foreach ( keys %{ $sysconfig_sysID{'self'} } ) { $system_id = $sysconfig_sysID{'self'}{$FILER_C}; next if ( $_ eq $FILER_C ); $partne_id = $sysconfig_sysID{'self'}{$_}; } } logcomment("System_id = $system_id and Partner System_id = $partne_id"); { my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $prompt_fw; my $output; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_array = $Host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); #set the execution server to the current node my @filer_working = $Server->servers(); logcomment("Number of filers working are @filer_working"); my $timeout = 3600; ## Changing state to LOADER and setting Partner System_id logcomment("Filer - $FILER_C : Changing state to LOADER and setting partner-sysid"); $Transit_obj->change_state( to => "FIRMWARE" ); try { $Server->console( cmd => "setenv partner-sysid $partne_id", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => "saveenv", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); } catch NATE::BaseException with { logcomment("FILER : $FILER_C SOMETHING WENT WRONG IN SETTING PARTNER-SYSID"); }; sleep(60); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/i ) { logcomment("Filer $FILER_C are in FIRMWARE state, so change the state to $filer_state"); $Transit_obj->change_state( to => "$filer_state" ); } $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer are in state : $state"); sleep(60); if ( $state =~ /CLI|UP/i ) { $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $output = $Api_Set_Obj->execute_command( 'command' => 'sysconfig -v' ); } elsif ( $state =~ /MAINT/i ) { if ( $filer_state !~ /MAINT/ ) { $Transit_obj->change_state( to => "MAINT" ); } $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200 ); $output = $Api_Set_Obj->execute_raw_command( 'command' => 'sysconfig -v' ); } my $partne_id_1; if ( $output =~ /System\s*ID\s*:\s*(\S+)/ ) { $system_id = $1; } if ( $output =~ /partner\s*ID\s*:\s*(\S+)/ ) { $partne_id_1 = $1; } if ( !$partne_id_1 ) { logcomment("**FATAL** : FILERS ARE NOT DISPLAYING THE SYSTEM PARTNER_ID, PLEASE CONFIGURE THE FILER MANUALLY AND RE-RUN THE TEST WITH CORRECT CONFIGURATION"); logresult( 'FATAL', 'Filers are not displaying the system Partner_Id, Please configure the filer manually and re-run the test with the correct configuration' ); } else { logcomment("$FILER_C: SYSTEM ID - $system_id and PARTNER ID - $partne_id_1"); } } } sub check_partition_remove_evn_parti_var { my @Nodes = NACL::C::Node->find(); my $partition_enabled = 0; logcomment("Log dir - $LOGDIR"); my %change_to_filer_state = {}; foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); $change_to_filer_state{$FILER_C} = $filer_state; logcomment("Filers is in $filer_state state"); my $API_Object; my $drive_data; if ( $filer_state =~ /CLI|USER|pass/i ) { $Transit_obj->change_state( to => "CLI" ); $API_Object = $Node->get_7m_or_nodescope_apiset( connid => "console" ); $API_Object->set_timeout( "connectrec-timeout" => 6000 ); $drive_data = $API_Object->execute_raw_command( 'command' => "run local disk show -v" ); } else { if ( $filer_state !~ /MAINT/ ) { $Transit_obj->change_state( to => "MAINT" ); } my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /MAINT/ ) { $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); } $API_Object->set_timeout( "connectrec-timeout" => 6000 ); $drive_data = $API_Object->execute_raw_command( 'command' => "disk show -v" ); } logcomment("**DEBUG**: API_OBJECT and Drive data collected"); my @data = split( /\n/, $drive_data ); my @disk_part; foreach my $line (@data) { if ( $line =~ /(\S+)\s+\S+\s*\(\d+/ ) { my $disk = $1; if ( $disk =~ /P\d+$/ ) { push( @disk_part, $disk ); $partition_enabled = 1; } } } last if ( $partition_enabled == 1 ); } #End of forecach nodes if ( $partition_enabled == 1 ) { logcomment("PARTITION has been enabled on system, Removing partition and re-initilizing setups"); my @subtests; foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $FILER_C = $Node->name(); logcomment("$FILER_C : Changing state to MAINTENANCE Mode"); push( @subtests, subtest( \&boot_it_good_Maint, -runid, "boot_it_good_Maint_" . "$FILER_C", -bg, "--", Test => $Test, Filer => $FILER_C ) ); } Subtest::wait_finish( subtest => [@subtests] ); sleep 3; logcomment("Removing partitions "); remove_partition( Noderef => [@Nodes] ); logcomment("Partitioned removal complete"); logcomment("Halt Filer and Set environment variable"); my @re_subtest; foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $filer_name = $Node->name(); logcomment( "Calling change_state " . scalar( localtime() ) ); push( @re_subtest, subtest( \&boot_filer_in_req_state, -runid, "shutdown_node_" . "$filer_name", -bg, "--", filer_name => $filer_name, change_state_to => 'FIRMWARE' ) ); } Subtest::wait_finish( subtest => [@re_subtest] ); logcomment("Booted Filers in FIRMWARE"); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $FILER = $Node->name(); my $host = host($FILER); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; my $Transit_obj = NACL::Transit->new( name => $FILER ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("FILER is in LOADER prompt, disabling auto partition env variables"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); $Server->console( cmd => 'setenv root-uses-shared-disks? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'setenv allow-ssd-partitions? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'setenv root-uses-shared-ssds? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'saveenv', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); logcomment("Saved all required environment variables to disable autopartition"); } else { logcomment("CANNOT SET ENVIRONMENT VARIABLE , AS FILER IS NOT IN LOADER PROMPT"); } } #end of foreach node, setting env variable logcomment("Set-up Filer, Re-initilize filer"); setup_cmode_filers( node_present => \@Nodes, Test => $Test ); logcomment("unpartitioned setup and re-initilization completed"); my $handle; open( $handle, '>>', "$LOGDIR/../../../PARTITION_CREATED" ) or die("Cant open $LOGDIR/../../../PARTITION_CREATED"); print $handle "PARTITION_DISABLED=yes"; close($handle) or die("$LOGDIR/../../../PARTITION_CREATED"); logcomment("**DEBUG** : Check file created"); } else { logcomment("PARTITION HAS NOT BEEN ENABLED ON SYSTEM"); logcomment("PARTITION IS NOT ENABLED, Checking and Setting Enivornment vairable to avoid auto partition duing cluster setup or re-initilization"); logcomment("Halt Filer and Set environment variable"); my @re_subtest; foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $filer_name = $Node->name(); logcomment( "Calling change_state " . scalar( localtime() ) ); push( @re_subtest, subtest( \&boot_filer_in_req_state, -runid, "shutdown_node_" . "$filer_name", -bg, "--", filer_name => $filer_name, change_state_to => 'FIRMWARE' ) ); } Subtest::wait_finish( subtest => [@re_subtest] ); logcomment("Booted Filers in FIRMWARE"); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $FILER = $Node->name(); my $host = host($FILER); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; my $Transit_obj = NACL::Transit->new( name => $FILER ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("FILER is in LOADER prompt, disabling auto partition env variables"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); $Server->console( cmd => 'setenv root-uses-shared-disks? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'setenv allow-ssd-partitions? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'setenv root-uses-shared-ssds? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'saveenv', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); logcomment("Saved all required environment variables to disable autopartition"); my $handle; open( $handle, '>>', "$LOGDIR/../../../PARTITION_CREATED" ) or die("Cant open $LOGDIR/../../../PARTITION_CREATED"); print $handle "PARTITION_DISABLED=yes"; close($handle) or die("$LOGDIR/../../../PARTITION_CREATED"); logcomment("**DEBUG** : Check file created"); } else { logcomment("CANNOT SET ENVIRONMENT VARIABLE , AS FILER IS NOT IN LOADER PROMPT"); } #burt:1031419 my $state_filer = $change_to_filer_state{$FILER}; logcomment("Filer $FILER was in $state_filer state before setting ENVIRONMENT VARIABLE"); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer $FILER is in $filer_state state"); if ( $filer_state ne $state_filer ) { logcomment("Changing the Filer $FILER state back to $state_filer"); $Transit_obj->change_state( to => "$state_filer" ); } else { logcomment("Filer $FILER is already in required state $state_filer"); } } #end of foreach node, setting env variable } } # End of subroutin:1261 sub switch_img_n_check_ver { my ( $Node, $reqd_kernel ) = @_; my $Home = $ENV{HOME}; my $env_set_check; my $set_env; $Home = trim($Home); my $FILER = $Node->name(); my $host = host($FILER); my $Host = host($FILER); my $Mode = $Node->mode(); my $host_file = "$Home/NDATE/FILER_INFO/$FILER"; my $filer_config = `cat $host_file`; my $def_pass; logcomment(" Node -$FILER and Required Kernel - $reqd_kernel"); foreach my $line ( split /\n/, $filer_config ) { if ( $line =~ /^\#/ ) { next; # skip overall result report starting with a '#'. } elsif ( $line =~ /^default_password=(.*)/ ) { $def_pass = $1; } } my $Transit_obj = NACL::Transit->new( name => $FILER ); $Transit_obj->set( 'timeout' => 600, maxcycle => 20 ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer - $FILER state is $filer_state"); logcomment("Required Kernel for testing is - $reqd_kernel"); ################################################ ##requirements to create server object ################################################ my $prompt_fw; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); #set the execution server to the current node my @filer_working = $Server->servers(); logcomment("Number of filers working are @filer_working"); my $timeout = "7200"; # Change time-out in final code. logcomment(" Filer - $FILER : Changing state to LOADER to set the option AUTOBOOT to false"); $Transit_obj->change_state( to => "FIRMWARE" ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("FILER is in LOADER prompt, setting AUTOBOOT to false"); } else { logcomment("FILER IS NOT IN LOADER PROMPT"); } my $loop_chek = 0; boot_again: logcomment("Reset fields 'timeout' and 'maxcycle'"); my $boot_cmd; if ( $loop_chek == 1 ) { $boot_cmd = "boot_primary"; } else { $boot_cmd = "boot_backup"; } $Transit_obj->set( 'timeout' => 700, maxcycle => 30 ); $Transit_obj->set( 'change_state_timeout' => 600 ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); my $ret_flag = 0; if ( $state !~ /FIRMWARE/ ) { logcomment("FILER IS NOT IN LOADER, CHANGING STATE TO LOADER"); $Transit_obj->change_state( to => "FIRMWARE" ); } { #Burt 1033916 nmamatha logcomment( "FILER is in LOADER, Issuing " . "$boot_cmd" ); eval { $Server->console( cmd => $boot_cmd, additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { ".*This platform is not supported in this release.*" => "FATAL_PROMPT", ".*The system will now halt.*" => "FATAL_PROMPT", ".*Do you want to continue.*" => 'y', ".*Copyright (C) 1992-2016 NetApp.*" => "\003", ".*Copyright (C) 1992-2016 NetApp.*" => "^C", ".*All rights reserved.*" => '^C', ".*Press Ctrl-C for Boot Menu.*" => "^C", ".*Press Ctrl-C for Boot Menu.*" => "\003", ".*Selection (1-8).*" => '5', ".*Please choose one of the following.*" => '5', ".*What is the password for.*" => "\013", "$prompt_fw" => "FINAL_PROMPT", ".*Please answer yes or no.*" => "yes", ".*Continue with boot.*" => "yes", ".*Continue with boot?.*" => "yes", "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", ".*login.*" => "admin", ".*Password.*" => $def_pass, ".*::>.*" => "FINAL_PROMPT", "::*>" => "FINAL_PROMPT", "::>" => "FINAL_PROMPT", ".*Password.*" => $def_pass, ".*Retype the password.*" => $def_pass, ".*Cannot set variable.*" => "FATAL_PROMPT", } ); }; if ($@) { if ( $@ =~ /This platform is not supported in this release/i ) { logcomment("**FATAL**: THE PLATFORM IS NOT SUPPORTED IN THE RELEASE "); logresult( 'FATAL', "THE PLATFORM IS NOT SUPPORTED IN THE RELEASE" ); } } } sleep 300; logcomment("Check state of filer"); $Transit_obj->set( 'change_state_timeout' => 600 ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer state is after issuing $boot_cmd - $state"); logcomment("Booted kernel using $boot_cmd command"); sleep 30; if ( $state =~ /NONSTD_BOOT_QUERY/ ) { logcomment("Filer is in Non standard boot query state"); $Server->console( cmd => "\013", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { ".*Do you want to continue.*" => 'y', ".*Continue with boot?.*" => "yes", ".*Continue with boot.*" => "yes", "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", } ); } else { logcomment("FILER IS NOT IN NON STANDARD FILER STATE, PROCEEDING WITH KERNEL CHECK"); } logcomment("Check filer state again"); $Transit_obj->set( 'change_state_timeout' => 600 ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer state is after issuing $boot_cmd - $state"); logcomment("Booted kernel using $boot_cmd command"); logcomment("Checking kernel version"); my $Api_Set_Obj; my $filer_version; my $K_type_filer; if ( $state !~ /MAINT/ ) { if ( $state =~ /CLI|USER|pass/i ) { $Transit_obj->change_state( to => "CLI" ); my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); } logcomment("FILER IS NOT IN MAINTENANACE MODE"); $Transit_obj->change_state( to => "MAINT" ); sleep 300; my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG : State of filer check 2 - $state"); } $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); my $version = $Api_Set_Obj->execute_raw_command( command => "version" ); $filer_version = $version; if ( $filer_version =~ /7Mode/i ) { logcomment("**FATAL** : Filer is in 7Mode, please install CMode kernel and re-run the test"); logresult( 'FATAL', "Filer is in 7Mode, please install CMode kernel and re-run the test" ); } if ( $filer_version =~ /<1d>/i ) { logcomment("Kernel type - DEBUG kernel"); $K_type_filer = 'DEBUG'; } else { logcomment("Kernel type - NON-DEBUG kernel"); $K_type_filer = 'NON_DEBUG'; } logcomment("Required kernel - $reqd_kernel and kernel type on filer - $K_type_filer"); if ( $reqd_kernel eq $K_type_filer ) { logcomment("Required kernel is present on filer "); $env_set_check = $boot_cmd; logcomment("DEBUG: Command used to boot image - $env_set_check"); } else { logcomment("BACKUP KERNEL IS DIFFERENT FROM THE REQUIRED KERNEL"); logcomment("Checking with primary image"); $loop_chek++; logcomment("DEBUG : loope check - $loop_chek"); if ( $loop_chek < 2 ) { goto boot_again; } else { logcomment("**FATAL** : Executed both boot_backup and boot_primary, either of the image has required kernel version"); logcomment("use option 9 from ndate menu, and install same build version of debug and non-debug kernels"); logresult( "FATAL", "Executed \"boot_backup\" and \"boot_primary\", either of the image has required kernel, use option 9 and install image" ); } } logcomment("Set environment variable USE_SECONDARY for subsequent boot commands"); if ( $env_set_check eq "boot_backup" ) { logcomment("DEBUG: boot_backup command was used"); $set_env = "setenv USE_SECONDARY true"; } elsif ( $env_set_check eq "boot_primary" ) { logcomment("DEBUG: boot_primary command was used"); $set_env = "setenv USE_SECONDARY false"; } logcomment("Set env - $set_env"); $Transit_obj->set( 'change_state_timeout' => 600 ); $Transit_obj->change_state( to => "FIRMWARE" ); my $ret_flag = 0; try { #Burt 1033916 nmamatha eval { $Server->console( cmd => "$set_env", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { ".*This platform is not supported in this release.*" => "FATAL_PROMPT", ".*The system will now halt.*" => "FATAL_PROMPT", "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", ".*LOADER.*" => "FINAL_PROMPT", ".*Cannot set variable.*" => "FATAL_PROMPT" } ); }; if ($@) { if ( $@ =~ /This platform is not supported in this release/i ) { logcomment("**FATAL**: THE PLATFORM IS NOT SUPPORTED IN THE RELEASE "); logresult( 'FATAL', "THE PLATFORM IS NOT SUPPORTED IN THE RELEASE" ); } elsif ( $@ =~ /Cannot set variable/i ) { #Modified for Burt-1162786 logcomment("**FATAL**: The kernel image CAN NOT be changed as the boot arg is readonly in this setup"); logresult( 'FATAL', "The kernel image CAN NOT be changed as the boot arg is readonly in this setup" ); $ret_flag = 1; } } } catch NATE::BaseException with { logcomment("FILER : SOMETHING WENT WRONG IN SETTING ENV VARIABLE"); }; sleep(60); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer is in $state state "); logcomment("Return - $set_env"); if ($ret_flag) { return $ret_flag; } else { return $set_env; } } sub re_set_n_check_image { my ( $Node, $second_var ) = @_; my $set_env; my $FILER = $Node->name(); my $host = host($FILER); my $prompt_fw; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); my $Transit_obj = NACL::Transit->new( name => $FILER ); $Transit_obj->set( 'timeout' => 600, maxcycle => 20 ); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); #set the execution server to the current node my @filer_working = $Server->servers(); logcomment("Number of filers working are @filer_working"); my $timeout = "7200"; # Change time-out in final code. logcomment("Node -$FILER and secondary variable set - $second_var"); logcomment("Set secondary variable for subsequent booting"); if ( $second_var eq "setenv USE_SECONDARY false" ) { $set_env = "setenv USE_SECONDARY true"; } elsif ( $second_var eq "setenv USE_SECONDARY true" ) { $set_env = "setenv USE_SECONDARY false"; } logcomment("Set env - $set_env"); $Transit_obj->set( 'change_state_timeout' => 600 ); $Transit_obj->change_state( to => "FIRMWARE" ); sleep 30; my $ret_flag = 0; try { #Burt 1033916 nmamatha eval { $Server->console( cmd => "$set_env", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { ".*This platform is not supported in this release.*" => "FATAL_PROMPT", ".*The system will now halt.*" => "FATAL_PROMPT", "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", ".*LOADER.*" => "FINAL_PROMPT", ".*Cannot set variable.*" => "FATAL_PROMPT", } ); }; if ($@) { if ( $@ =~ /This platform is not supported in this release/i ) { logcomment("**FATAL**: THE PLATFORM IS NOT SUPPORTED IN THE RELEASE "); logresult( 'FATAL', "THE PLATFORM IS NOT SUPPORTED IN THE RELEASE" ); } elsif ( $@ =~ /Cannot set variable/i ) { #Modified for Burt-1162786 logcomment("**FATAL**: The kernel image CAN NOT be changed as the boot arg is readonly in this setup"); $ret_flag = 1; } } } catch NATE::BaseException with { logcomment("FILER :SOMETHING WENT WRONG IN SETTING ENV VARIABLE"); }; sleep 60; my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer is in $state state "); if ($ret_flag) { return 0; } else { return 1; } } sub check_encrypt_disk { my @Nodes; push( @Nodes, shift(@_) ); foreach my $node (@Nodes) { my $FILER_C = $node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer state is : $filer_state"); my $Host = host($FILER_C); my $prompt_fw; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_array = $Host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER_C ); #set the execution server to the current node my @disk_list; my $API_Object; my @FILER_ARR; if ( $filer_state !~ /LOADER|FIRMWARE/ ) { logcomment("Changing the state to FIRMWARE"); $Transit_obj->change_state( to => "FIRMWARE" ); } my $filer_state1 = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state1 =~ /LOADER|FIRMWARE/ ) { logcomment("Filer - $FILER_C : Setting the environment variable bootarg.raid.allow-raid-tec "); $Server->console( cmd => "setenv bootarg.raid.allow-raid-tec false", additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => "saveenv", additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => "printenv bootarg.raid.allow-raid-tec", additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); } logcomment("As filer $FILER_C was in $filer_state before setting the environment variable, changing back to $filer_state state"); $Transit_obj->change_state( to => "$filer_state" ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /LOADER|FIRMWARE/ ) { boot_filer_in_req_state( filer_name => $FILER_C, change_state_to => 'MAINT' ); } my $loop = 0; foreach ( $loop = 0 ; $loop < 3 ; $loop++ ) { $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state !~ /MAINT/ ) { sleep 120; } last if ( $filer_state =~ /MAINT/ ); $loop++; } $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("checking for the disk_list in the Filer"); if ( $filer_state =~ /MAINT/ ) { $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $API_Object->set_timeout( "connectrec-timeout" => 1200 ); my $return = disk_list_data( API_Obj => $API_Object ); @disk_list = keys(%$return); } if ( $filer_state =~ /CLI|UP/ ) { my $return = disk_show_v( Node => $node ); @disk_list = keys(%$return); } my $encryption = 0; if ( scalar @disk_list == 0 ) { logcomment("$FILER_C : disk_list result shows no disks present in the filers "); logcomment("Changing state and checking for Encrypting disk"); $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); $Transit_obj->change_state( to => "FIRMWARE" ); my $i = 0; Loop1: $i++; sleep 60; $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /FIRMWARE|LOADER/ ) { $Transit_obj->change_state( to => "MAINT" ); Loop2: sleep 60; my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $filer_state =~ /MAINT/ ) { $Transit_obj->change_state( to => "FIRMWARE" ); sleep 60; } else { last if ( $i == 3 ); goto Loop2; } } else { logcomment("Filer is not in FIRMWARE State"); last if ( $i == 3 ); goto Loop1; } logcomment("$LOGDIR"); my $logfile = "$LOGDIR/check_filer_setup/check_encrypt_disk_$FILER_C.log"; open( FH, "$logfile" ) || logresult( 'FATAL', " Cannot open the file $logfile" ); my @arr1 = ; close(FH); foreach my $line (@arr1) { $line =~ s/\[HDD\S+\]//g; if ( ( $line =~ /Detected encrypting disk/ ) && ( $line =~ /(\d+) of (\d+) disks checked/ ) ) { my $drv1 = $1; my $drv2 = $2; if ( $drv1 == $drv2 ) { logcomment("Filer has only Encrypted Drives"); $encryption = 1; } else { logcomment("**FATAL**: $FILER_C : MIXED DRIVES ARE PRESENT ON THE FILER ,BOTH Encrypted and Normal Drives "); logresult( 'FATAL', " $FILER_C : MIXED DRIVES ARE PRESENT ON THE FILER ,BOTH Encrypted and Normal Drives " ); } } if ( $line =~ /Detected encrypting disk/ ) { logcomment("Enabling the bootarg.storageencryption.support variable"); $Transit_obj->change_state( to => "FIRMWARE" ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("Filer - $FILER : Setting the environment variables"); $Server->console( cmd => "setenv bootarg.storageencryption.support true", additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); } $Transit_obj->change_state( to => "MAINT" ); } } if ( $encryption != 1 ) { logcomment("**FATAL** : $FILER_C : NOT DETECTED THE ENCRYPTED DRIVES MESSAGE AND NO DRIVRS PRESENT IN THE FILER,So Exiting From The Tese"); logresult( 'FATAL', "$FILER_C : NOT DETECTED THE ENCRYPTED DRIVES MESSAGE AND NO DRIVRS PRESENT IN THE FILER,So Exiting From The Tese" ); } } } } #################################################################################################### # NVME - change filer from partition to ClL- Un-partition . # Boot in loader , set boot arguments for un-partition , clean_config and cluster join. # Return : NA # Burt : 1159590 # Author : shiwani #################################################################################################### sub nvme_unparition_cli { my @Nodes = @_; logcomment("Removing partitions "); remove_partition( Noderef => [@Nodes] ); logcomment("Partitioned removal complete"); nvme_drv_assign( Nodes => \@Nodes, assign => "yes" ); my @re_subtest; foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); logcomment( "Calling change_state " . scalar( localtime() ) ); push( @re_subtest, subtest( \&nvme_boot_in_req_mode, -runid, "boot_to_loader_$FILER_C", -bg, "--", Trans => $Transit_obj, boot_to => 'FIRMWARE', ) ); } Subtest::wait_finish( subtest => [@re_subtest] ); logcomment("Booted filers in Firmware"); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $FILER = $Node->name(); my $host = host($FILER); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; my $Transit_obj = NACL::Transit->new( name => $FILER ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("Filer is in Loader prompt, disabling auto partition ENV variables"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); $Server->console( cmd => 'setenv root-uses-shared-disks? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'setenv allow-ssd-partitions? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'setenv root-uses-shared-ssds? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'saveenv', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); logcomment("Saved all required environment variables to disable autopartition"); } else { logcomment("CANNOT set environment variable, as filer is NOT in loader prompt"); } } my @subtests; foreach my $Node (@Nodes) { nvme_boot_to_maint( Node => $Node ); my $filer_name = $Node->name(); my $env_data = "Enter path"; push( @subtests, subtest( \&nvme_clean_config, -runid, "clean_config_node_$filer_name", -bg, "--", Node => $Node, env_var => $env_data ) ); } Subtest::wait_finish( subtest => [@subtests] ); logcomment("Creating cluster using nodes available"); create_join_cluster( node_present => [@Nodes] ); logcomment("Completed creating and joining cluster "); } #################################################################################################### # NVME - change filer to ClL- Un-partition . # Boot in loader , set boot arguments for un-partition , clean_config and cluster join. # Return : NA # Burt : 1159590 # Author : shiwani #################################################################################################### sub nvme_cli_unaprt { my @Nodes = @_; my @re_subtest; foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); logcomment( "Calling change_state " . scalar( localtime() ) ); push( @re_subtest, subtest( \&nvme_boot_in_req_mode, -runid, "boot_to_loader_$FILER_C", -bg, "--", Trans => $Transit_obj, boot_to => 'FIRMWARE', ) ); } Subtest::wait_finish( subtest => [@re_subtest] ); logcomment("Booted filers in Firmware"); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $FILER = $Node->name(); my $host = host($FILER); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; my $Transit_obj = NACL::Transit->new( name => $FILER ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("Filer is in Loader prompt, disabling auto partition ENV variables"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); $Server->console( cmd => 'setenv root-uses-shared-disks? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'setenv allow-ssd-partitions? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'saveenv', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); logcomment("Saved all required environment variables to disable autopartition"); } else { logcomment("CANNOT set environment variable, as filer is NOT in loader prompt"); } } foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $env_data = "Enter path"; push( @subtests, subtest( \&nvme_clean_config, -runid, "clean_config_node_$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) ); } Subtest::wait_finish( subtest => [@subtests] ); logcomment("Creating cluster using nodes available"); create_join_cluster( node_present => [@Nodes] ); logcomment("Completed creating and joining cluster "); } #################################################################################################### # NVME - change from Un-partition to partition. # Boot in loader , set boot arguments for partition , clean_config and cluster join. # Return : NA # Burt : 1159590 # Author : shiwani #################################################################################################### sub check_partition_set_to_partition { my @Nodes = @_; my @re_subtest; foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); logcomment( "Calling change_state " . scalar( localtime() ) ); push( @re_subtest, subtest( \&nvme_boot_in_req_mode, -runid, "boot_to_loader_$FILER_C", -bg, "--", Trans => $Transit_obj, boot_to => 'FIRMWARE', ) ); } Subtest::wait_finish( subtest => [@re_subtest] ); logcomment("Booted Filers in Firmware state"); foreach my $Node (@Nodes) { $Node->refresh_command_interface(); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $FILER = $Node->name(); my $host = host($FILER); my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $prompt_fw = "($prompts)"; my $Transit_obj = NACL::Transit->new( name => $FILER ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("Filer is in Loader prompt, disabling auto partition env variables"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); $Server->console( cmd => 'unsetenv root-uses-shared-disks? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'unsetenv allow-ssd-partitions? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'unsetenv root-uses-shared-ssds? false', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'saveenv', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); logcomment("Set all required environment variables to enable autopartition"); } else { logcomment("Could not set environment variable, as filer is not in loader prompt"); } } nvme_drv_assign( Nodes => \@Nodes, assign => "yes" ); logcomment("Set-up Filer, Re-initilize filer"); my @subtests; foreach my $Node (@Nodes) { my $filer_name = $Node->name(); my $env_data = "Enter path"; push( @subtests, subtest( \&nvme_clean_config, -runid, "clean_config_node_$filer_name", -bg, "--", Node => $Node, env_var => $env_data ) ); } Subtest::wait_finish( subtest => [@subtests] ); logcomment("Creating cluster using nodes available"); create_join_cluster( node_present => [@Nodes] ); logcomment("Completed creating and joining cluster "); } sub assign_partitions { my %fil_owner = (); my @parti = (); my $check = 1; my $drv_count; my $FILER_C = $Nodes[0]->name(); my $FILERB = $Nodes[1]->name(); my $Host_A = host($FILER_C); my @disks = (); my $API_Obj_A = NACL::APISet->new( hostobj => $Host_A, category => "Node", interface => "CLI", set => "Maintenance", ); $API_Obj_A->set_timeout( "connectrec-timeout" => 7200 ); my @free_drives = keys %{ disk_show_n( Node => $Nodes[0] ) }; foreach my $dsk (@free_drives) { my ($part) = ( $dsk =~ /(P\d+)$/ ); if ($part) { push( @{ $fil_owner{$FILER_C}{$part} }, $dsk ); } else { push( @disks, $dsk ); } if ( $part =~ /(P(\d+))$/ ) { push( @parti, $1 ); } } @parti = sort( unique(@parti) ); logcomment(" PARTI : @parti"); logcomment("Drives are : @disks"); my $Host_B = host($FILERB); my $API_Obj_B = NACL::APISet->new( hostobj => $Host_B, category => "Node", interface => "CLI", set => "Maintenance", ); $API_Obj_B->set_timeout( "connectrec-timeout" => 7200 ); my @free_drives = keys %{ disk_show_n( Node => $Nodes[1] ) }; foreach my $dsk (@free_drives) { my ($part) = ( $dsk =~ /(P\d+)$/ ); if ($part) { push( @{ $fil_owner{$FILER_C}{$part} }, $dsk ); } else { push( @disks, $dsk ); } } logcomment("Drives1 are : @disks"); @disks = unique(@disks); my @partitions = keys $fil_owner{$FILER_C}; @partitions = sort(@partitions); grep { logcomment(" $FILER_C : Drives belongs to partition $_ : @{$fil_owner{$FILER_C}{$_}}") } @partitions; $drv_count = @disks; logcomment("Total $drv_count drives are unowned $FILER_C"); if ( $drv_count >= 2 ) { $drv_count = floor( $drv_count / 2 ); } logcomment("DEBUG : Drv to assign : $drv_count"); my $sysidA; my $sysidB; my $output = $API_Obj_A->execute_raw_command( command => "disk show -a" ); foreach my $line ( split( /\n/, $output ) ) { if ( ( $line =~ /$FILER_C\s+(\S+)/ ) || ( $line =~ /Local System ID\:\s+(\S+)/ ) ) { $sysidA = $1; ($sysidA) = $sysidA =~ /\((\d+)\)/ if ( $sysidA =~ /\(/ ); logcomment("Filer : $FILER_C and System ID : $sysidA"); } } my $output = $API_Obj_B->execute_raw_command( command => "disk show -a" ); foreach my $line ( split( /\n/, $output ) ) { if ( ( $line =~ /$FILERB\s+(\S+)/ ) || ( $line =~ /Local System ID\:\s+(\S+)/ ) ) { $sysidB = $1; ($sysidB) = $sysidB =~ /\((\d+)\)/ if ( $sysidB =~ /\(/ ); logcomment("Filer : $FILERB and System ID : $sysidB"); } } my @dsks = @disks[ 0 .. $drv_count ]; foreach my $dsk (@dsks) { $API_Obj_A->execute_raw_command( command => "disk assign $dsk -s $sysidA -o $FILER_C" ); } my @dsks = @disks[ $drv_count + 1 .. $#disks ]; foreach my $dsk (@dsks) { $API_Obj_B->execute_raw_command( command => "disk assign $dsk -s $sysidB -o $FILERB" ); } #Assigning partitions to one Nodes foreach my $pa (@parti) { logcomment("$FILER_C : $pa partitions are found, so assigning the $pa partitions"); my @part_temp = @{ $fil_owner{$FILER_C}{"$pa"} }; foreach (@part_temp) { my $out = $API_Obj_A->execute_raw_command( 'command' => "disk assign $_ -o $FILER_C" ); if ( $out =~ /Try the command again/i ) { my $out = $API_Obj_B->execute_raw_command( 'command' => "disk assign $_ -o $FILERB" ); } } } }