#!/usr/software/bin/perl # $Id: //depot/prod/test/main/storage/hdd/NADQ_SEA/CMODE/CONVERT_SETUP_INSTALL_IMAGE.thpl#8 $ # Copyright (c) 2005 Network Appliance, Inc. # All rights reserved # ## @summary Mode ## Setup conversion 7Mode to CMode and vice versa ## ## @description ## Set-up conversion with or without installation of kernel image ## ## @Test Mode ## Maintenance mode/File system mode /LOADER ## ## ## ## @usage ## The test has run independently as part of STEST. ## ## @dependencies ## for conversion both the Filer-configuration files are necessary(_BR and _CMODE) ## ## @status Automated # @auther Arjun.Lavu@netapp.com ## @burt 758407 ## @change khosur: Modified script for Burt-1029180 ## ############################################################################# # Compiler directives. use strict; use warnings; # MODULES imports use Storage::Common_Lib; use Storage::NVMe_Common_Lib; use Storage::Tahiti_Common_Lib; use List::MoreUtils qw(each_array); #User defined functions # Global parameters use vars qw( $FILER $TEST_CONFIG $TEST_SETUP $MODE $LOGDIR $EOE $EMAIL $MAIL_TO $MAIL_FROM $Primary_Node $CONVERT_TO $IMAGE $MODE $ADD_TEST_CASE $WIPE_LBAS $DISK_SERIAL_NUMBER @Serial_numbers $NVMe_SETUP $BOOT_STATE $ENV_FIL_LOC ); my $params = NATE::ParamSet->new( global => 1 ); $FILER = $params->get( 'FILER', default => undef ); $LOGDIR = $params->get( 'LOGDIR', default => '' ); $EOE = $params->get( 'EOE', default => 'default' ); $EMAIL = $params->get( 'EMAIL', default => 'y' ); $MAIL_TO = $params->get( 'MAIL_TO', default => 'Email to' ); $MAIL_FROM = $params->get( 'MAIL_FROM', default => 'Email from' ); $TEST_CONFIG = $params->get( 'TEST_CONFIG', default => 'D' ); $Primary_Node = $params->get( 'Primary_Node', default => 'N' ); $CONVERT_TO = $params->get( 'CONVERT_TO', default => 'N' ); $IMAGE = $params->get( 'IMAGE', default => 'N' ); $MODE = $params->get( 'MODE', default => 'MAINT' ); $ADD_TEST_CASE = $params->get( 'ADD_TEST_CASE', default => undef ); $WIPE_LBAS = $params->get( 'WIPE_LBAS', default => '' ); $DISK_SERIAL_NUMBER = $params->get( 'DISK_SERIAL_NUMBER', default => '' ); $NVMe_SETUP = $params->get( 'NVMe_SETUP', default => 'NO' ); $BOOT_STATE = $params->get( 'BOOT_STATE', default => 'MAINT' ); $ENV_FIL_LOC = $params->get( 'ENV_FIL_LOC', default => 'NA' ); my $Test; my $encryption; my $Mode; my $Filer_names; my %FILER_STATE; my @Nodes; my @Nodes_all; my %nodes_filer; my %FILER_enc_val; my %FILER_API; my ( $email_subject, $email_body ); my $test_status = 0; my $filer_env_details = {}; my $TC_name = "CONVERT_SETUP_INSTALL_IMAGE"; $TC_name = "CONVERT_NVMe_SETUP_INSTALL_IMAGE" if ( $NVMe_SETUP =~ /YES/i ); my $Req_Version; ######################################### # Testcase available for execution ######################################### my @Testcases = ( CONVERT_SETUP_INSTALL_IMAGE => "CONVERT_SETUP_INSTALL_IMAGE" ); @Testcases = ( NVME_SETUP_BOOT_IN_REQ => "NVMe Setup and Boot In Required mode" ) if ( $NVMe_SETUP =~ /YES/i ); &main(); sub main { # Debug break point $DB::single = 2; # Create Test Case Driver object $Test = new TCD( -testcases => [@Testcases] ); if ( $Test->error ) { $Test->nlog( $Test->errmsg ); $test_status = 1; $email_subject = "$TC_name : Test FAILED: FAIL"; $email_body = "Failed to instantiate Test object.\nLog Location : $LOGDIR\n"; send_message( mail_subject => $email_subject, mail_body => $email_body, MAIL_FROM => $MAIL_FROM, MAIL_TO => $MAIL_TO, EOE => $EOE, EMAIL => $EMAIL ); return $TCD::FAIL; } # Performs method callbacks $Test->run_test(); if ( $Test->error ) { $Test->nlog( $Test->errmsg ); return $TCD::FAIL; } exit(0); } ## end sub main ########## INIT ################################################### # This init subroutine will initialise the filer. #################################################################### sub init() { $Test->description("########## Finding Nodes ######### Initialising the Filer"); $Filer_names = $Test->get_param("FILER"); logcomment("Filers present are $Filer_names"); pnate_versions_display(); my @Nodes_temp = NACL::C::Node->find(); foreach my $Node (@Nodes_temp) { my $FILER_C = $Node->name(); $nodes_filer{$FILER_C} = [$Node]; } @Nodes = map { @{$_} } values(%nodes_filer); @Nodes_all = @Nodes; logcomment( "Test $TC_name starting: " . scalar( localtime() ) ); if ( ( scalar @Nodes < 2 ) && ( $CONVERT_TO eq 'CMode' ) ) { logcomment("**FATAL** - To convert to CMODE atleast two Nodes are required"); logresult( 'FATAL', "To convert to CMODE atleast two Nodes are required" ); } my @FILER_ARR = split( /,/, $Filer_names ); check_filer_connection( Filer_name => \@FILER_ARR ); my %original_state; foreach (@Nodes) { my $FILER = $_->name(); my $Host = host($FILER); my $transit_obj = NACL::Transit->new( name => $Host ); my $filer_state = $transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); $original_state{$FILER} = $filer_state; logcomment("DEBUG : Original filer state : $filer_state"); if ( $filer_state !~ /CLI|MAINT|FIRMWARE|UP|LOADER/ ) { logcomment("FILER : $FILER is not in CLI | UP | MAINTENANCE | LOADER | FIRMWARE state"); logcomment("**NO_RUN** : Check Filer State : $FILER is not in CLI | UP | MAINTENANCE | LOADER | FIRMWARE state"); logresult( 'FATAL', "$FILER is not in CLI | UP | MAINTENANCE | LOADER | FIRMWARE state" ); return $TCD::FAIL; } if ( $filer_state =~ /CLI|UP/ ) { logcomment(" $FILER is in CLI, Set Privilage to test"); logcomment("C-LOGIN CONNECTION "); my $API_Set_Obj = $_->get_7m_or_nodescope_apiset(); $API_Set_Obj->set_timeout( "connectrec-timeout" => 7200 ); my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ]; $API_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers ); logcomment("CONSOLE CONNECTION "); $API_Set_Obj->execute_raw_command( 'command' => "\013" ); my $API_Set_Obj_CC = $_->get_7m_or_nodescope_apiset( connid => "console" ); $API_Set_Obj_CC->set_timeout( "connectrec-timeout" => 7200 ); my $prompts_answers_CC = [ ".*Do you want to continue.*" => 'y' ]; $API_Set_Obj_CC->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers_CC ); logcomment("Set test in run local mode"); $API_Set_Obj_CC->execute_raw_command( 'command' => "run local priv set -q test" ); logcomment("Privilage level set to test in all modes"); } } if ( $NVMe_SETUP =~ /NO/i ) { foreach (@Nodes) { my $FILER = $_->name(); $Mode = $_->mode(); logcomment("Filer : $FILER - Filer Mode found from the configuration file is $Mode"); logcomment("Filer - $FILER : Checking the Mode of filer before proceeding"); handle_prompts( Node => $_ ); my $Transit_obj = NACL::Transit->new( name => $FILER ); try { $FILER_STATE{$FILER} = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); ##capturing the filer state and booting to the same state if necessary in the clean-up } catch NACL::Transit::Exceptions::Timeout with { my $Host = host($FILER); my $prompt_array = $Host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_fw = "($prompts)"; logcomment("This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); #set the execution server to the current node my @filer_working = $Server->servers(); logcomment("Number of filers working are @filer_working"); try { $Server->console( cmd => " ", additional_prompts_timeout => 600, ignore => $ignore_strings, additional_prompts => { ".*::>.*" => "FATAL_PROMPT", "$prompt_fw" => "FINAL_PROMPT", } ); } catch NATE::BaseException with { $Server->console( cmd => "run local halt -f -t 0", additional_prompts_timeout => 600, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); }; $FILER_STATE{$FILER} = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); }; #$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ my $prompt_fw; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $Host = host($FILER); my $prompt_array = $Host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); #set the execution server to the current node my @filer_working = $Server->servers(); logcomment("Number of filers working are @filer_working"); #$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ logcomment("Filer : $FILER : Current state of the filer is $FILER_STATE{$FILER}"); ##If configuration file is in 7Mode and filer state is in CMode CLI the halt -f -t 0 command will not work so executing like below if ( ( $Mode =~ /7Mode/i ) && ( $FILER_STATE{$FILER} =~ /^CLI$/ ) ) { logcomment("Filer - $FILER : halting the filer"); $Server->console( cmd => "run local halt -f -t 0", additional_prompts_timeout => 600, ignore => $ignore_strings, additional_prompts => { ".*not found.*" => "FINAL_PROMPT", "$prompt_fw" => "FINAL_PROMPT", } ); $Transit_obj->change_state( to => "FIRMWARE" ); ##if filer is not CMODE this will work } elsif ( $FILER_STATE{$FILER} =~ /USERNAME/ig ) { ##if filer is in USERNAME mode that means filer is in CMODE, so for safety reading my $Host = host($FILER); my $passwd = $Host->{default_password}; my $username = 'admin'; logcomment("password is $passwd"); try { $Server->console( cmd => "\n", additional_prompts_timeout => 600, ignore => $ignore_strings, additional_prompts => { ".*login.*" => "admin", ".*Password.*" => $passwd, ".*::>.*" => "FINAL_PROMPT", ".*Login incorrect.*" => 'FATAL_PROMPT', "$prompt_fw" => "FINAL_PROMPT", } ); } catch NATE::BaseException with { logcomment("FILER - $FILER : WARNING : Incorrect login, Please check the configuration file "); }; logcomment("Filer : $FILER : Filer is not in LOADER prompt, so changing state to LOADER"); logcomment("Filer - $FILER : halting the filer"); $Server->console( cmd => "run local halt -f -t 0", additional_prompts_timeout => 600, ignore => $ignore_strings, additional_prompts => { ".*not found.*" => "FINAL_PROMPT", "$prompt_fw" => "FINAL_PROMPT", } ); } elsif ( $FILER_STATE{$FILER} !~ /FIRMWARE/ig ) { logcomment("Filer : $FILER - changing state to FIRMWARE"); $Transit_obj->change_state( to => "FIRMWARE" ); } else { logcomment("Filer : $FILER : Filer is already in FIRMWARE state"); } $FILER_STATE{$FILER} = 'CLI' if ( $FILER_STATE{$FILER} =~ /Password|USERNAME|DBLADECLI/ig ); my $get = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $get !~ /FIRMWARE/ ) { logcomment("**FATAL**: Couldn't change the state of filer to LOADER"); logresult( 'FATAL', "Couldn't change the state of filer to LOADER" ); } my $data = $Server->console( cmd => "printenv", additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); my $actual_state; ($actual_state) = $data =~ /bootarg.init.boot_clustered\s+(\S+)/; if ( $actual_state =~ /true/gi ) { logcomment("Filer - $FILER is in CMode"); } elsif ( $actual_state =~ /false/gi ) { logcomment("Filer - $FILER is in 7Mode"); if ( ( $CONVERT_TO =~ /7Mode/i ) && ( $IMAGE =~ /fullsteam/ig ) ) { logcomment("Kernel Image given by the user is :: $IMAGE"); logcomment("$FILER : Mode of the filer found from bootarg.init.boot_clustered is 7Mode, so Cannot install fullsteam kernel on 7Mode filer"); logcomment("**FATAL** : $FILER : Mode of the filer found from bootarg.init.boot_clustered is 7Mode, so Cannot install fullsteam kernel on 7Mode filer"); logresult( 'FATAL', "Cannot install fullsteam kernel on 7Mode filer" ); } } else { logcomment("Filer - $FILER : Variable bootarg.init.boot_clustered is not not set either true or false"); logcomment("Filer - $FILER : Converting the setup based on the user input i.e $CONVERT_TO"); } } #Burt 1033916 nmamatha my @subtests; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); if ( $original_state{$FILER_C} =~ /LOADER|FIRMWARE/ ) { push( @subtests, subtest( \&check_valid_platform, -runid, "check_valid_platform_$FILER_C", -bg, "--", Node => $Node, Test => $Test ) ); } } Subtest::wait_finish( subtest => [@subtests] ); my @subtests; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); push( @subtests, subtest( \&change_to_maint_check_for_chasis, -runid, "change_to_maint_check_for_chasis_$FILER_C", -bg, "--", $Node ) ); } Subtest::wait_finish( subtest => [@subtests] ); if ( $DISK_SERIAL_NUMBER !~ /^\s*$/ ) { $DISK_SERIAL_NUMBER =~ s/\s*//g; @Serial_numbers = split m/[,]/, $DISK_SERIAL_NUMBER; logcomment( "Filer - $FILER : User provided serial numbers are ", @Serial_numbers ); } } #End of NVME setup No if ( $NVMe_SETUP =~ /YES/i ) { logcomment("INIT : NVMe Code"); if ( $BOOT_STATE =~ /INSTALL_IMAGE/ ) { logcomment("User selected to install kernel , Proceed with installing provided DEBUG and NON-DEBUG kernel"); } } return $TCD::PASS; } ## end sub init() ##################################################################### # setup ##################################################################### sub setup() { $Test->description("Inside Setup Setting the variables in LOADER prompt if required"); if ( $NVMe_SETUP =~ /NO/i ) { foreach my $Node (@Nodes) { my $FILER = $Node->name(); my @required; my $logfile = "$LOGDIR/" . "CONVERT_SETUP_INSTALL_IMAGE/change_to_maint_check_for_chasis_$FILER" . "\.log"; open( FH, "$logfile" ) || logresult( 'FATAL', " Cannot open the file $logfile" ); my @arr1 = ; close(FH); foreach my $line (@arr1) { $line =~ s/\[HDD\S+\]//g; push( @required, $line ); if ( ( $line =~ /Detected encrypting disk / ) && ( $line =~ /(\d+) of (\d+) disks checked/ ) ) { my $drv1 = $1; my $drv2 = $2; if ( $drv1 == $drv2 ) { logcomment("Filer has only Encrypted Drives"); $encryption = "true"; } else { logcomment("**FATAL**: $FILER : MIXED DRIVES ARE PRESENT ON THE FILER ,BOTH Encrypted and Normal Drives "); logresult( 'FATAL', " $FILER : MIXED DRIVES ARE PRESENT ON THE FILER ,BOTH Encrypted and Normal Drives " ); } } } } _set_environment(); ##Setting environment variables based on the mode of the filer my @subtest; if ( $CONVERT_TO =~ /CMode/i ) { logcomment("Check for filer state and change to maintenance to verify drive assignment"); my @state_subtest; foreach (@Nodes) { my $FILER = $_->name(); push( @state_subtest, subtest( \&boot_filer_in_req_state, -runid, "change_state_to_maint_dsk" . "$FILER", -bg, "--", filer_name => $FILER, Test => $Test, change_state_to => 'MAINT', ) ); } Subtest::wait_finish( subtest => [@state_subtest] ); logcomment("Check for drive assignment"); disk_remv_assign_ownership( Node_ref => \@Nodes ); logcomment("Proceed with next steps"); foreach (@Nodes) { my $FILER = $_->name(); logcomment("Filer : $FILER : executing wipe-config on the filer"); push( @subtest, subtest( \&clean_config, -runid, "clean_config_" . "$FILER", -bg, "--", Node => $_, ) ); } Subtest::wait_finish( subtest => [@subtest] ); foreach (@Nodes) { my $FILER = $_->name(); my $filer_con = connect("$FILER#console"); say( $filer_con, "printenv", "LOADER", -timeout, "120", '-ok' ); if ( $FILER_enc_val{$FILER} eq "true" ) { logcomment("Encryption value present for the filer $FILER, So setting again the same value"); my ( $match, $after, $prompt, $before ) = say( $filer_con, "setenv bootarg.storageencryption.support $FILER_enc_val{$FILER}", "LOADER", -timeout, "120", '-ok' ); say( $filer_con, "printenv bootarg.storageencryption.support", "LOADER", -timeout, "120", '-ok' ); say( $filer_con, "printenv ", "LOADER", -timeout, "120", '-ok' ); } } } } #End of NEVE setup NO if ( $NVMe_SETUP =~ /YES/i ) { logcomment("NVME _setup code "); logcomment("Read ENV variables"); foreach my $Node (@Nodes) { my $FILER_Ca = $Node->name(); my $Home = $ENV{HOME}; my $ENV_FIL_LOC = "$Home/NDATE/FILER_INFO/$FILER_Ca"; open( ENV_FL, "$ENV_FIL_LOC" ) || die "Can not open file $ENV_FIL_LOC"; my @env_data = ; logcomment("Get path for filer "); my $FILER_C; my ( $env, $nb ); foreach my $line (@env_data) { chomp($line); if ( $line =~ /FILER=(\S+)/ ) { print "Filer : $1\n"; $FILER_C = $1; $filer_env_details->{$FILER_C} = { 'filer' => $1 }; } if ( $line =~ /PATH=(\S+)/ ) { $env = $1; print "ENV : $env\n"; $filer_env_details->{$FILER_C} = { 'env_p' => $env }; } if ( $line =~ /NETBOOT_IMG=(\S+)/ ) { $nb = $1; print "NB : $nb\n"; } } } logcomment( "Dumper " . Dumper($filer_env_details) ); } return $TCD::PASS; } sub NVME_SETUP_BOOT_IN_REQ { $Test->description("NVMe System Setup or booting in required mode") if ( $BOOT_STATE !~ /INSTALL_IMAGE/ ); $Test->description("NVMe Install Image ") if ( $BOOT_STATE =~ /INSTALL_IMAGE/ ); my $home = $ENV{HOME}; my $boot_flag; my $nvme_install_image = 0; my $Tahiti_Setup = 0; my @tahiti_drv; my @internal_drv; logcomment("Get user parameters "); logcomment("NVMe Setup : $NVMe_SETUP"); logcomment("Change Filer State to : $BOOT_STATE"); logcomment("Environment Variable file location : $ENV_FIL_LOC"); logcomment("Boot filer in Maintenance mode and assign drives"); my @maint_subtest; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Host = host($FILER_C); logcomment("Filer(s) Selected : $FILER_C"); logcomment("Check State of filer $FILER_C"); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state(); logcomment("FILER : $FILER_C state is $filer_state"); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); if ( $filer_state !~ /MAINT/ ) { logcomment("$FILER_C is not in Maintenance mode, Change state to MAINTENANCE"); push( @maint_subtest, subtest( \&nvme_boot_to_maint, -runid, "boot_in_maint_" . "$FILER_C", -bg, "--", Node => $Node, env_var => "NA" ) ); } else { logcomment("Check Filer State "); } } Subtest::wait_finish( subtest => [@maint_subtest] ); logcomment("Filer Changed to Maintenance Mode"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); my $Host = host($FILER_C); my $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); logcomment("Check For Shelf"); my $out = $Api_Set_Obj->execute_command( 'command' => 'sysconfig -v', 'timeout' => 600 ); if ( $out =~ /NS224NS/ ) { logcomment("TAHITI SHELF IS ATTACHED to APOLLO "); $Tahiti_Setup = 1; } else { logcomment("No External Shelf / Tahiti Shelf is attached"); } logcomment("Get Apollo Internal Drives"); $Api_Set_Obj->execute_raw_command( 'command' => "\013" ); my $dsk_out = $Api_Set_Obj->execute_raw_command( 'command' => "disk_list" ); foreach my $line ( split( /\n/, $dsk_out ) ) { if ( $line =~ /(\S+\.\S+)\s+/ ) { my $drv = $1; if ( $drv =~ /\S+\.\S+\.\S+\.\S+/ ) { push( @tahiti_drv, $drv ); } else { push( @internal_drv, $drv ); } } } last; } logcomment("check for Drive assignment"); logcomment("Check for Partition and Remove Partition"); remove_partition( Noderef => [@Nodes], handle => "NVMe" ); logcomment("Removed Partition"); if ( $Tahiti_Setup == 1 ) { logcomment("Internal drives : @internal_drv"); logcomment("Tahiti Drives : @tahiti_drv"); my $slot_staus = tahiti_check_drv_slots( APOLLO_DRVS => \@internal_drv ); logcomment("DEBUG: Drive Slot Status : $slot_staus"); logcomment("Tahiti shelf Assignment policy will be followed"); my $part = 1; tahiti_boot_main_unpart_assign_parts( NODES => \@Nodes, partition => $part ); } else { logcomment("Apollo System Drive assignment policy will be followed"); nvme_drv_assign( Nodes => \@Nodes, force => 1, assign => 'yes' ); } logcomment("Drive Assignment Checked, Proceed with next Steps"); foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Filer(s) Selected : $FILER_C"); logcomment("Check State of filer $FILER_C"); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); my $filer_state = $Transit_obj->get_state(); logcomment("FILER : $FILER_C state is $filer_state"); if ( ( $filer_state =~ /MAINT/ ) && ( $BOOT_STATE =~ /MAINT|MAINTENANCE/ ) ) { logcomment("Filer :$FILER_C is in required state : $BOOT_STATE"); $boot_flag = "DONE"; } elsif ( ( $filer_state =~ /CLI|UP/ ) && ( $BOOT_STATE =~ /FILE_SYSTEM|UP|FS/ ) ) { logcomment("Filer :$FILER_C is in required state : $BOOT_STATE"); $boot_flag = "FS"; } else { logcomment("Filer is either in FILE_SYSTEM or MAINTENANCE MODE"); logcomment("Booting Filer in User selected mode : $BOOT_STATE"); if ( $BOOT_STATE =~ /MAINTENANCE/ ) { my $env_data; foreach my $fil ( keys %$filer_env_details ) { logcomment("Get Env path to be set for filer $fil"); if ( $fil =~ /$FILER_C/ ) { $env_data = $filer_env_details->{$fil}->{'env_p'}; logcomment("Env variable to be used : $env_data"); } } logcomment("Halt Filer : $FILER_C"); nvme_halt_Node( Node => $Node ); logcomment("Filer is in LOADER"); logcomment("FILER : $FILER_C - DEBUG PASS - $env_data and \n $Node"); nvme_set_env_var( Node => $Node, env_path => $env_data ); $boot_flag = "MA"; } elsif ( $BOOT_STATE =~ /FILE_SYSTEM/ ) { logcomment("Halt Filer "); nvme_halt_Node( Node => $Node ); $boot_flag = "FS"; } elsif ( $BOOT_STATE =~ /INSTALL_IMAGE/ ) { logcomment("Install Image on filers"); $nvme_install_image = 1; } else { logcomment("User Has not specified any state or NDATE couldn't get user requested state"); logcomment("**FATAL** : $FILER User Has not specified any state or NDATE couldn't get user requested state"); logresult( "FATAL", " $FILER User Has not specified any state or NDATE couldn't get user requested state" ); $test_status = 1; } } } if ( $nvme_install_image == 1 ) { my $home = $ENV{HOME}; my @image_ary; my $debug_file_path = `cat $home/NDATE/FILER_INFO/DEBUG`; chomp($debug_file_path); push( @image_ary, $debug_file_path ); my $non_debug_file_path = `cat $home/NDATE/FILER_INFO/NON_DEBUG`; chomp($non_debug_file_path); push( @image_ary, $non_debug_file_path ); foreach my $img (@image_ary) { print "$img\n"; logcomment("Installing - $img image "); my @subtests; foreach my $nod (@Nodes) { my $FILER = $nod->name(); logcomment("Filer : $FILER installing new Image - $img "); push( @subtests, subtest( \&compact_flash, -runid, "Install_Image_" . "$FILER", -bg, "--", Node => $nod, tar_file => $img, ) ); } Subtest::wait_finish( subtest => [@subtests] ); logcomment("Installing image1 completed, Installing second image on both filers"); } logcomment("Installing Image completed "); logcomment("TO-DO : New-Burt to auto select user selected kernel"); logcomment("Boot in mainteanance mode to verify installed image"); logcomment("User selected or required image on filer is $IMAGE"); my @maint_subtest; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Check filer :$FILER_C"); my $env_data; foreach my $fil ( keys %$filer_env_details ) { logcomment("Get Env path to be set for filer $fil"); if ( $fil =~ /$FILER_C/ ) { $env_data = $filer_env_details->{$fil}->{'env_p'}; logcomment("Env variable to be used : $env_data"); } } push( @maint_subtest, subtest( \&nvme_boot_to_maint, -runid, "boot_in_maint_" . "$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) ); } Subtest::wait_finish( subtest => [@maint_subtest] ); logcomment("Booting process completed, Filer must be in maintenance mode"); logcomment("User Selected $IMAGE kernel to be loaded as primary image"); } else { logcomment("FILER : $FILER Basic Check completed, Proceeding with booting. "); my @maint_subtest; foreach my $Node (@Nodes) { my $FILER_C = $Node->name(); logcomment("Check filer :$FILER_C"); my $env_data; foreach my $fil ( keys %$filer_env_details ) { logcomment("Get Env path to be set for filer $fil"); if ( $fil =~ /$FILER_C/ ) { $env_data = $filer_env_details->{$fil}->{'env_p'}; logcomment("Env variable to be used : $env_data"); } } $env_data = "NA" if ( $env_data == undef ); if ( $boot_flag =~ /FS/ ) { push( @maint_subtest, subtest( \&nvme_clean_config, -runid, "clean_config_" . "$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) ); } elsif ( $boot_flag =~ /MA/ ) { push( @maint_subtest, subtest( \&nvme_boot_to_maint, -runid, "boot_in_maint_" . "$FILER_C", -bg, "--", Node => $Node, env_var => $env_data ) ); } elsif ( $boot_flag =~ /DONE/ ) { logcomment("Filer :$FILER_C is in user requested state"); } else { logcomment("Cannot capture required boot state"); $test_status = 1; } } Subtest::wait_finish( subtest => [@maint_subtest] ); logcomment("Booting process completed"); logcomment("Create cluater if user selected to boot in FILE SYSTEM mode"); if ( $boot_flag =~ /FS/ ) { create_join_cluster( node_present => [@Nodes], nvme_setup => "yes" ); } logcomment("NVMe Setup compelted"); } if ( $test_status == 1 ) { return $TCD::FAIL; } else { return $TCD::PASS; } } ##################################################################### # Calling sub CONVERT_SETUP_INSTALL_IMAGE ##################################################################### sub CONVERT_SETUP_INSTALL_IMAGE { #$Test->description(" Installing Image if required"); if ( $IMAGE ne 'N' ) { $Test->description("Installing Image on the filers"); { my $home = $ENV{HOME}; my @image_ary; my $debug_file_path = `cat $home/NDATE/FILER_INFO/DEBUG`; chomp($debug_file_path); push( @image_ary, $debug_file_path ); my $non_debug_file_path = `cat $home/NDATE/FILER_INFO/NON_DEBUG`; chomp($non_debug_file_path); push( @image_ary, $non_debug_file_path ); foreach my $img (@image_ary) { print "$img\n"; logcomment("Installing - $img image "); my @subtests; foreach my $nod (@Nodes) { my $FILER = $nod->name(); logcomment("Filer : $FILER installing new Image - $img "); push( @subtests, subtest( \&compact_flash, -runid, "Install_Image_" . "$FILER", -bg, "--", Node => $nod, tar_file => $img, ) ); } Subtest::wait_finish( subtest => [@subtests] ); logcomment("Installing image1 completed, Installing second image on both filers"); } } foreach my $Node (@Nodes) { my ($secondary_var); my $loop = 0; recheck: my $FILER = $Node->name(); my $Host = host($FILER); logcomment("Filer - $FILER : Booting the filer in MAINTENANCE "); my $Transit_obj = NACL::Transit->new( name => $FILER ); $Transit_obj->change_state( to => "MAINT" ); sleep(120); my $node_version; my $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $API_Object->set_timeout( "connectrec-timeout" => 1200 ); $node_version = $API_Object->execute_raw_command( command => "version" ); my $K_type_filer; if ( $IMAGE =~ /nondebug/ ) { $Req_Version = "NON_DEBUG"; } elsif ( $IMAGE =~ /debug/ ) { $Req_Version = "DEBUG"; } if ( $node_version =~ /<1d>/i ) { logcomment("Kernel type - DEBUG kernel"); $K_type_filer = "DEBUG"; } else { logcomment("Kernel type - NON-DEBUG kernel"); $K_type_filer = "NON_DEBUG"; } if ( $Req_Version eq $K_type_filer ) { logcomment("$FILER : filer has Required Version $Req_Version "); } elsif ( $loop == 0 ) { $loop++; $secondary_var = switch_img_n_check_ver( $Node, $Req_Version ); goto recheck; # logcomment("$FILER : filer has Required Version $Req_Version "); } elsif ( ( $loop > 0 ) && ( $loop < 2 ) && ( $Req_Version ne $K_type_filer ) ) { my $ret = re_set_n_check_image( $Node, $secondary_var ); goto recheck; } elsif ( ( $loop >= 2 ) && ( $Req_Version ne $K_type_filer ) ) { logcomment("**FATAL** : $FILER: USER SELECTED VERSION $Req_Version IS NOT AVAILABLE ON BOTH PRIMARY AND SECONDARY IMAGES , So Exiting From Test"); logresult( 'FATAL', "$FILER: USER SELECTED VERSION $Req_Version IS NOT AVAILABLE ON BOTH PRIMARY AND SECONDARY IMAGES , So Exiting From Test" ); } } } else { $Test->description("Not installing Image on filers, user selected not to install Image"); logcomment("Not installing Image on filers, user selected not to install Image"); } ######################################################################################### if ( ( $IMAGE ne 'N' ) && ( $CONVERT_TO eq 'N' ) ) { ## if only IMAGE is installed then booting the filer to MAINT state foreach (@Nodes) { $FILER = $_->name(); logcomment("Filer - $FILER : Booting the filer in MAINTENANCE "); my $Transit_obj = NACL::Transit->new( name => $FILER ); $Transit_obj->change_state( to => "MAINT" ); logcomment("Booted.."); sleep 10; } foreach (@Nodes) { $FILER = $_->name(); logcomment("Checking for root aggregate on $FILER"); destroy_root_volume( Noderef => [$_] ); } if ( $WIPE_LBAS !~ /WIPE_LBAS/i ) { goto end; } } else { foreach (@Nodes) { my $FILER = $_->name(); logcomment("Filer : $FILER - Changing state to MAINTENANCE"); my $Transit_obj = NACL::Transit->new( name => $FILER ); my $Api_Set_Obj; try { $Transit_obj->change_state( to => "MAINT" ); my $Host = host($FILER); $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); } catch NACL::Transit::Exceptions::RetriesExceeded with { logcomment("Expected: Caught a RetriesExceeded exception"); } catch NACL::Transit::Exceptions::TransitException with { my $exception_object = shift; logcomment( "Exception is of type: " . ref($exception_object) ); logcomment( "Error text is: " . $exception_object->text() ); } finally { logcomment("Reset fields 'timeout' and 'maxcycle'"); $Transit_obj->set( 'timeout' => 700, maxcycle => 30 ); $Transit_obj->set( 'change_state_timeout' => 600 ); logcomment( "New object settings for timeout and maxcycle are: " . Dumper $Transit_obj->get( 'timeout', 'maxcycle' ) ); $Transit_obj->change_state( to => "MAINT" ); }; } foreach (@Nodes) { my $FILER = $_->name(); my $Host = host($FILER); my $Api_Set_Obj; $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); logcomment("Checking for root aggregate on $FILER"); destroy_root_volume( Noderef => [$_] ); $Api_Set_Obj->execute_raw_command( command => 'version' ); } } if ( $CONVERT_TO =~ /CMode/i || ( $WIPE_LBAS =~ /WIPE_LBAS/i ) ) { if ($ADD_TEST_CASE) { logcomment("Filer will boot in MAINTENANCE mode, because user selected to add test cases, So test-case will boot the filer in required mode"); } elsif ( $MODE =~ /FILE/ ) { setup_cmode_filers( node_present => \@Nodes, Test => $Test, delete_mini_disk => 'NO' ); foreach my $node (@Nodes) { spare_count_cmode( Node => $node, waiting => 'yes' ); } } else { logcomment("Filer will boot in MAINTENANCE mode based on user choice"); } if ( $WIPE_LBAS =~ /WIPE_LBAS/i ) { my @disks_with_serial = (); my @disks_serial_nums = (); my @disk_blocks = (); my @unique_blocks = (); my @pass_drives = (); my @failed_drives = (); my @last_sectors = (); my @real_last_sectors = (); my @fs_starts = (); my @disks = (); my %hash; my $counter = 0; my $disk_not_owned = 0; # Checking whether the nodes own disks or not.If nodes do not own any disks then assigning disks equally foreach (@Nodes) { @disks = (); my $FILER = $_->name(); my $console = Tharn::connect("$FILER#console"); # Set the properties for the Connectrec used for creating the APIset $console->{prompts} = [qr/\*>/]; $console->{max_idle} = 5; $console->{on_idle} = sub { $console->execute(""); }; my $API_obj = NACL::APISet->new( connection => $console, category => "Node", interface => "CLI", set => "Maintenance" ); # Execute disk show to get disks attached to the node logcomment("$FILER : Execute disk show to get disks attached to the node $FILER"); my $disk_show_o = $API_obj->execute_raw_command( 'command' => "disk show -o $FILER" ); my @disk_lines = split( /\n/, $disk_show_o ); foreach my $line (@disk_lines) { if ( $line !~ /DISK|OWNER|POOL|SERIAL NUMBER|HOME/ ) { if ( $line =~ /(\w+\.\w+\.\w+)\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+/ ) { my $data = $1; push( @disks, $data ); } } } # Taking only container drives if setup is partitioned my @disk_array_1 = (); grep { if ( $_ !~ /p\d+/i ) { push( @disk_array_1, $_ ) } } @disks; @disks = @disk_array_1; if ( scalar @disks == 0 ) { $disk_not_owned = 1; } } if ($disk_not_owned) { disk_remv_assign_ownership( Node_ref => [@Nodes] ); } foreach (@Nodes) { @fs_starts = (); @real_last_sectors = (); @last_sectors = (); @disks_with_serial = (); @disks = (); my $FILER = $_->name(); my $console = Tharn::connect("$FILER#console"); # Set the properties for the Connectrec used for creating the APIset $console->{prompts} = [qr/\*>/]; $console->{max_idle} = 5; $console->{on_idle} = sub { $console->execute(""); }; my $API_obj = NACL::APISet->new( connection => $console, category => "Node", interface => "CLI", set => "Maintenance" ); # Perform disk_list in Maintenace Mode get block number and serial number my $disk_list_data = disk_list_data( API_Obj => $API_obj ); # Execute disk show to get disks attached to the specific node logcomment("$FILER : Execute disk show to get disks attached to the specific node"); my $disk_show_output = $API_obj->execute_raw_command( 'command' => "disk show -o $FILER" ); my @disk_lines = split( /\n/, $disk_show_output ); foreach my $line (@disk_lines) { if ( $line !~ /DISK|OWNER|POOL|SERIAL NUMBER|HOME/ ) { if ( $line =~ /(\w+\.\w+\.\w+)\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+/ ) { my $data = $1; push( @disks, $data ); } } } # Taking only container drives if setup is partitioned my @disk_array_1 = (); grep { if ( $_ !~ /p\d+/i ) { push( @disk_array_1, $_ ) } } @disks; @disks = @disk_array_1; if ( scalar @disks == 0 ) { logcomment("**FAIL**: $FILER: DISK REMOVE ASSIGN IS NOT ASSIGNED DISKS PROPERLY"); return $TCD::FAIL; } else { logcomment( "$FILER : Drives present in the filer are " . scalar @disks . " and the drives are @disks" ); } # Group all drive that have user provided serial number in group if ( $DISK_SERIAL_NUMBER !~ /^\s*$/ ) { foreach (@disks) { if ( $disk_list_data->{$_}->{SERIAL} =~ /$Serial_numbers[0]/ ) { push( @disks_with_serial, $_ ); } elsif ( $disk_list_data->{$_}->{SERIAL} =~ /$Serial_numbers[1]/ ) { push( @disks_with_serial, $_ ); } } if ( scalar @disks_with_serial == 0 ) { logcomment("**FAIL**: $FILER: THERE ARE NO DISKS WITH SERIAL NUMBER $Serial_numbers[$counter] PROVIDED BY USER"); return $TCD::FAIL; } else { logcomment( "$FILER : Drives with user provided serial number are : ", @disks_with_serial ); } # Perform scsi diskinfo on user provided serial numbered disks logcomment("$FILER : Performing scsi diskinfo on user provided serial numbered disks to get last_sector, real_last_sector and FS START"); foreach my $dsk (@disks_with_serial) { my $scsi_disk_info_s2 = my_scsi_diskinfo_disk( API_Obj => $API_obj, disk_list => [$dsk] ); my $last_sector = $scsi_disk_info_s2->{$dsk}->{'last_sector'}; push( @last_sectors, $last_sector ); my $real_last_sector = $scsi_disk_info_s2->{$dsk}->{'real_last_sector'}; push( @real_last_sectors, $real_last_sector ); my $fs_start = $scsi_disk_info_s2->{$dsk}->{'FS START'}; push( @fs_starts, $fs_start ); } # Execute the disk_qual command for the target disks logcomment("$FILER : Execute the disk_qual command for the target disks"); my $it = each_array( @fs_starts, @disks_with_serial ); while ( my ( $fstart, $dsk ) = $it->() ) { my $raw_output = $API_obj->execute_raw_command( command => "disk_qual start -t 5 -n 256 -f 0 -l $fstart -p 1 -d $dsk -o noverify" ); @pass_drives = (); @failed_drives = (); # Check status of disk_qual my $kill_time = 36000; my $time_out_temp = 0; my $timeout_flag = 0; $API_obj->set_timeout( "connectrec-timeout" => ( $kill_time + 600 ) ); while ( $kill_time > $time_out_temp ) { my ( $s, $m, $h ) = gmtime($kill_time); my ( $s1, $m1, $h1 ) = gmtime($time_out_temp); logcomment("$FILER : Total Wait time: $h HRS $m MINS $s SECS and Elapsed Time: $h1 HRS $m1 MINS $s1 SECS"); $API_obj->set_timeout( "connectrec-timeout" => ( $kill_time + 600 ) ); my $status = $API_obj->execute_raw_command( 'command' => "disk_qual status" ); if ( $status =~ /idle/ ) { logcomment("$FILER : Disk_qual is completed"); $timeout_flag = 1; last; } elsif ( $status =~ /running/ ) { logcomment("$FILER : disk_qual is still in Progress"); sleep 120; $time_out_temp += 120; } } if ( $timeout_flag == 1 ) { logcomment("$FILER : Disk_qual is completed on the drives, Now Checking the disk_qual report to find the result of disk_qual command"); my $total_data = $API_obj->execute_raw_command( 'command' => "disk_qual report -d $dsk" ); if ( $total_data =~ /Passed/i ) { push( @pass_drives, $dsk ); } elsif ( $total_data =~ /Failed/i ) { push( @failed_drives, $dsk ); } } # Report if disk_qual is failed if ( (@failed_drives) && (@pass_drives) ) { logcomment( "**FAIL**: $FILER: DISK_QUAL PASSED FOR " . scalar @pass_drives . " DRIVES AND DRIVES ARE : @pass_drives" . " AND FAILED FOR " . scalar @failed_drives . " DRIVES AND DRIVES ARE : @failed_drives" ); return $TCD::FAIL; } elsif ( (@pass_drives) && ( !@failed_drives ) ) { logcomment( "$FILER : disk_qual passed for all " . scalar @pass_drives . " drives and drives are : @pass_drives" ); } else { logcomment( "**FAIL**: $FILER: DISK_QUAL FAILED ON ALL " . scalar @failed_drives . " DRIVES AND DRIVES DRIVES: @failed_drives" ); return $TCD::FAIL; } } # Wipe Out of Lower range LBAs and the Higher Range LBAs logcomment("$FILER : Wipe Out the Lower range LBAs and the Higher Range LBAs"); $it = each_array( @last_sectors, @real_last_sectors, @disks_with_serial ); while ( my ( $last_sector, $real_last_sector, $dsk ) = $it->() ) { my $raw_output = $API_obj->execute_raw_command( command => "disk_qual start -t 5 -n 256 -f $last_sector -l $real_last_sector -p 1 -d $dsk -o noverify" ); @pass_drives = (); @failed_drives = (); # Check status of disk_qual my $kill_time = 36000; my $time_out_temp = 0; my $timeout_flag = 0; $API_obj->set_timeout( "connectrec-timeout" => ( $kill_time + 600 ) ); while ( $kill_time > $time_out_temp ) { my ( $s, $m, $h ) = gmtime($kill_time); my ( $s1, $m1, $h1 ) = gmtime($time_out_temp); logcomment("$FILER : Total Wait time: $h HRS $m MINS $s SECS and Elapsed Time: $h1 HRS $m1 MINS $s1 SECS"); $API_obj->set_timeout( "connectrec-timeout" => ( $kill_time + 600 ) ); my $status = $API_obj->execute_raw_command( 'command' => "disk_qual status" ); if ( $status =~ /idle/ ) { logcomment("$FILER : Disk_qual is completed"); $timeout_flag = 1; last; } elsif ( $status =~ /running/ ) { logcomment("$FILER : Disk_qual is still in Progress"); sleep 120; $time_out_temp += 120; } } if ( $timeout_flag == 1 ) { logcomment("$FILER : Disk_qual is completed on the drives, Now Checking the disk_qual report to find the result of disk_qual command"); my $total_data = $API_obj->execute_raw_command( 'command' => "disk_qual report -d $dsk" ); if ( $total_data =~ /Passed/i ) { push( @pass_drives, $dsk ); } elsif ( $total_data =~ /Failed/i ) { push( @failed_drives, $dsk ); } } # Report if disk_qual is failed if ( (@failed_drives) && (@pass_drives) ) { logcomment( "**FAIL**: $FILER: LBA WIPEOUT PASSED FOR " . scalar @pass_drives . " DRIVES AND DRIVES ARE : @pass_drives" . " AND FAILED FOR " . scalar @failed_drives . "AND DRIVES ARE : @failed_drives" ); return $TCD::FAIL; } elsif ( (@pass_drives) && ( !@failed_drives ) ) { logcomment( "Lba wipeout passed for all " . scalar @pass_drives . " drives and drives are : @pass_drives" ); } else { logcomment( "**FAIL**: $FILER: LBA WIPEOUT FAILED FOR ALL DRIVES " . scalar @failed_drives . " AND DRIVES DRIVES: @failed_drives" ); return $TCD::FAIL; } } $counter++; logcomment("$FILER : Wipe Out of Lower range LBAs and the Higher Range LBAs completed"); } else { @disks_with_serial = (); @disk_blocks = (); @unique_blocks = (); # Group all drive that have common blocks count in to individual group logcomment("$FILER : Group all drives that have common blocks count in to individual group"); foreach (@disks) { push( @disk_blocks, $disk_list_data->{$_}->{BLOCKS} ); } @unique_blocks = unique(@disk_blocks); logcomment("$FILER : Block Numbers of groupped disks are : @unique_blocks"); @hash{@unique_blocks} = (); foreach my $dsk (@disks) { foreach my $block (@unique_blocks) { if ( $disk_list_data->{$dsk}->{BLOCKS} eq $block ) { push( @{ $hash{$block} }, $dsk ); } } push( @disk_blocks, $disk_list_data->{$_}->{BLOCKS} ); } # Perform scsi diskinfo on one drive for each group logcomment("$FILER : Perform scsi diskinfo on one drive for each group"); my @my_scsi_diskinfo_disk = (); foreach my $blk (@unique_blocks) { logcomment( "$FILER : Unique block is $blk and disks with this block : ", @{ $hash{$blk} } ); my $scsi_disk_info_s2 = my_scsi_diskinfo_disk( API_Obj => $API_obj, disk_list => [ $hash{$blk}[0] ] ); my $last_sector = $scsi_disk_info_s2->{ $hash{$blk}[0] }->{'last_sector'}; my $real_last_sector = $scsi_disk_info_s2->{ $hash{$blk}[0] }->{'real_last_sector'}; my $fs_start = $scsi_disk_info_s2->{ $hash{$blk}[0] }->{'FS START'}; my @disks = @{ $hash{$blk} }; # Execute the disk_qual command for all disks logcomment("$FILER : Execute the disk_qual command for disks : @disks"); foreach my $dsk (@disks) { my $raw_output = $API_obj->execute_raw_command( command => "disk_qual start -t 5 -n 256 -f 0 -l $fs_start -p 1 -d $dsk -o noverify" ); } @pass_drives = (); @failed_drives = (); # Check status of disk_qual my $kill_time = 36000; my $time_out_temp = 0; my $timeout_flag = 0; $API_obj->set_timeout( "connectrec-timeout" => ( $kill_time + 600 ) ); while ( $kill_time > $time_out_temp ) { my ( $s, $m, $h ) = gmtime($kill_time); my ( $s1, $m1, $h1 ) = gmtime($time_out_temp); logcomment("Total Wait time: $h HRS $m MINS $s SECS and Elapsed Time: $h1 HRS $m1 MINS $s1 SECS"); $API_obj->set_timeout( "connectrec-timeout" => ( $kill_time + 600 ) ); my $status = $API_obj->execute_raw_command( 'command' => "disk_qual status" ); if ( $status =~ /idle/ ) { logcomment("$FILER : Disk_qual is completed"); $timeout_flag = 1; last; } elsif ( $status =~ /running/ ) { logcomment("disk_qual is still in Progress"); sleep 120; $time_out_temp += 120; $timeout_flag = 0; } } if ( $timeout_flag == 1 ) { logcomment("$FILER : Disk_qual is completed on the drives, Now Checking the disk_qual report to find the result of disk_qual command"); foreach my $dsk (@disks) { my $total_data = $API_obj->execute_raw_command( 'command' => "disk_qual report -d $dsk" ); if ( $total_data =~ /Passed/i ) { push( @pass_drives, $dsk ); } elsif ( $total_data =~ /Failed/i ) { push( @failed_drives, $dsk ); } } } else { logcomment("**FAIL**: $FILER: DISK_QUAL NOT COMPLETED WITHIN THE GIVEN TIME"); return $TCD::FAIL; } # Report if disk_qual is failed if ( (@failed_drives) && (@pass_drives) ) { logcomment( "**FAIL**: $FILER: DISK_QUAL PASSED FOR " . scalar @pass_drives . " DRIVES AND DRIVES ARE : @pass_drives" . " AND FAILED FOR " . scalar @failed_drives . " DRIVES AND DRIVES ARE : @failed_drives" ); return $TCD::FAIL; } elsif ( (@pass_drives) && ( !@failed_drives ) ) { logcomment( "disk_qual passed for all " . scalar @pass_drives . " drives and drives are : @pass_drives" ); } else { logcomment( "**FAIL**: $FILER: DISK_QUAL FAILED ON ALL " . scalar @failed_drives . " DRIVES AND DRIVES DRIVES: @failed_drives" ); return $TCD::FAIL; } # Wipe Out of Lower range LBAs and the Higher Range LBAs for all disks logcomment("$FILER : Wipe Out the Lower range LBAs and the Higher Range LBAs for all disks"); foreach my $dsk (@disks) { my $raw_output = $API_obj->execute_raw_command( command => "disk_qual start -t 5 -n 256 -f $last_sector -l $real_last_sector -p 1 -d $dsk -o noverify" ); } @pass_drives = (); @failed_drives = (); # Check status of disk_qual $kill_time = 36000; $time_out_temp = 0; $timeout_flag = 0; $API_obj->set_timeout( "connectrec-timeout" => ( $kill_time + 600 ) ); while ( $kill_time > $time_out_temp ) { my ( $s, $m, $h ) = gmtime($kill_time); my ( $s1, $m1, $h1 ) = gmtime($time_out_temp); logcomment("Total Wait time: $h HRS $m MINS $s SECS and Elapsed Time: $h1 HRS $m1 MINS $s1 SECS"); $API_obj->set_timeout( "connectrec-timeout" => ( $kill_time + 600 ) ); my $status = $API_obj->execute_raw_command( 'command' => "disk_qual status" ); if ( $status =~ /idle/ ) { logcomment("$FILER : Disk_qual is completed"); $timeout_flag = 1; last; } elsif ( $status =~ /running/ ) { logcomment("Disk_qual is still in Progress"); sleep 120; $time_out_temp += 120; $timeout_flag = 0; } } if ( $timeout_flag == 1 ) { logcomment("$FILER : Disk_qual is completed on the drives, Now Checking the disk_qual report to find the result of disk_qual command"); foreach my $dsk (@disks) { my $total_data = $API_obj->execute_raw_command( 'command' => "disk_qual report -d $dsk" ); if ( $total_data =~ /Passed/i ) { push( @pass_drives, $dsk ); } elsif ( $total_data =~ /Failed/i ) { push( @failed_drives, $dsk ); } } } else { logcomment("**FAIL**: $FILER: DISK_QUAL NOT COMPLETED WITHIN THE GIVEN TIME"); return $TCD::FAIL; } # Report if disk_qual is failed if ( (@failed_drives) && (@pass_drives) ) { logcomment( "**FAIL**: $FILER: LBA WIPEOUT PASSED FOR " . scalar @pass_drives . " DRIVES AND DRIVES ARE : @pass_drives" . " AND FAILED FOR " . scalar @failed_drives . "AND DRIVES ARE : @failed_drives" ); return $TCD::FAIL; } elsif ( (@pass_drives) && ( !@failed_drives ) ) { logcomment( "Lba wipeout passed for all " . scalar @pass_drives . " drives and drives are : @pass_drives" ); } else { logcomment( "**FAIL**: $FILER: LBA WIPEOUT FAILED FOR ALL DRIVES " . scalar @failed_drives . " AND DRIVES DRIVES: @failed_drives" ); return $TCD::FAIL; } } } logcomment("LBAs Wipe out completed"); } # Assigning disks after LBA's Wipeout logcomment("Assigning disks after LBA's Wipeout"); my $unowned = disk_show_n( Node => $Nodes[0] ); my @unowned_drives = keys %$unowned; if ( scalar @unowned_drives ) { logcomment("Drives @unowned_drives are left unowned."); disk_remv_assign_ownership( Node_ref => [ $Nodes[0] ] ); } else { logcomment("No drives are left unowned"); } } } else { logcomment("Not Booting the filers in 4"); } ############################################################################################ end: if ( $test_status == 0 ) { return $TCD::PASS; } else { return $TCD::FAIL; } } ##################################################################### # Cleanup ##################################################################### sub cleanup() { $Test->description("Cleaning"); ####################################################### ##Copying SUBTEST logs to main directory if present return $TCD::PASS; } ############################################################### ## Setting all the Environment variables to convert the Set-up ############################################################### sub _set_environment { if ( $CONVERT_TO =~ /CMode/ ) { foreach (@Nodes) { my $FILER = $_->name(); logcomment("Setting the environment variables based on the mode"); my ( $variables_set_1, $variables_set_2, $variables_set_3, $variables_set_4 ); logcomment("Filer - $FILER : Converting the filer to $CONVERT_TO"); my $Transit_obj = NACL::Transit->new( name => $FILER ); $Transit_obj->set( 'timeout' => 600, maxcycle => 5 ); $Transit_obj->change_state( to => "FIRMWARE" ); my $filer_con = connect("$FILER#console"); my ( $match, $after, $prompt, $before ) = say( $filer_con, "printenv", "LOADER", -timeout, "120", '-ok' ); my ( $match, $after, $prompt, $before ) = say( $filer_con, "printenv bootarg.storageencryption.support", "LOADER", -timeout, "120", '-ok' ); # logcomment("match $match , after $after , prompt $prompt, before $before"); if ( $encryption eq "true" ) { logcomment("Value of storage encrytion is true"); $FILER_enc_val{$FILER} = "true"; } elsif ( ( $match =~ /bootarg\.storageencryption\.support\s+(true|false)/ig ) || ( $after =~ /bootarg\.storageencryption\.support\s+(true|false)/ig ) || ( $prompt =~ /bootarg\.storageencryption\.support\s+(true|false)/ig ) || ( $before =~ /bootarg\.storageencryption\.support\s+(true|false)/ig ) ) { logcomment("Value of storage encrytion is $1"); $FILER_enc_val{$FILER} = $1; } $variables_set_1 = 'set-defaults;setenv bootarg.init.boot_clustered true;setenv bootarg.init.usebootp false;setenv bootarg.init.clearvarfsnvram true;setenv bootarg.mgwd.autoconf.disable true;setenv bootarg.init.verbose 1'; $variables_set_2 = 'setenv bootarg.init.console_muted true;setenv AUTOBOOT false;setenv ONTAP_NG true;setenv bootarg.mlm.enable? true;setenv bootarg.init.cf_new false;setenv bootarg.init.nbmntcf true'; $variables_set_3 = "setenv bootarg.storageencryption.support $FILER_enc_val{$FILER}"; $variables_set_4 = 'setenv bootarg.raid.allow-raid-tec false'; #$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ my $prompt_fw; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $Host = host($FILER); my $prompt_array = $Host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); #my $Server = new Server(nodes =>$FILER, no_cli=>1,filers_on_demand =>1); $Server->set_execution_server( -Server => $FILER ); #set the execution server to the current node my @filer_working = $Server->servers(); logcomment("Number of filers working are @filer_working"); #$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("Filer - $FILER : Setting the environment variables"); $Server->console( cmd => "$variables_set_1", additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => "$variables_set_2", additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => "$variables_set_3", additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ) if ( $FILER_enc_val{$FILER} ); $Server->console( cmd => "$variables_set_4", additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'saveenv', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); $Server->console( cmd => 'printenv', additional_prompts_timeout => 60, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", } ); } else { logcomment("**FATAL** - Filer : $FILER is not in LOADER Prompt, So cannot set environment variables"); logresult( 'FATAL', "Filer : $FILER is not in LOADER Prompt, So cannot set environment variables" ); } } } else { logcomment("Only Image Installtion will be done"); } } sub change_to_maint_check_for_chasis { my $Node = shift; my $FILER_C = $Node->name(); my $Transit_obj = NACL::Transit->new( name => $FILER_C ); logcomment("Changing state to Maintenance Mode and checking or chasis"); $Transit_obj->change_state( to => "MAINT" ); my $Host = host($FILER_C); my $API_Object = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); $API_Object->set_timeout( "connectrec-timeout" => 1200 ); HA_config( API_Obj => $API_Object ); sleep 30; $Transit_obj->change_state( to => "FIRMWARE" ); } sub switch_img_n_check_ver { my ( $Node, $reqd_kernel ) = @_; logcomment(" Node -$Node and Required Kernel - $reqd_kernel"); my $Home = $ENV{HOME}; my $env_set_check; my $set_env; $Home = trim($Home); my $FILER = $Node->name(); my $host = host($FILER); my $Host = host($FILER); my $Mode = $Node->mode(); my $host_file = "$Home/NDATE/FILER_INFO/$FILER"; my $filer_config = `cat $host_file`; my $def_pass; foreach my $line ( split /\n/, $filer_config ) { if ( $line =~ /^\#/ ) { next; # skip overall result report starting with a '#'. } elsif ( $line =~ /^default_password=(.*)/ ) { $def_pass = $1; } } my $Transit_obj = NACL::Transit->new( name => $FILER ); $Transit_obj->set( 'timeout' => 600, maxcycle => 20 ); my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer - $FILER state is $filer_state"); logcomment("Required Kernel for testing is - $reqd_kernel"); ################################################ ##requirements to create server object ################################################ my $prompt_fw; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); #set the execution server to the current node my @filer_working = $Server->servers(); logcomment("Number of filers working are @filer_working"); my $timeout = "7200"; # Change time-out in final code. logcomment(" Filer - $FILER : Changing state to LOADER to set the option AUTOBOOT to false"); $Transit_obj->change_state( to => "FIRMWARE" ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state =~ /FIRMWARE/ ) { logcomment("FILER is in LOADER prompt, setting AUTOBOOT to false"); } else { logcomment("FILER IS NOT IN LOADER PROMPT"); } my $loop_chek = 0; boot_again: logcomment("Reset fields 'timeout' and 'maxcycle'"); my $boot_cmd; if ( $loop_chek == 1 ) { $boot_cmd = "boot_primary"; } else { $boot_cmd = "boot_backup"; } $Transit_obj->set( 'timeout' => 700, maxcycle => 30 ); $Transit_obj->set( 'change_state_timeout' => 600 ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); if ( $state !~ /FIRMWARE/ ) { logcomment("FILER IS NOT IN LOADER, CHANGING STATE TO LOADER"); $Transit_obj->change_state( to => "FIRMWARE" ); } { logcomment( "FILER is in LOADER, Issuing " . "$boot_cmd" ); $Server->console( cmd => $boot_cmd, additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { ".*Do you want to continue.*" => 'y', ".*Copyright (C) 1992-2016 NetApp.*" => "\003", ".*Copyright (C) 1992-2016 NetApp.*" => "^C", ".*All rights reserved.*" => '^C', ".*Press Ctrl-C for Boot Menu.*" => "^C", ".*Press Ctrl-C for Boot Menu.*" => "\003", ".*Selection (1-8).*" => '5', ".*Please choose one of the following.*" => '5', ".*What is the password for.*" => "\013", "$prompt_fw" => "FINAL_PROMPT", ".*Please answer yes or no.*" => "yes", ".*Continue with boot.*" => "yes", ".*Continue with boot?.*" => "yes", "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", ".*login.*" => "admin", ".*Password.*" => $def_pass, ".*::>.*" => "FINAL_PROMPT", "::*>" => "FINAL_PROMPT", "::>" => "FINAL_PROMPT", ".*Password.*" => $def_pass, ".*Retype the password.*" => $def_pass, } ); } sleep 300; logcomment("Check state of filer"); $Transit_obj->set( 'change_state_timeout' => 600 ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer state is after issuing $boot_cmd - $state"); logcomment("Booted kernel using $boot_cmd command"); sleep 30; if ( $state =~ /NONSTD_BOOT_QUERY/ ) { logcomment("Filer is in Non standard boot query state"); $Server->console( cmd => "\013", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { ".*Do you want to continue.*" => 'y', ".*Continue with boot?.*" => "yes", ".*Continue with boot.*" => "yes", "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", } ); } else { logcomment("FILER IS NOT IN NON STANDARD FILER STATE, PROCEEDING WITH KERNEL CHECK"); } logcomment("Check filer state again"); $Transit_obj->set( 'change_state_timeout' => 600 ); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer state is after issuing $boot_cmd - $state"); logcomment("Booted kernel using $boot_cmd command"); logcomment("Checking kernel version"); my $Api_Set_Obj; my $filer_version; my $K_type_filer; if ( $state !~ /MAINT/ ) { logcomment("FILER IS NOT IN MAINTENANACE MODE"); $Transit_obj->change_state( to => "MAINT" ); sleep 300; my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("DEBUG : State of filer check 2 - $state"); } $Api_Set_Obj = NACL::APISet->new( hostobj => $Host, category => "Node", interface => "CLI", set => "Maintenance" ); my $version = $Api_Set_Obj->execute_raw_command( command => "version" ); $filer_version = $version; if ( $filer_version =~ /7Mode/i ) { logcomment("**FATAL** : Filer is in 7Mode, please install CMode kernel and re-run the test"); logresult( 'FATAL', "Filer is in 7Mode, please install CMode kernel and re-run the test" ); } if ( $filer_version =~ /<1d>/i ) { logcomment("Kernel type - DEBUG kernel"); $K_type_filer = 'DEBUG'; } else { logcomment("Kernel type - NON-DEBUG kernel"); $K_type_filer = 'NON_DEBUG'; } logcomment("Required kernel - $reqd_kernel and kernel type on filer - $K_type_filer"); if ( $reqd_kernel eq $K_type_filer ) { logcomment("Required kernel is present on filer "); $env_set_check = $boot_cmd; logcomment("DEBUG: Command used to boot image - $env_set_check"); } else { logcomment("BACKUP KERNEL IS DIFFERENT FROM THE REQUIRED KERNEL"); logcomment("Checking with primary image"); $loop_chek++; logcomment("DEBUG : loope check - $loop_chek"); if ( $loop_chek < 2 ) { goto boot_again; } else { logcomment("**FATAL** : EXECUTED BOTH boot_backup and boot_primary, EITHER OF THE IMAGE HAS REQUIRED KERNEL VERSION"); logcomment("USE OPTION 9 FROM NDATE MENU, AND INSTALL SAME BUILD VERSION OF DEBUG AND NON-DEBUG KERNELS"); logresult( "FATAL", "EXECUTED \"boot_backup\" and \"boot_primary\", EITHER OF THE IMAGE HAS REQUIRED KERNEL, USE OPTION 9 AND INSTALL IMAGE" ); } } logcomment("Set environment variable USE_SECONDARY for subsequent boot commands"); if ( $env_set_check eq "boot_backup" ) { logcomment("DEBUG: boot_backup command was used"); $set_env = "setenv USE_SECONDARY true"; } elsif ( $env_set_check eq "boot_primary" ) { logcomment("DEBUG: boot_primary command was used"); $set_env = "setenv USE_SECONDARY false"; } logcomment("Set env - $set_env"); $Transit_obj->set( 'change_state_timeout' => 600 ); $Transit_obj->change_state( to => "FIRMWARE" ); try { $Server->console( cmd => "$set_env", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", ".*LOADER.*" => "FINAL_PROMPT", } ); } catch NATE::BaseException with { logcomment("FILER : SOMETHING WENT WRONG IN SETTING ENV VARIABLE"); }; sleep(60); my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer is in $state state "); logcomment("Return - $set_env"); return $set_env; } sub re_set_n_check_image { my ( $Node, $second_var ) = @_; my $set_env; my $FILER = $Node->name(); my $host = host($FILER); my $prompt_fw; my $ignore_strings = [ "MADIMM\\d+->", "DIMM\\d+->", "^T>" ]; my $prompt_array = $host->prompt_fw(); my $prompts = join( "|", @$prompt_array ); $prompt_fw = "($prompts)"; logcomment("DEBUG:This is the prompt regex: $prompt_fw"); my $Transit_obj = NACL::Transit->new( name => $FILER ); $Transit_obj->set( 'timeout' => 600, maxcycle => 20 ); $Transit_obj->change_state( to => "FIRMWARE" ); my $Server = new Server( no_cli => 1 ); $Server->set_execution_server( -Server => $FILER ); #set the execution server to the current node my @filer_working = $Server->servers(); logcomment("Number of filers working are @filer_working"); my $timeout = "7200"; # Change time-out in final code. logcomment("Node -$Node and secondary variable set - $second_var"); logcomment("Set secondary variable for subsequent booting"); if ( $second_var eq "setenv USE_SECONDARY false" ) { $set_env = "setenv USE_SECONDARY true"; } elsif ( $second_var eq "setenv USE_SECONDARY true" ) { $set_env = "setenv USE_SECONDARY false"; } logcomment("Set env - $set_env"); $Transit_obj->set( 'change_state_timeout' => 600 ); $Transit_obj->change_state( to => "FIRMWARE" ); sleep 30; try { $Server->console( cmd => "$set_env", additional_prompts_timeout => $timeout, ignore => $ignore_strings, additional_prompts => { "$prompt_fw" => "FINAL_PROMPT", "\.*\>" => "FINAL_PROMPT", ".*LOADER.*" => "FINAL_PROMPT", } ); } catch NATE::BaseException with { logcomment("FILER :SOMETHING WENT WRONG IN SETTING ENV VARIABLE"); }; sleep 60; my $state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 ); logcomment("Filer is in $state state "); return 1; } #################################################### # Subroutine to get scsi info #################################################### sub my_scsi_diskinfo_disk() { my %options = @_; my @disks = @{ $options{disk_list} }; my $api_obj = $options{API_Obj}; my $scsi_info; $api_obj->set_timeout( "connectrec-timeout" => 1200 ); foreach my $disk (@disks) { my $output = $api_obj->execute_raw_command( 'command' => "scsi diskinfo $disk" ); foreach my $line ( split( /\n/, $output ) ) { if ( $line =~ /\s+(\S+)\s+(\S+)\s+(.*)/ ) { if ( $line =~ /FS START:/ ) { if ( $line =~ /\s*(\S+\s*\S+)s*(.*)\s*(0[xX][0-9a-fA-F]+)/i ) { my $temp = $1; chop($temp); $scsi_info->{$disk}->{$temp} = $3; } } else { $scsi_info->{$disk}->{$1} = $2; } } } } return $scsi_info; }