#$Id: //depot/prod/test/Rsierranevada/storage/hdd/NADQ_SEA/FS/Common_Lib.pm#39 $
#
#  Library of utility functions for HDD qualification tests
#
#
## @summary Library of routines to help simply HDD qualfication test development
## @description Library of routines to help in simplify the verfication required for various test cases oh hdd test plan.
##
## @synopsis
## use Storage::Common_Lib;
##
##
## @author
##         arunak@netapp.com
####################################################################
##
## Design guidelines to be followed
##
## All subroutine must start with nvme_<subroutine>
## All Subriutine must return has with all details or values in output <wherever itt is applicable>
## Each subroutine must use validate options
## Each subroutine must contain author and usage of subroutines
## Every optional value must contain default value assigned
##
##
###################################################################

package Storage::Tahiti_Common_Lib;
use TCD;
use CR::Setup::TCD qw(nlog);
use strict;
use Storage::Common_Lib;
use Tharn;
use NACL::C::StorageAggregate;
use NACL::APISet;
use Data::Dumper;
use NACL::C::Disk;
use NACL::C::Volume;
use NACL::C::Aggregate;
use NACL::C::Vserver;
use NACL::C::VolumeMember;
use NACL::CS::StorageAggregateMember;
use NACL::MTask::EventLogDetector;
use Utils;
use HTML::Mail;
use NACL::MTask::SCP;
use NACL::C::Client;
use NACL::STask::SecurityLogin;
use NACL::STask::Node;
use NACL::MTask::SystemLogDetector;
use boot;
use San;
use List::Util qw( min max shuffle);
use Utils qw(make_conn);
use Storage::SASUtils qw(filer_cmd);
use Storage::Misc qw(filer_cmd);
use install;
use misc qw(cli_localhost);
use Storage::Storage qw(storage_show_adapter);
use POSIX;
use NACL::Transit;
use Storage::SASUtils qw(disable_dbg_msgs);
use HTML::Mail;
use NACL::C::Node;
use Tharn qw(sleep);
use Tharn qw(param);
use Hostrec;
use Server;
use Options;
use Expect;
use SayRetry;
use mount;
use NACL::MTask::EventLogDetector;
use NACL::STask::StorageFailover;
use NACL::CS::Cluster;
use lib "/usr/PNATE/pnate/lib";
use PNATE;
use lib "/usr/PNATE/nate/lib";
use NATE;

my $params = NATE::ParamSet->new( global => 1 );
my $FILER          = $params->get( 'FILER',          default => undef );
my $Kernel         = $params->get( 'KERNEL',         default => 'SN' );
my $EMAIL          = $params->get( 'EMAIL',          default => undef );
my $MAIL_TO        = $params->get( 'MAIL_TO',        default => 'Email to' );
my $MAIL_FROM      = $params->get( 'MAIL_FROM',      default => 'Email from' );
my $RUNID          = $params->get( 'RUNID',          default => 'undef' );
my $STEST_FILE     = $params->get( 'STEST_FILE',     default => '' );
my $BOOT_MODE      = $params->get( 'BOOT_MODE',      default => undef );
my $Filer_mode     = $params->get( 'Filer_mode',     default => 'F' );
my $TEST_WAIT_TIME = $params->get( 'TEST_WAIT_TIME', default => '3' );
my $GROUP_NO       = $params->get( 'GROUP_NO',       default => undef );
my $DRIVE_TYPE     = $params->get( 'DRIVE_TYPE',     default => '' );
my $ERT_VOL        = $params->get( "ERT_VOL",        default => '1' );
my $LOGDIR         = $params->get( "LOGDIR",         default => 'NA' );
my $PRE_DETAILS;
my $POST_DETAILS;
my @pre_disk_count;
my @post_disk_count;
my $pre_post_recon_copy;
my %Non_Medium_Model;
logcomment("Kernel $Kernel");
logcomment("Id $RUNID");
my @run_id = split( '/', $RUNID );
my $parse_id = $run_id[-1];

#LOCAL VAR
my $SCSI_DISK_ERRORS = {};
our $GET_PHY_STATE;
our $STO_DISK_SHOW = {};

my @Filers;
if ( $FILER =~ /\,/ ) {
    @Filers = split( /\,/, $FILER );
} else {
    push( @Filers, $FILER );
}
if ( $Kernel =~ /FS|LB/i ) {
    require NACL::C::StorageDiskPartition;
    require NACL::CS::StorageDiskPartition;
}

# Import exceptions
#
use NACL::APISet::Exceptions::ResponseException qw(:try);
use NACL::APISet::Exceptions::CommandFailedException qw (:try);
use NACL::APISet::Exceptions::MissingRequiredParamException qw(:try);
use NACL::APISet::Exceptions::InvalidParamValueException qw(:try);
use NACL::APISet::Exceptions::MethodNotFoundException qw(:try);
use NACL::Exceptions::NoElementsFound qw(:try);
use NATE::BaseException qw(:try);
use NACL::Transit::Exceptions::Timeout qw(:try);
use NACL::Transit::Exceptions::RetriesExceeded qw(:try);
use NACL::Transit::Exceptions::TransitException qw(:try);
use NACL::APISet::Exceptions::TimeoutException qw(:try);
use NACL::Exceptions::EventCheckFailure qw(:try);
use NACL::APISet::Exceptions::MgwdCrashedException qw(:try);
use NACL::APISet::Exceptions::NoMatchingEntriesException qw(:try);

#use NATE::Result::Fatal qw(:try);
#
use Params::Validate qw(validate OBJECT SCALAR HASHREF ARRAYREF);
use Tharn qw( host logcomment );

BEGIN {
    Params::Validate::validation_options(
        on_fail => sub {
            my @msgs = @_;
            chomp(@msgs);
            Carp::confess( Tharn::script_result(@msgs) );
        }
    );
}

require Exporter;
use vars ( '@ISA', '@EXPORT' );
@ISA = ('Exporter');

@EXPORT = qw(
  basic_check_n_setup
  tahiti_eye_catcher
  tahiti_get_drv_by_user_sel_mod_id
  tahiti_list_only_tahiti_drv
  tahiti_get_all_disk
  tahiti_get_disk_owned
  tahiti_get_local_spares
  tahiti_sto_agg_show_spare_dsk
  tahiti_map_fw_dsk_rev_model
  tahiti_boot_main_unpart_assign_parts
  tahiti_check_drv_slots
  tahiti_system_health_check
  tahiti_disk_fw_update_low_or_high
);

sub tahiti_system_health_check {
    my (%opts) = validate(
        @_,
        {
            Nodes     => { type => ARRAYREF },
            handle    => { type => SCALAR, default => undef, optional => 1 },
            part_flag => { type => SCALAR },
        }
    );
    my @APOLLO_DRVS = @{ $opts{APOLLO_DRVS} };
    my $handle      = $opts{handle};
    my $partition   = $opts{part_flag};
    my $health      = 0;

    #check for partition flag
    #check for cluster
    #check for drive assignment ( P1 - nodeA P2-node B and p3-nodeowner )

    return $health;
}

####################################################################################################
# NVME - Check drive slots - Node A -[0-11] or [24-35] & Node B =[12-23] or [36-47]
# Return : N/A
# Burt :
# Author : arunak
####################################################################################################
sub tahiti_check_drv_slots {
    my (%opts) = validate(
        @_,
        {
            APOLLO_DRVS => { type => ARRAYREF },
            handle      => { type => SCALAR, default => undef, optional => 1 },
        }
    );
    my @APOLLO_DRVS = @{ $opts{APOLLO_DRVS} };
    my $handle      = $opts{handle};
    my @validA;
    my @validB;
    my $status = 0;
    logcomment("Internal Drive should be in particular slots for autopartition");
    logcomment("Apollo Internal Drives slot check");
    logcomment("Internal Drives must be in describe slot, Drives should be equally added accross these slots");
    logcomment("DRIVES for NODE-A : [0 to 11] or [24 to 35] slots.\nDRIVES for NODE-B : [12 to 23] or [36 to 47] slots.");
    my $total = @APOLLO_DRVS;

    foreach $_ ( 0 .. 11 ) {
        foreach my $d (@APOLLO_DRVS) {
            if ( $d =~ /0n\.$_$/ ) {
                push( @validA, $d );
            }
        }
    }
    foreach $_ ( 12 .. 23 ) {
        foreach my $d (@APOLLO_DRVS) {
            if ( $d =~ /0n\.$_$/ ) {
                push( @validB, $d );
            }
        }
    }
    foreach $_ ( 24 .. 35 ) {
        foreach my $d (@APOLLO_DRVS) {
            if ( $d =~ /0n\.$_$/ ) {
                push( @validA, $d );
            }
        }
    }
    foreach $_ ( 36 .. 47 ) {
        foreach my $d (@APOLLO_DRVS) {
            if ( $d =~ /0n\.$_$/ ) {
                push( @validB, $d );
            }
        }
    }
    logcomment("Total Internal Drives : $total");
    my $alen = @validA;
    my $blen = @validB;
    logcomment("NodeA : $alen and NodeB: $blen");
    if ( $total = ( $alen + $blen ) ) {
        if ( $alen == $blen ) {
            logcomment("Drives are equal and Drives are in expected slots");
        } else {
            logcomment("**WARNING** : Drives are not in expected slots, Move drives to slots as per requirement NODE-A:[0-11] or [24-35] NODE-B:[12-23] or [36-47] slots.");
            logcomment("Drives are not in expected slots, Move drives to slots as per requirement");
            logcomment("Move half drives to slots - [0-11] or [24-35] and other half to slots [12-23] or [36-47]");
            $status = 1;
        }
    } else {
        logcomment("**WARNING** : Total drive count mis-matched, Check drives physically and add drives in expected slots");
        $status = 1;
    }

    return $status;

}

####################################################################################################
# NVME - Boot in maintenance mode and unparttion and assign equally
# Return : N/A
# Burt :
# Author : arunak
####################################################################################################

sub tahiti_boot_main_unpart_assign_parts {
    my (%opts) = validate(
        @_,
        {
            NODES     => { type => ARRAYREF },
            handle    => { type => SCALAR, default => undef, optional => 1 },
            partition => { type => SCALAR },
        }
    );
    my @Nodes     = @{ $opts{NODES} };
    my $handle    = $opts{handle};
    my $partition = $opts{partition};
    my $status    = 0;
    logcomment("PARTITION : $partition");
    my $filer_details = {};
    logcomment("Nodes Available : @Nodes");
    logcomment("Get State of filer");

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        $Transit_obj->set( 'timeout'              => 7200 );
        $Transit_obj->set( 'change_state_timeout' => 7200 );
        my $FILER_STATE = $Transit_obj->get_state( 'timeout' => 7200, 'get_state_timeout' => 7200 );
        logcomment("Filer : $FILER_C is in $FILER_STATE");
        if ( $FILER_STATE ne "MAINT" ) {
            logcomment("FILER : $FILER_C is not in Maintenance mode , Changing state to maintenance mode");
            $Transit_obj->change_state( to => "MAINT" );
        }
        logcomment("Checking state of $FILER_C");
        my $TEMP_STATE = $Transit_obj->get_state( 'timeout' => 7200, 'get_state_timeout' => 7200 );
        logcomment("$FILER_C : Is in $TEMP_STATE");
        logcomment("Get Filer details ");
        my $Host        = host($FILER_C);
        my $Api_Set_Obj = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance",
        );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 8887200 );
        $filer_details->{$FILER_C}->{'FILER_NAME'}    = $FILER_C;
        $filer_details->{$FILER_C}->{'API_OBJ_MAINT'} = $Api_Set_Obj;
        $filer_details->{$FILER_C}->{'NODE_OBJ'}      = $Node;
        my $disk_out = $Api_Set_Obj->execute_raw_command( 'command' => "disk show -n" );

        foreach my $line ( split( /\n/, $disk_out ) ) {
            if ( $line =~ /Local System ID:\s+(\S+)/ ) {
                my $sysid = $1;
                logcomment("$FILER_C System ID : $sysid");
                $filer_details->{$FILER_C}->{'SYSID'} = $sysid;
                last;
            } else {
                logcomment("$FILER_C : System ID not found");
                my $temp_out = $Api_Set_Obj->execute_raw_command( 'command' => "sysconfig" );
                foreach my $line ( split( /\n/, $temp_out ) ) {
                    if ( $line =~ /System ID:\s+(\S+)\s+\($FILER_C\)/ ) {
                        my $sysid = $1;
                        logcomment("$FILER_C System ID from sysconfig :  $sysid");
                        $filer_details->{$FILER_C}->{'SYSID'} = $sysid;
                    }
                }
            }
        }
    }

    #logcomment("Filer details captured is :");
    #logcomment( "DEBUG:Details " . Dumper($filer_details) );

    #remove all drives and asign to one node
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Host        = host($FILER_C);
        my $Api_Set_Obj = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance",
        );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 8887200 );
        logcomment("Remove ownership of all drives");
        $Api_Set_Obj->execute_raw_command( 'command' => "disk remove_ownership all -f" );
        $Api_Set_Obj->execute_raw_command( 'command' => " " );
        logcomment("Check for any owned drives");
        my $out = $Api_Set_Obj->execute_raw_command( 'command' => "disk show -a" );

        foreach my $line ( split( /\n/, $out ) ) {
            if ( $line =~ /$FILER_C/ ) {
                logcomment("Some of the drives are still owned by filer $FILER_C , Removing drive owenership");
                logcomment("Remove ownership of all drives");
                $Api_Set_Obj->execute_raw_command( 'command' => "disk remove_ownership all -f" );
                $Api_Set_Obj->execute_raw_command( 'command' => " " );
                last;
            }
        }
        logcomment("All drive ownership removed, Proceed with next steps ");
        sleep 2;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $prompts_answers = [ ".*Do you wish to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "storage release disks", 'connectrec-match_table' => $prompts_answers );
        sleep 2;
    }
    logcomment("Assign all drives to one node and unpartition drives");
    logcomment("Consider one node and operate on one node");
    my $NODE    = $Nodes[0];
    my $N_FILER = $NODE->name();
    my $Host    = host($N_FILER);
    logcomment("Assign all drives to Node : $N_FILER");
    my $N_SYSID = $filer_details->{$N_FILER}->{'SYSID'};
    logcomment("$N_FILER system ID : $N_SYSID");
    my $Api_Set_Obj_N = NACL::APISet->new(
        hostobj   => $Host,
        category  => "Node",
        interface => "CLI",
        set       => "Maintenance",
    );
    $Api_Set_Obj_N->execute_raw_command( 'command' => "\013" );
    $Api_Set_Obj_N->execute_raw_command( 'command' => "disk assign all -s $N_SYSID -o $N_FILER -f" );
    logcomment("Check for any unowned drives");
    my $d_out = $Api_Set_Obj_N->execute_raw_command( 'command' => "disk show -n" );

    if ( $d_out =~ /Not Owned|Unowned/ ) {
        logcomment("Still Some of the drives are Not Owned, Assign all drive again");
        $Api_Set_Obj_N->execute_raw_command( 'command' => "disk assign all -s $N_SYSID -o $N_FILER -f" );
        $Api_Set_Obj_N->execute_raw_command( 'command' => "\013" );
        sleep 2;
        logcomment("Check for unowned drives again");
        my $d_out = $Api_Set_Obj_N->execute_raw_command( 'command' => "disk show -n" );
        if ( $d_out =~ /Not Owned|Unowned/ ) {
            logcomment("**FAIL** : Assign drives manually");
            logresult( 'FATAL', "Assign drives manually" );
        } else {
            logcomment("Drives assigned to $N_FILER");
        }
    } else {
        logcomment("Drives assigned to $N_FILER");
    }
    logcomment("All drives assigned to one node, Un-partition drives");
    my $prompts_answers = [ ".*Abort unpartition (y/n).*" => 'n', "Abort unpartition" => 'n' ];
    $Api_Set_Obj_N->execute_raw_command( 'command' => "disk unpartition all -F", 'connectrec-match_table' => $prompts_answers );
    $Api_Set_Obj_N->execute_raw_command( 'command' => "\013" );
    sleep 2;
    logcomment("All drives un-partitioned");
    logcomment("Remove ownership of all drives again");

    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Host        = host($FILER_C);
        my $Api_Set_Obj = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance",
        );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 8887200 );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        $Api_Set_Obj->execute_raw_command( 'command' => "disk remove_ownership all -f" );
        sleep 2;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $prompts_answers = [ ".*Do you wish to continue.*" => 'y' ];
        $Api_Set_Obj->execute_raw_command( 'command' => "storage release disks", 'connectrec-match_table' => $prompts_answers );
    }
    logcomment("All drive ownership removed");
    my @internal_drv;
    my @tahiti_drv;

    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Host        = host($FILER_C);
        my $Api_Set_Obj = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance",
        );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 8887200 );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        $Api_Set_Obj->execute_raw_command( 'command' => "disk show" );
        sleep 3;
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $dsk_out = $Api_Set_Obj->execute_raw_command( 'command' => "disk_list" );

        foreach my $line ( split( /\n/, $dsk_out ) ) {
            if ( $line =~ /(\S+\.\S+)\s+/ ) {
                my $drv = $1;
                if ( $drv =~ /\S+\.\S+\.\S+\.\S+/ ) {
                    push( @tahiti_drv, $drv );
                } else {
                    push( @internal_drv, $drv );
                }
            }
        }
        logcomment("Internal drives : @internal_drv");
        logcomment("Tahiti Drives : @tahiti_drv");

        #my $Partner = NACL::STask::Node->get_partner_obj( command_interface => $Node );
        my $Partner_name = $filer_details->{$FILER_C}->{'NODE_OBJ'}->{'hostrec'}->{'part_host'};
        logcomment("FILER : $FILER_C and PARTNER : $Partner_name");
        my $sys_id     = $filer_details->{$FILER_C}->{'SYSID'};
        my $part_sysid = $filer_details->{$Partner_name}->{'SYSID'};
        my $int_len    = @internal_drv;
        my $hlf        = int( $int_len / 2 );
        my @INT_DRV_A  = splice( @internal_drv, 0, $hlf );
        logcomment("NODE A : @INT_DRV_A");
        logcomment("NODE B : @internal_drv");
        logcomment("FILER : $FILER_C and SYSID :$sys_id");
        logcomment("Assigning half of drives to $FILER_C with sysid $sys_id");
        logcomment("$FILER_C : Assigning : @INT_DRV_A");
        $Api_Set_Obj->execute_raw_command( 'command' => "disk assign @INT_DRV_A -s $sys_id -o $FILER_C -f" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        logcomment("Assigned internal drives");
        sleep 1;
        logcomment("Assigning remaining drives to $Partner_name");
        logcomment("$FILER_C : Assigning : @internal_drv");
        $Api_Set_Obj->execute_raw_command( 'command' => "disk assign @internal_drv -s $part_sysid -o $Partner_name -f" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        logcomment("Assigned internal drives");
        sleep 1;
        logcomment("Assigned all internal drives to both nodes");
        logcomment("$FILER_C : Tahiti Drives : @tahiti_drv");
        my $th_len   = @tahiti_drv;
        my $thlf     = int( $th_len / 2 );
        my @TH_DRV_A = splice( @tahiti_drv, 0, $thlf );
        logcomment("NODE A : @TH_DRV_A ");
        logcomment("NODE B : @tahiti_drv ");
        logcomment("FILER : $FILER_C and SYSID :$sys_id");
        logcomment("Assigning half of Tahiti drives to $FILER_C with sysid $sys_id");
        logcomment("$FILER_C : Assigning : @TH_DRV_A");
        $Api_Set_Obj->execute_raw_command( 'command' => "disk assign @TH_DRV_A -s $sys_id -o $FILER_C -f" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        logcomment("Assigned tahiti drives");
        sleep 1;
        $Api_Set_Obj->execute_raw_command( 'command' => "disk assign @tahiti_drv -s $part_sysid -o $Partner_name -f" );
        logcomment("Assign tahiti drives");
        last;
    }
    logcomment("All Drives Assigned ");
    logcomment("Partitioned flag : $partition");
    if ( $partition == 1 ) {
        logcomment("User Selected group requires setup to be partitioned");
        tahiti_part_assig_parts( NODES => \@Nodes );
    } else {
        logcomment("User Selected group requires setup to be un-partitioned");
        logcomment("Completed Assignment, Proceed with next steps");
    }

    return $status;

}

sub tahiti_part_assig_parts {
    my (%opts) = validate(
        @_,
        {
            NODES  => { type => ARRAYREF },
            handle => { type => SCALAR, default => undef, optional => 1 },
        }
    );
    my @Nodes          = @{ $opts{NODES} };
    my $handle         = $opts{handle};
    my $status         = 0;
    my $filer_drv_part = {};
    my $Api_Set_Obj;
    logcomment("Nodes Available : @Nodes");
    logcomment("Get State of filer");

    foreach my $Node (@Nodes) {
        my $FILER_C = $Node->name();
        my $Host    = host($FILER_C);
        $Api_Set_Obj = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance",
        );
        $Api_Set_Obj->set_timeout( "connectrec-timeout" => 8887200 );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $sysout = $Api_Set_Obj->execute_raw_command( 'command' => "disk show -o $FILER_C" );
        foreach my $line ( split( /\n/, $sysout ) ) {
            if ( $line =~ /Local System ID:\s+(\S+)/ ) {
                my $sys = $1;
                logcomment("$FILER_C : Sysid : $sys");
                $filer_drv_part->{$FILER_C}->{'SYSID'} = $1;
            }
        }
    }
    my $loop = 0;
    foreach my $Node (@Nodes) {
        my $FILER_C     = $Node->name();
        my $Host        = host($FILER_C);
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        $Transit_obj->set( 'timeout'              => 7200 );
        $Transit_obj->set( 'change_state_timeout' => 7200 );
        my $FILER_C_STATE = $Transit_obj->get_state( 'timeout' => 7200, 'get_state_timeout' => 7200 );
        logcomment("Filer : $FILER_C is in $FILER_C_STATE");
        if ( $FILER_C_STATE =~ /MAINT/ ) {
            logcomment("Filer is in Maintenance mode");
            my $Host = host($FILER_C);
            $Api_Set_Obj = NACL::APISet->new(
                hostobj   => $Host,
                category  => "Node",
                interface => "CLI",
                set       => "Maintenance",
            );
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 8887200 );
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        } else {
            logcomment("Partition of drives - Filer should be in Maintenance mode ");
            $status = 1;
            return $status;
        }
        my $disk_out = $Api_Set_Obj->execute_raw_command( 'command' => "disk show -o $FILER_C" );
        my @part_th_drv;
        foreach my $line ( split( /\n/, $disk_out ) ) {
            if ( $line =~ /(\S+\.\S+)\s+/ ) {
                my $drv = $1;
                if ( $drv =~ /(\S+\.\S+\.\S+\.\S+)/ ) {
                    next if ( $drv =~ /P\d+/ );
                    my $th_drv = $1;
                    push( @part_th_drv, $th_drv );
                    logcomment("Partition Tahiti Drive : $th_drv");
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                    $Api_Set_Obj->execute_raw_command( 'command' => "disk partition -n 3 $th_drv" );
                    sleep 3;
                    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
                }
            }
        }
        logcomment("$FILER_C: Drives owned by filer $FILER_C : All Drives partitioned");
        $Api_Set_Obj->execute_raw_command( 'command' => "disk show -n" );
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        sleep 1;

        #my $Partner = NACL::STask::Node->get_partner_obj( command_interface => $Node );
        my $Partner_name = $Node->{'hostrec'}->{'part_host'};
        logcomment("FILER : $FILER_C and PARTNER : $Partner_name");
        my $sysid      = $filer_drv_part->{$FILER_C}->{'SYSID'};
        my $part_sysid = $filer_drv_part->{$Partner_name}->{'SYSID'};
        logcomment("$FILER_C - $sysid : $Partner_name - $part_sysid");
        my @drv_p1;
        my @drv_p2;
        my @drv_p3;

        foreach my $thdrv (@part_th_drv) {
            my $P1 = "$thdrv" . "P1";
            my $P2 = "$thdrv" . "P2";
            my $P3 = "$thdrv" . "P3";
            push( @drv_p3, $P3 );
            push( @drv_p2, $P2 );
            push( @drv_p1, $P1 );
        }
        if ( $loop == 0 ) {
            logcomment("Assign P1 and P3 to $FILER_C");
            $Api_Set_Obj->execute_raw_command( 'command' => "disk assign @drv_p2 @drv_p3 -s $sysid -o $FILER_C -f" );
            logcomment("Partition P1 and P3 assigned to $FILER_C ");
            sleep 3;
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            logcomment("Assign P2 to $Partner_name");
            $Api_Set_Obj->execute_raw_command( 'command' => "disk assign @drv_p1 -s $part_sysid -o $Partner_name -f" );
            logcomment("Partition P2 assigned to $Partner_name");
            sleep 3;
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        } else {
            logcomment("Assign P2 and P3 to $FILER_C");
            $Api_Set_Obj->execute_raw_command( 'command' => "disk assign @drv_p1 @drv_p3 -s $sysid -o $FILER_C -f" );
            sleep 3;
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
            logcomment("Assign P1 to $Partner_name");
            $Api_Set_Obj->execute_raw_command( 'command' => "disk assign @drv_p2 -s $part_sysid -o $Partner_name -f" );
            logcomment("Partition P1 assigned to $Partner_name");
            sleep 3;
            $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        }
        $loop++;
    }
    logcomment("Assigned all partitions to both nodes");
}

####################################################################################################
# NVME - Basic state check
# Return : N/A
# Burt :
# Author : arunak
####################################################################################################

sub basic_check_n_setup {
    my (%opts) = validate(
        @_,
        {
            Node           => { type => OBJECT },
            handle         => { type => SCALAR, default => undef, optional => 1 },
            change_state_2 => { type => SCALAR, default => undef, optional => 1 },
        }
    );
    my $Node           = $opts{Node};
    my $handle         = $opts{handle};
    my $change_state_2 = $opts{change_state_2};
    my $API_Obj;
    my $FILER_C = $Node->name();
    my $version = "NA";
    logcomment("Check Filer state and Set Test privilage and execute basic setup commands");
    my $Transit_obj = NACL::Transit->new( name => $FILER_C );
    $Transit_obj->set( 'timeout'              => 7200 );
    $Transit_obj->set( 'change_state_timeout' => 7200 );
    my $FILER_C_STATE = $Transit_obj->get_state( 'timeout' => 7200, 'get_state_timeout' => 7200 );
    logcomment("Filer : $FILER_C is in $FILER_C_STATE");

    if ( $FILER_C_STATE =~ /CLI/ ) {
        logcomment("Filer is in CLI/File system mode");
        $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $API_Obj->set_timeout( "connectrec-timeout" => 8887200 );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $API_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        logcomment("Privilage level test set");
        $API_Obj->execute_raw_command( 'command' => "set -rows 0" );
        logcomment("Get kernel version");
        $version = $API_Obj->execute_raw_command( 'command' => "run local version" );
        logcomment("$FILER_C : Kernel version : $version");
        logcomment("Set log-level and system timeout");
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $API_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        $API_Obj->execute_raw_command( 'command' => "set -rows 0" );
        $API_Obj->execute_raw_command( 'command' => 'system timeout modify 0' );
        $API_Obj->execute_raw_command( 'command' => 'run local dbg to off' );
        $API_Obj->execute_raw_command( 'command' => 'event config modify -suppression on -console on -console-log-level DEBUG' );
        sleep 5;
    } elsif ( $FILER_C_STATE =~ /MAINT/ ) {
        logcomment("Filer is in Maintenance mode");
        my $Host = host($FILER_C);
        $API_Obj = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance",
        );
        $API_Obj->set_timeout( "connectrec-timeout" => 8887200 );
        logcomment("Get kernel version");
        $version = $API_Obj->execute_raw_command( 'command' => "version" );
        logcomment("$FILER_C : Kernel version : $version");
    } elsif ( $FILER_C_STATE =~ /USERNAME/ ) {
        logcomment("Filer is in login prompt");
        $Transit_obj->change_state( to => "CLI" );
        logcomment("Filer Changed to CLI");
        $API_Obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $API_Obj->set_timeout( "connectrec-timeout" => 8887200 );
        my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
        $API_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
        logcomment("Privilage level test set");
        $API_Obj->execute_raw_command( 'command' => "set -rows 0" );
        logcomment("Get kernel version");
        $version = $API_Obj->execute_raw_command( 'command' => "run local version" );
        logcomment("$FILER_C : Kernel version : $version");
    } elsif ( $FILER_C_STATE =~ /LOADER|FIRMWARE/ ) {
        logcomment("Filer is in LOADER, Change filer to Maintenance mode");
        $Transit_obj->change_state( to => "MAINT" );
        logcomment("Filer is in MAINT mode");
        my $Host = host($FILER_C);
        $API_Obj = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance",
        );
        $API_Obj->set_timeout( "connectrec-timeout" => 8887200 );
        logcomment("Get kernel version");
        $version = $API_Obj->execute_raw_command( 'command' => "version" );
        logcomment("$FILER_C : Kernel version : $version");
    } else {
        logcomment("FILER is in un stable mode, boot filer manually");
    }
    return $version;
}

####################################################################################################
# NVME - Eye Catcher to be called in all NVMe-Scripts
# Return : N/A
# Burt :
# Author : arunak
####################################################################################################
sub tahiti_eye_catcher {
    my (%opts) = validate(
        @_,
        {
            Test   => { type => OBJECT },
            string => { type => SCALAR, default => undef, optional => 1 },
        }
    );
    my $Test   = $opts{Test};
    my $string = $opts{string};
    $Test->nlog("###################################################################################");
    $Test->nlog("### Tahiti : $string");
    $Test->nlog("###################################################################################");
}

sub tahiti_get_drv_by_user_sel_mod_id {

    my (%opts) = validate(
        @_,
        {
            API_con    => { type => OBJECT },
            Node       => { type => OBJECT },
            handle     => { type => SCALAR, optional => 1, default => 0 },
            model      => { type => SCALAR },
            local_name => { type => SCALAR },
            Filer      => { type => SCALAR },
        }
    );
    my $Api_Set_Obj = $opts{API_con};
    my $handle      = $opts{handle};
    my $FILER_C     = $opts{Filer};
    my $MODEL       = $opts{model};
    my $local_name  = $opts{local_name};
    my $Node        = $opts{Node};

    my @drv_2_return;
    logcomment(" $FILER_C : Get all drives ");
    my $disk_list_result = tahiti_get_all_disk( node_present => [$Node] );
    my @dsk              = keys %$disk_list_result;
    my @disk_info        = sort(@dsk);
    my @user_sel_drv;
    my @drvs;
    logcomment("Drive Present : @disk_info");
    my $Transit_obj = NACL::Transit->new( name => $FILER_C );
    my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
    logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state");

    if ( $filer_state =~ /CLI|UP/ ) {
        logcomment("Get spare belongs to $FILER_C");
        my @local_spare_drv = tahiti_get_local_spares( API_con => $Api_Set_Obj, Filer => $FILER_C );
        logcomment("$FILER_C : Spare drives are @local_spare_drv");
        logcomment("USER SEELCTED MODEL : $MODEL");

        foreach my $dsk (@disk_info) {
            my $model_name = $disk_list_result->{$dsk}->{'MODEL'};
            if ( $model_name =~ /$MODEL/ ) {
                push( @drvs, $dsk );
            }
        }

        #logcomment("Drives with model : $MODEL are :");
        #logcomment("@drvs");
        foreach my $drv (@drvs) {
            if ( $drv =~ /\w+.\w+\.\w+\.\w+/ ) {
                push( @user_sel_drv, $drv );
            }
        }
        logcomment("User Selected Drives are : @user_sel_drv");

        my @drv_node_scp;
        if ( $local_name =~ /yes/i ) {
            logcomment("Retrun drives with local name");
            return @user_sel_drv;
        } else {
            logcomment("Get node scope name and return drives");
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 1200000 );
            $Api_Set_Obj->execute_raw_command( command => "run local \"set -rows 0;system timeout modify 0\"" );
            my $spare_list = tahiti_sto_agg_show_spare_dsk( Node => $Node );
            @drv_node_scp = keys %$spare_list;
            logcomment("Drive Belongs to Filer : $FILER_C : @drv_node_scp");
            foreach my $sdrv (@drv_node_scp) {

                # my ($chk) = $sdrv =~ /\w+.\w+\.\w+.(\w+)/;
                my ($chk) = $sdrv =~ /\w+\.\w+\.(\w+)/;
                foreach my $usr_sel (@user_sel_drv) {

                    # my ($matc) = $usr_sel =~ /\w+\.(\S+)/;
                    my ($matc) = $usr_sel =~ /\w\.\d+\.\d+\.(\d+)/;

                    #logcomment("MATC : $matc and CHK : $chk");
                    if ( $matc == $chk ) {
                        push( @drv_2_return, $sdrv );
                    }
                }
            }
            @drv_2_return = unique(@drv_2_return);
            logcomment("Drives to return : @drv_2_return");
            return @drv_2_return;
        }
    } elsif ( $filer_state =~ /MAINT/ ) {
        logcomment("Filer is in Maintenece mode - Get drives owned by node");
        logcomment("USER SEELCTED MODEL : $MODEL");
        my @fil_drvs = tahiti_get_disk_owned( Node => $Node );
        foreach my $dsk (@disk_info) {
            my $model_name = $disk_list_result->{$dsk}->{'MODEL'};
            if ( $model_name =~ /$MODEL/ ) {
                push( @user_sel_drv, $dsk );
            }
        }
        logcomment("Drives with model : $MODEL are : @user_sel_drv");
        logcomment("$FILER_C : All Drives : @fil_drvs");
        logcomment("$FILER_C : Check 1 - Drives user model selected returned - @user_sel_drv");
        foreach my $all_drv (@fil_drvs) {
            foreach my $loc_own (@user_sel_drv) {
                if ( $all_drv eq $loc_own ) {
                    logcomment("Drive with $MODEL owned by $FILER_C : $all_drv");
                    push( @drv_2_return, $all_drv );
                }
            }
        }
        @drv_2_return = unique(@drv_2_return);
        return @drv_2_return;
    }
}

sub tahiti_list_only_tahiti_drv {
    my (%opts) = validate( @_, { node_present => { type => ARRAYREF }, } );
    my @Nodes = @{ $opts{node_present} };
    my $dsk_result = {};
    foreach my $node (@Nodes) {
        my $Api_Set_Obj;
        my $FILER_C = $node->name();
        my $Host    = host($FILER_C);
        logcomment("Get all Tahiti drives attached");
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state");
        my $output;

        if ( $filer_state =~ /UP|CLI/i ) {
            $Api_Set_Obj = $node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 );
            $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;disk_list\"" );
        } elsif ( $filer_state =~ /MAINT/i ) {
            $Api_Set_Obj = NACL::APISet->new(
                hostobj   => $Host,
                category  => "Node",
                interface => "CLI",
                set       => "Maintenance"
            );
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 );
            $output = $Api_Set_Obj->execute_raw_command( command => "disk_list" );
        }
        my @lines = split( /\n/, $output );
        my $n = 0;
        while ( $n <= $#lines ) {
            last if ( $lines[$n] =~ /^-----/ );
            $n++;
        }
        $n++;
        foreach my $line ( @lines[ $n .. $#lines ] ) {
            my ( $disk, $chan, $ven, $prodid, $rev, $serialno, $hw, $blocks, $bps, $dq ) = ( $line =~ /(\S+)\s+(.*)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ );
            if ( ( $line =~ /\d\w\.\d\d/ ) || ( $line =~ /\d\w\.\d\d\.\d\d/ ) || ( $line =~ /\d\w\.\S+/ ) || ( $line =~ /\d\w\.\d\d\.\S+/ ) || ( $line =~ /\w+\d+\w+\.\d\d\.\S+/ ) ) {
                $dsk_result->{$disk} = {
                    'DISK'       => $disk,
                    'VENDOR'     => $ven,
                    'PRODUCT_ID' => $prodid,
                    'REV'        => $rev,
                    'SERIAL'     => $serialno,
                    'HW'         => $hw,
                    'BLOCKS'     => $blocks,
                    'BPS'        => $bps,
                    'DQ'         => $dq
                };
            }
        }
        last;
    }
    return $dsk_result;
}

sub tahiti_get_all_disk {
    my (%opts) = validate( @_, { node_present => { type => ARRAYREF }, } );
    my @Nodes = @{ $opts{node_present} };
    my $dsk_result = {};
    foreach my $node (@Nodes) {
        my $Api_Set_Obj;
        my $FILER_C = $node->name();
        my $Host    = host($FILER_C);
        logcomment("Get all Tahiti drives attached");
        my $Transit_obj = NACL::Transit->new( name => $FILER_C );
        my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
        logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state");
        my $output;

        if ( $filer_state =~ /UP|CLI/i ) {
            $Api_Set_Obj = $node->get_7m_or_nodescope_apiset( connid => 'console' );
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 );
            $output = $Api_Set_Obj->execute_raw_command( command => "run local \"priv set -q test;storage show disk -T -x\"" );
        } elsif ( $filer_state =~ /MAINT/i ) {
            $Api_Set_Obj = NACL::APISet->new(
                hostobj   => $Host,
                category  => "Node",
                interface => "CLI",
                set       => "Maintenance"
            );
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => 12000 );
            $output = $Api_Set_Obj->execute_raw_command( command => "storage show disk -T -x" );
        }
        my @lines = split( /\n/, $output );
        my $n = 0;
        while ( $n <= $#lines ) {
            last if ( $lines[$n] =~ /^-----/ );
            $n++;
        }
        $n++;
        foreach my $line ( @lines[ $n .. $#lines ] ) {
            my ( $disk, $shlf, $bay, $serial, $vendor, $model, $rev, $type ) = ( $line =~ /(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ );

            #logcomment("DEBUG:$line ");
            if ( ( $line =~ /\d\w\.\d\d/ ) || ( $line =~ /\d\w\.\d\d\.\d\d/ ) || ( $line =~ /\d\w\.\S+/ ) || ( $line =~ /\d\w\.\d\d\.\S+/ ) || ( $line =~ /\w\d\w\.\d\d\.\S+/ ) ) {
                $dsk_result->{$disk} = {
                    'DISK'   => $disk,
                    'SHELF'  => $shlf,
                    'BAY'    => $bay,
                    'SERIAL' => $serial,
                    'VENDOR' => $vendor,
                    'MODEL'  => $model,
                    'REV'    => $rev,
                    'TYPE'   => $type,
                };
            }
        }
        last;
    }
    return $dsk_result;
}

sub tahiti_get_disk_owned {
    my (%opts) = validate( @_, { Node => { type => OBJECT }, } );
    my $Node = $opts{Node};
    my $FILER_C = $Node->name();
    my $Host    = host($FILER_C);
    my @drve_local;
    my $API_obj;
    my $dsk_out;
    my $flag        = 0;
    my $Transit_obj = NACL::Transit->new( name => $FILER_C );
    my $filer_state = $Transit_obj->get_state( 'timeout' => 3600, 'get_state_timeout' => 7200 );
    logcomment("DEBUG:Node $FILER_C is in the Filer state $filer_state");
    logcomment("Filer : $FILER_C is in $filer_state state");

    if ( $filer_state =~ /UP|CLI/i ) {
        $API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
        $API_obj->set_timeout( "connectrec-timeout" => 1200000 );
        $API_obj->execute_raw_command( command => "set -rows 0;system timeout modify 0" );
        $dsk_out = $API_obj->execute_raw_command( command => "run local disk show -o $FILER_C" );
    } elsif ( $filer_state =~ /MAINT/i ) {
        my $API_obj = NACL::APISet->new(
            hostobj   => $Host,
            category  => "Node",
            interface => "CLI",
            set       => "Maintenance"
        );
        my $sys_id_a = Storage::Common_Lib::sysconfig_v_data( node => $Node, API_Obj => $API_obj );
        my $sys_id = $sys_id_a->{'system_id'};
        logcomment("FILER : $FILER_C : System ID : $sys_id");
        $dsk_out = $API_obj->execute_raw_command( 'command' => "disk show -s $sys_id" );
    } else {
        logcomment("**WARNING**: Cannot get filer state, Check filer is accessible and in proper state");
        $flag = 1;
    }
    if ( $flag == 0 ) {
        foreach my $line ( split( /\n/, $dsk_out ) ) {
            next if ( $line =~ /---/ );
            next if ( $line =~ /DISK|NOTE/ );
            if ( $line =~ /(\w+.\w+.\w+\.\w+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)\s+(\S+)/ ) {
                push( @drve_local, $1 );
            }
        }
    }
    logcomment("Drive Belongs to Filer : $FILER_C : @drve_local");
    return @drve_local;
}

sub tahiti_get_local_spares {
    my (%opts) = validate(
        @_,
        {
            API_con => { type => OBJECT },
            handle  => { type => SCALAR, optional => 1, default => 0 },
            Filer   => { type => SCALAR },
        }
    );
    my $Api_Set_Obj = $opts{API_con};
    my $handle      = $opts{handle};
    my $FILER_C     = $opts{Filer};
    my @spare_disk_list;
    my @not_zeroed_spares;
    $Api_Set_Obj->set_timeout( "connectrec-timeout" => 72000 );
    try {
        $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
        my $output = $Api_Set_Obj->execute_command( 'command' => "aggr status -s" );
        my $n      = 0;
        my @lines  = split( /\n/, $output );
        while ( $n <= $#lines ) {
            last if ( $lines[$n] =~ /^-----/ );
            $n++;
        }
        $n++;
        foreach my $line ( @lines[ $n .. $#lines ] ) {
            if ( $line =~ /spare\s+(\S+)/ ) {

                #   logcomment("DEBUG: Spare drive : $1");
                push( @spare_disk_list, $1 );
            }

            if ( $line =~ /spare\s+(\S+)\s*(.*)\s*\((.*)\)/ ) {

                # logcomment("DEBUG:drive is : $1");
                # logcomment("DEBUG:zeroed or not zeroed drive ?: $3");
                my $var = $3;
                push( @not_zeroed_spares, $1 ) if ( $var =~ /not zeroed/i );
            }
        }

    }
    catch NACL::Exceptions::NoElementsFound with {
        logcomment("NO SPARE DISKS FOUND FOR THE FILER $FILER_C");
    };
    logcomment( "$FILER_C : The number of spare disks present are " . scalar @spare_disk_list . " and the disks are: @spare_disk_list " ) if (@spare_disk_list);
    logcomment("$FILER_C : No Spare drives found ") if ( !@spare_disk_list );
    if ( $handle == 1 ) {
        if (@not_zeroed_spares) {
            while (1) {
                my $output = $Api_Set_Obj->execute_command( 'command' => "disk zero spares" );
                sleep(60);
                my $out = $Api_Set_Obj->execute_command( 'command' => "aggr status -s" );
                last if ( $out =~ /fast zeroed/i );
            }
        }
    } elsif ( $handle == 2 ) {

        #  return @not_zeroed_spares if (@not_zeroed_spares);
        return @not_zeroed_spares;
    }
    return @spare_disk_list;
}

sub tahiti_sto_agg_show_spare_dsk {
    my (%opts) = validate( @_, { Node => { type => OBJECT } } );
    my $Node = $opts{Node};
    my @spare_drav;
    my $dsk_out;
    my $filer_spare_rst = {};
    my $API_obj;
    my $FILER_C = $Node->name();

    #$API_obj = $Node->get_7m_or_nodescope_apiset( connid => 'console' );
    $API_obj = $Node->get_7m_or_nodescope_apiset();
    $API_obj->set_timeout( "connectrec-timeout" => 1200000 );
    $API_obj->execute_raw_command( command => "set -rows 0" );
    $API_obj->execute_raw_command( command => "system timeout modify 0" );
    $API_obj->execute_raw_command( command => "\013" );
    sleep 2;
    $API_obj->execute_raw_command( command   => "\013" );
    $API_obj->execute_raw_command( 'command' => "event config modify -console-log-level INFO -console off" );

    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $API_obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );

    my $shelf;
    my $dr_out = $API_obj->execute_raw_command( 'command' => "run local disk_list" );
    my @drv_list_tahi;

    foreach my $line ( split( /\n/, $dr_out ) ) {
        if ( $line =~ /(\S+\.(\S+)\.\S+\.\S+)/ ) {
            $shelf = $2;
            push( @drv_list_tahi, $1 );
        }
    }

    $shelf = int($shelf);
    my $dsks = _print_array( \@drv_list_tahi );
    logcomment( "Drives list :\n" . $dsks );
    logcomment("Tahiti shelf ID : $shelf");
    $shelf =~ s/0//g if ( $shelf =~ /^0/ );

    #logcomment("DEBUG : Tahiti shelf ID : $shelf");

    $dsk_out = $API_obj->execute_raw_command( command => "storage aggregate show-spare-disks -original-owner $FILER_C" );
    $dsk_out = $API_obj->execute_raw_command( command => "storage aggregate show-spare-disks -original-owner $FILER_C" ) if ( $dsk_out eq '' );
    logcomment("Get Spare drive for Filer : $FILER_C");

    foreach my $line ( split( /\n/, $dsk_out ) ) {
        next if ( $line =~ /Disk|Origina|Pool|Root|Spar|-------|entries/ );
        if ( $line =~ /(\d+\.$shelf.\d+)/ ) {

            #aa6-167:0n.12    SSD-NVM solid-state     - block            3.29TB       0B   3.49TB zeroed
            if ( $line =~ /(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ) {
                my $drive = $1;
                $filer_spare_rst->{$drive}->{'disk'}       = $drive;
                $filer_spare_rst->{$drive}->{'type'}       = $2;
                $filer_spare_rst->{$drive}->{'class'}      = $3;
                $filer_spare_rst->{$drive}->{'rpm'}        = $4;
                $filer_spare_rst->{$drive}->{'chksum'}     = $5;
                $filer_spare_rst->{$drive}->{'usebldata'}  = $6;
                $filer_spare_rst->{$drive}->{'root_usebl'} = $7;
                $filer_spare_rst->{$drive}->{'size'}       = $8;
                $filer_spare_rst->{$drive}->{'status'}     = $9;
            } elsif ( $line =~ /(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ) {
                my $drive = $1;
                $filer_spare_rst->{$drive}->{'disk'}      = $drive;
                $filer_spare_rst->{$drive}->{'type'}      = $2;
                $filer_spare_rst->{$drive}->{'class'}     = $3;
                $filer_spare_rst->{$drive}->{'rpm'}       = $4;
                $filer_spare_rst->{$drive}->{'chksum'}    = $5;
                $filer_spare_rst->{$drive}->{'usebldata'} = $6;
                $filer_spare_rst->{$drive}->{'size'}      = $7;
                $filer_spare_rst->{$drive}->{'status'}    = $8;
            }
        }
    }
    return $filer_spare_rst;
}

sub tahiti_map_fw_dsk_rev_model {
    my (%opts) = validate(
        @_,
        {
            Node             => { type => OBJECT },
            disk_fw_location => { type => SCALAR },
            timeout          => { type => SCALAR, default => 1200, optional => 1 },
        }
    );
    my $Node          = $opts{Node};
    my $disk_fw_path  = $opts{disk_fw_location};
    my $sudo          = "/usr/bin/sudo";
    my $Mode          = $Node->mode();
    my %new_disk_fw   = ();
    my $new_disk_fw_p = [];
    $new_disk_fw_p = \%new_disk_fw;
    my %model   = ();
    my %rev     = ();
    my @disk_fw = `cd $opts{disk_fw_location};ls *.LOD`;
    my @failed_array;
    logcomment("Following Firmwares found in the directory $opts{disk_fw_location}:");
    logcomment("@disk_fw");

    # This function validates the same firmwares with same product id's
    if ( @disk_fw < 1 ) {
        logcomment("NO DISK FIRMWARE FILES ARE AVAILABLE");
        return ($new_disk_fw_p);
    }
    my $disk_list_result = tahiti_get_all_disk( node_present => [$Node] );
    my @dsk              = keys %$disk_list_result;
    my @disk_info        = sort(@dsk);
    foreach my $dsk (@disk_info) {
        my $model_name = $disk_list_result->{$dsk}->{'MODEL'};
        my $rev_num    = $disk_list_result->{$dsk}->{'REV'};
        my $drive_type = $disk_list_result->{$dsk}->{'Type'};
        my $flag       = 0;
        foreach my $fw (@disk_fw) {
            $fw         =~ s/^\s+|\s+$//g;
            $model_name =~ s/^\s+|\s+$//g;
            if ( $fw =~ /^$model_name\.\S+\.LOD$/ ) {
                logcomment("DEBUG:Disk : $dsk ::: Model : $model_name ::: Rve No : $rev_num");
                $flag = 1;
                if ( !( $new_disk_fw{$dsk} ) ) {
                    $new_disk_fw{$dsk} = $fw;
                }
            }    ## if ( $fw =~ /$model_name/ )a
        }
        if ( $flag == 0 ) {
            push( @failed_array, "No firmware file found for the disk $dsk with model number : $model_name and version $rev_num" );
        }
    }
    grep { logcomment("$_") } @failed_array;
    return ($new_disk_fw_p);
}

####################################################################################################
# Tahiti - Drive firmware to lower version, downgrade firmware from /FIRMWARE/DISKFW/LOW
# Return : Test Status ( 1 - Failed | 0 - Pass )
####################################################################################################
sub tahiti_disk_fw_update_low_or_high {
    my (%opts) = validate(
        @_,
        {
            Node        => { type => OBJECT },
            FW_DIR_LOC  => { type => SCALAR },
            FORCE_UPDTE => { type => SCALAR, optional => 1, default => 'yes' },
        }
    );

    my $Node          = $opts{Node};
    my $disk_dir      = $opts{FW_DIR_LOC};
    my $force_update  = $opts{FORCE_UPDTE};
    my $Home          = $ENV{HOME};
    my %firmware_hash = ();
    my @firmware_list = ();
    my $test_status   = 0;
    logcomment("MAP Firmware, Copy Firmware from $disk_dir and force update : $force_update");

    #    my $disk_dir      = "$Home/NDATE/FIRMWARE/DISKFW/LOW";
    my $firmware_list = tahiti_map_fw_dsk_rev_model(
        Node             => $Node,
        disk_fw_location => $disk_dir,
    );
    %firmware_hash = %$firmware_list;
    my %hash;
    my ( $key, $value );
    while ( ( $key, $value ) = each %firmware_hash ) {
        push( @{ $hash{$value} }, $key );
    }
    my @drive_list = grep { (/\S/) } keys(%firmware_hash);
    logcomment("The list of drives on which lower firmware will download : @drive_list ");
    @firmware_list = unique( values(%firmware_hash) );
    logcomment("The list of lower firmware files present : @firmware_list ");
    if ( !@firmware_list ) {
        logcomment("**FAIL**:  NO LOWER FIRMWARE FILES ARE AVAILABLE FOR FILER ");
        $test_status = 1;
        return $test_status;
    }
    logcomment("Copying the FW files to filer directory");
    my $FILER_C = $Node->name();
    logcomment( "Lower firmware files to be copied are " . scalar(@firmware_list) . " - @firmware_list " );
    my @firmwares;
    foreach (@firmware_list) {
        my $val = $disk_dir . "/" . $_;
        push( @firmwares, $val );
    }
    logcomment("Calling Common lib subroutine copy_fw");
    my @fw_files = copy_FW_files_to_filer_directory(
        Node  => $Node,
        Files => \@firmwares
    );
    if ( !@fw_files ) {
        logcomment("**FAIL**: $FILER_C - DISK FIRMWARE FILES ARE NOT FOUND IN /etc/disk_fw");
        $test_status = 1;    # to mark as sub routine fail
        return $test_status;
    }
    logcomment("Firmware files copied, Proceeding with update based on force update");
    logcomment("DEBUG : FORCE UPDATE : $force_update");
    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=1" );
    my $time_out = 10000;
    my @drv_loaded_pas;
    my @fw_dwl_drives;
    my @not_found;
    my $wait_time = 180;    #3min

    if ( $force_update =~ /yes/i ) {
        logcomment("$FILER_C :Firmware option foce update selected : $force_update");
        foreach my $value ( keys %hash ) {
            $Api_Set_Obj->set_timeout( "connectrec-timeout" => $time_out );
            my @arr_t = split( /\./, $value );
            my $prod_id = $arr_t[0];
            logcomment("DEBUG: Executing disk_fw_update -F $value $prod_id");
            my @prompts = ( 'Are you sure you want to' => 'y', );
            my $output = $Api_Set_Obj->execute_raw_command( 'command' => "run local \"disk_fw_update -F $value -t $prod_id\"", "connectrec-match_table" => \@prompts );
            logcomment("Wait for $wait_time seconds");
            sleep($wait_time);

            foreach my $line ( split( /\n/, $output ) ) {

                #STARTING download request  - 1 requests pending to shelf 0 -- disk 0n.8
                if ( $line =~ /STARTING download request/ ) {
                    if ( $line =~ /\s+disk\s+(\S+)/ ) {
                        logcomment("Starting Firmware download on drive : $1");
                        chomp($1);
                        push( @fw_dwl_drives, $1 );
                    }
                    logcomment("Started Firmware download on drives @fw_dwl_drives");
                }
                if ( $line =~ /Firmware is up-to-date/ ) {
                    logcomment("Firmware is up-to-date on all eligible disks");
                }
            }
            logcomment("Check console messages");
            my $c_out = $Api_Set_Obj->execute_raw_command( 'command' => " " );
            logcomment("CONSOLE: $c_out");
            foreach my $line ( split( /\n/, $c_out ) ) {
                if ( $line =~ /Firmware downloaded|dfu.fwDownloaded/ ) {
                    logcomment("DEBUG : $line");
                    if ( $line =~ /disk\s+(\S+)\s+/ ) {
                        my $drv = $1;
                        push( @drv_loaded_pas, $drv );
                    }
                }
            }
        }
        logcomment("Firmware started on @fw_dwl_drives");
        logcomment("FIrmware Downloaded to @drv_loaded_pas");
        foreach my $d (@fw_dwl_drives) {
            logcomment("Checking for Drive : $d");
            my $found = grep /$d/, @drv_loaded_pas;
            if ( $found == 0 ) {
                push( @not_found, $d );
            } else {
                logcomment("Drive $d dfu.fwDownloaded");
            }
        }
        logcomment("**FAIL** : Drive(s) @not_found firmware download message - dfu.fwDownloaded not found") if (@not_found);
    } else {
        logcomment("Valid Fimrware files copied from $disk_dir to filer $FILER_C");
    }

    my $Api_Set_Obj = $Node->get_7m_or_nodescope_apiset( connid => "console" );
    my $prompts_answers = [ ".*Do you want to continue.*" => 'y' ];
    $Api_Set_Obj->execute_raw_command( 'command' => "set test", 'connectrec-match_table' => $prompts_answers );
    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
    $Api_Set_Obj->execute_raw_command( 'command' => "run local priv set -q test" );
    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
    sleep 2;
    $Api_Set_Obj->execute_raw_command( 'command' => "run local dbg level diskfw=0" );
    $Api_Set_Obj->execute_raw_command( 'command' => "\013" );
    sleep 2;

    if ( $force_update =~ /yes/i ) {
        return \@fw_dwl_drives;
    } else {
        return $test_status;
    }
}