# # Copyright (c) 2011 NetApp, Inc., All Rights Reserved # Any use, modification, or distribution is prohibited # without prior written consent from NetApp, Inc. # ## @summary Volume File Task Module ## @author dl-nacl-dev@netapp.com ## @status shared ## @pod here package NACL::STask::_VolumeFile; use strict; use warnings; use NATE::Log qw(log_global); my $Log = log_global(); my $may_enter = $Log->may_enter(); my $may_exit = $Log->may_exit(); use Params::Validate qw(:all); use NATE::Exceptions::Argument qw(:try); use NACL::APISet::Exceptions::TimeoutException qw(:try); use NACL::Exceptions::UnexpectedState qw(:try); use NATE::BaseException qw(:try); use Scalar::Util qw(blessed); BEGIN { use Exporter qw(import); our %EXPORT_TAGS = ( all => [ qw(_start_standard _start_enhanced start wait_for_finish _is_standard_args_call _is_enhanced_args_call wait_for_destination_ready _wait_helper _extract_sfod_path) ] ); our @EXPORT_OK = @{ $EXPORT_TAGS{all} }; } ## end BEGIN =head1 NAME NACL::STask::_VolumeFile =head1 DESCRIPTION C This class is a MixIn to the C and tasks. Classes are able to import the symbols provided by this class to provide that functionality natively within their own class. This class should not be used directly. The functionality provided by this package will be available directly from classes augmenting their class with this mixin. =head1 METHODS =head2 start Start a file operation by either using the standard form supported by the component or by using an enhanced form that allows specification of files via relative paths plus volume component instances. This method defaults to waiting for completion of the operations. # Standard form my @moves = NACL::STask::VolumeFileMove->start( command_interface => $command_interface, "source-paths" => "vs0:vol/path" | ["path1", ...], "destination-paths" => "vs0:vol/path" | ["path1", ...], nacltask_wait => $boolean, # default "1" %other_options ); # Enhanced form my @moves = NACL::STask::VolumeFileMove->start( source_volume => NACL::C::Volume instance, destination_volume => NACL::C::Volume instance, source_filenames => "/path/to/file" | ["path1", ...], %other_options ); my @copies = NACL::STask::VolumeFileCopy->start( source_volume => NACL::C::Volume instance, destination_volume => NACL::C::Volume instance, source_filenames => "/path/to/file" | ["path1", ...], destination_filenames => "/path/to/file" | ["path1", ...], %other_options ); (Class method) Start an ondemand job. This method allows the user to directly pass volume components and relative file paths to the constructor along with waiting for completion. =over =item Options =over =item C<< source-paths >> (Required unless covered by other arguments) A reference to a list of NACL::C::VolumeFile instances or ondemand style paths of vs:volume/dir/file. =item C<< destination-paths >> (Required unless covered by other arguments) A reference to a list of NACL::C::VolumeFile instances or ondemand style paths of vs:volume/dir/file. =item C<< source_volume >> (Enhanced form only) A NACL::C::Volume instance for the volume to be used as the source of the single file ondemand job. =item C<< destination_volume >> (Enhanced form only) A NACL::C::Volume instance for the volume to be used as the destination of the single file ondemand job. =item C<< source_filenames >> (Enhanced form only) Either a path to a single file relative to the volume root or a arrayref of paths relative to the volume root. =item C<< destination_files >> (Optional - Enhanced form only) Either a path to a single file relative to the volume root or a arrayref of paths relative to the volume root. If not specified the source_filenames will be used as the destination resulting in the preservation of the paths and names of the files on the destination. =item C<< nacltask_wait => 0|1 >> (Optional) If 1 (the default), wait for the ondemand op to finish (i.e. waits both for the destination file to be ready and for the scanner to complete) =item C<< nacltask_wait_for_destination_ready => 0 | 1 >> (Optional) This is applicable only if nacltask_wait=0. If this option is set to 1, then only wait till the destination file is ready (don't wait till the scanner has completed) =item C<< nacltask_fail_on_destroy => 0 | 1 >> (Optional; Default is 1) When set to 1 (the default), a scanner-status of C indicates a failure has occurred. For the rare cases where we do not want to consider a scanner-status of C to be a failure, then this should be set to 0. =item C<< nacltask_verify_cutover_time => 0 | 1 >> (Optional; Default is 1) This is only applicable if nacltask_wait or nacltask_wait_for_destination_ready options is set to 1. If nacltask_verify_cutover_time option is set to 1, verify if the cutover time expectation is met. Actual cutover time will be compared with Max cutover time value used for single file ondemand operation. =item etc. All of the other various options to L<< NACL::C::VolumeFileMove->start | lib-NACL-C-VolumeFileMove-pm/start >> L<< NACL::C::VolumeFileCopy->start | lib-NACL-C-VolumeFileCopy-pm/start >> =back =back =head2 wait_for_finish Wait for an ondemand job to finish NACL::STask::VolumeFileMove->wait_for_finish( command_interface => $command_interface, "job-uuid" => ..., "file-index" => ..., ); $volume_file_copy_stask->wait_for_finish(); (Class or instance method) Wait for an ondemand job to finish. =over =item Instance and Static method Options =over =item C<< method-timeout >> (Optional) How long to wait for the job to finish. Defaults to -1 for no timeout. =item other options Any of the standard options to NACL component methods such as the apiset must/should arguments are passed on to the underlying component calls. =back =back =over =item Static method Options =over =item C<< command_interface >> As required to instantiate a NACL::C::VolumeFile{Move,Copy} instance. =item C<< job-uuid >> As required to instantiate a NACL::C::VolumeFile{Move,Copy} instance. =item C<< file-index >> As required to instantiate a NACL::C::VolumeFile{Move,Copy} instance. =back =back =head2 wait_for_destination_ready Wait for a destination file to be ready to access. NACL::STask::VolumeFileMove->wait_for_destination_ready( command_interface => $command_interface, "is-destination-ready" => ..., ); $volume_file_copy_stask->wait_for_destination_ready(); (Class or instance method) Wait for a destination file to be ready to access. =over =item Instance and Static method Options =over =item C<< method-timeout >> (Optional) How long to wait for the job to finish. Defaults to -1 for no timeout. =item other options Any of the standard options to NACL component methods such as the apiset must/should arguments are passed on to the underlying component calls. =back =back =over =item Static method Options =over =item C<< command_interface >> As required to instantiate a NACL::C::VolumeFile{Move,Copy} instance. =item C<< job-uuid >> As required to instantiate a NACL::C::VolumeFile{Move,Copy} instance. =item C<< file-index >> As required to instantiate a NACL::C::VolumeFile{Move,Copy} instance. =back =back =cut sub _start_standard { my ($pkg, @opts) = @_; my %opts = $pkg->_common_validate_with( params => \@opts, additional_spec => { nacltask_wait => { type => SCALAR, default => 1 }, nacltask_verify_cutover_time => { type => BOOLEAN, default => 1 }, # Defaults for these are set in the wait_for_finish method, # so don't duplicate here nacltask_wait_for_destination => { type => BOOLEAN, optional => 1 }, nacltask_fail_on_destroy => { type => BOOLEAN, optional => 1 }, polling_interval => { type => SCALAR, optional => 1 }, }, ignore_primary_keys => 1, allow_extra => 1, ); my $wait = delete $opts{nacltask_wait}; my $destination_wait = delete $opts{nacltask_wait_for_destination}; my %wait_args; $pkg->_hash_move( source => \%opts, target => \%wait_args, move => [qw(polling_interval nacltask_fail_on_destroy nacltask_verify_cutover_time)] ); # Convert any NACL::C::VolumeFile instances in {source,destination}-paths foreach my $key (qw(source-paths destination-paths)) { # Don't modify the hash reference in place since the caller might not # appreciate us changing that. if ( ref $opts{$key} ) { my @translated_args = (); foreach my $file (@{$opts{$key}}) { my $sfod_path = $pkg->_extract_sfod_path($file); push @translated_args, $sfod_path; } $opts{$key} = \@translated_args; } } if ( defined $opts{'reference-path'} ) { $opts{'reference-path'} = $pkg->_extract_sfod_path($opts{'reference-path'}); } my @operations = $pkg->C_SUPER::start(%opts); if ($wait || $destination_wait) { # Occasionally it takes a little time for the copy/move table to be # updated with this operation, so doing a show/get-iter would not show # any entries. Let's give a couple of seconds to ensure that the # table is populated. Tharn::snooze(5); $pkg->_copy_common_component_params( source => \%opts, target => \%wait_args ); if ($wait) { foreach my $operation (@operations) { $operation->wait_for_finish(%wait_args); } } else { foreach my $operation (@operations) { $operation->wait_for_destination_ready(%wait_args); } } } return (wantarray) ? @operations : $operations[0]; } ## end sub _start_standard sub _start_enhanced { my $pkg = shift; my %opts = @_; if ( !exists $opts{command_interface} ) { # Allow the user to override the command_interface from the source push @_, ( command_interface => $opts{source_volume}->command_interface() ); } ## end if ( !exists $opts{command_interface...}) %opts = $pkg->_common_validate_with( params => \@_, additional_spec => { source_volume => { isa => 'NACL::C::Volume' }, destination_volume => { isa => 'NACL::C::Volume' }, source_filenames => { type => SCALAR | ARRAYREF }, destination_filenames => { optional => 1, type => SCALAR | ARRAYREF }, }, ignore_primary_keys => 1, allow_extra => 1, ); my $source_volume = delete $opts{source_volume}; my $destination_volume = delete $opts{destination_volume}; # Convert a single scalar to a list my $source_filenames = delete $opts{source_filenames}; if ( !ref $source_filenames ) { $source_filenames = [$source_filenames]; } my $destination_filenames; # Fill in destination filenames with source names if it doesn't exist if ( $opts{destination_filenames} ) { $destination_filenames = delete $opts{destination_filenames}; } else { $destination_filenames = $source_filenames; } # Convert a single scalar to a list if ( !ref $destination_filenames ) { $destination_filenames = [$destination_filenames]; } my $source_paths = []; my $destination_paths = []; # Convert the filename to a vserver:volume/filepath foreach my $source_file (@$source_filenames) { my $normalized_file = $source_file; if ( $normalized_file !~ /^\// ) { $normalized_file = "/$normalized_file"; } my $path = $source_volume->vserver() . ":" . $source_volume->volume() . $normalized_file; push @$source_paths, $path; } ## end foreach my $source_file (@$source_filenames) # Convert the filename to a vserver:volume/filepath foreach my $destination_file (@$destination_filenames) { my $normalized_file = $destination_file; if ( $normalized_file !~ /^\// ) { $normalized_file = "/$normalized_file"; } my $path = $destination_volume->vserver() . ":" . $destination_volume->volume() . $normalized_file; push @$destination_paths, $path; } ## end foreach my $destination_file... return $pkg->_start_standard( command_interface => $opts{command_interface}, 'source-paths' => $source_paths, 'destination-paths' => $destination_paths, %opts, ); } ## end sub _start_enhanced sub start { $Log->enter() if $may_enter; # We don't know enough to validate here my ($pkg, %opts) = @_; # Copy all of the possible required options for different forms. We will # then decide which form is being used to make the call my %validate_options; $pkg->_hash_copy( source => \%opts, copy => [ qw( command_interface source-paths destination-paths reference-path source_volume destination_volume source_filenames ) ], target => \%validate_options, ); my @od_jobs; if ( $pkg->_is_standard_args_call(%validate_options) ) { @od_jobs = $pkg->_start_standard(%opts); } elsif ( $pkg->_is_enhanced_args_call(%validate_options) ) { @od_jobs = $pkg->_start_enhanced(%opts); } else { NATE::Exceptions::Argument->throw( "Insufficient arguments to start, please use either the " . "standard or enhanced form but not combination of the two" ); } ## end else [ if ( $pkg->_is_standard_args_call...)] $Log->exit() if $may_exit; return (wantarray) ? @od_jobs : $od_jobs[0]; } ## end sub start sub wait_for_finish { $Log->enter() if $may_enter; my ($pkg_or_obj, @args) = @_; my $check_coderef = sub { my ($state) = validate_pos(@_, { type => OBJECT, isa => 'NACL::CS::ComponentState::ONTAP' }); return ($state->scanner_status() =~ /complete/i); }; $pkg_or_obj->_wait_helper(@args, _check_coderef => $check_coderef); $Log->exit() if $may_exit; } ## end sub wait_for_finish sub wait_for_destination_ready { $Log->enter() if $may_enter; my ($pkg_or_obj, @args) = @_; my $check_coderef = sub { my ($state) = validate_pos(@_, { type => OBJECT, isa => 'NACL::CS::ComponentState::ONTAP' }); return ($state->is_destination_ready() =~ /true/i); }; $pkg_or_obj->_wait_helper(@args, _check_coderef => $check_coderef); $Log->exit() if $may_exit; } ## end sub wait_for_destination_ready sub _wait_helper { $Log->enter() if $may_enter; my ($pkg_or_obj, @opts) = @_; require NACL::CS::EventLog; my %opts = $pkg_or_obj->_common_validate_with( params => \@opts, additional_spec => { 'method-timeout' => { type => SCALAR, default => -1, }, polling_interval => { type => SCALAR, default => 5 }, nacltask_fail_on_destroy => { type => BOOLEAN, default => 1 }, nacltask_verify_cutover_time => { type => BOOLEAN, default => 1 }, _check_coderef => { type => CODEREF }, }, allow_extra => 1, ); my $fail_on_destroy = delete $opts{nacltask_fail_on_destroy}; my $polling_interval = delete $opts{polling_interval}; my $check_coderef = delete $opts{_check_coderef}; my $verify_cutover_time = delete $opts{nacltask_verify_cutover_time}; my $filter = {}; $pkg_or_obj->_move_primary_keys(source => \%opts, target => $filter); my $job_uuid = $filter->{'job-uuid'}; my $file_index = $filter->{'file-index'}; # Call this sub if final state is destroy. This sub throws an # exception if fail_on_destroy is set. my $destroy_sub = sub { # If status is "destroy", then the operation is complete. # Whether this terminal state is a failure is controlled # by the "nacltask_fail_on_destroy" flag. my ($reason) = validate_pos(@_, { type => SCALAR }); if ($fail_on_destroy) { $Log->exit() if $may_exit; NACL::Exceptions::UnexpectedState->throw( "Job: $job_uuid; index: $file_index is in destroy state." . "\nFailure Reason: $reason", monitoring_fields => [qw(scanner_status failure_reason)], scanner_status => 'Destroyed', failure_reason => $reason, ); } }; my $end_time = Tharn::timeout2time($opts{'method-timeout'}); while (1) { my $od_state = $pkg_or_obj->get_CS_package_name->fetch( %opts, requested_fields => [qw(scanner-status job-uuid file-index last-failure-reason is-destination-ready max-cutover-time cutover-time)], filter => $filter, allow_empty => 1, apiset_must => { interface => 'CLI' }, ); if ($od_state) { my $scanner_status = $od_state->scanner_status(); my $done = $check_coderef->($od_state); if($done && $verify_cutover_time) { my $actual_cutover_time = $od_state->cutover_time(); my $max_cutover_time = $od_state->max_cutover_time(); if($actual_cutover_time > $max_cutover_time) { # Please consider changing default value of # nacltask_verify_cutover_time to 0 if following # needs to be changed from warning to failure. $Log->warn("SFOD cutover time exceeded by " . ($actual_cutover_time - $max_cutover_time) . " seconds. Actual cutover time:". $actual_cutover_time . ", Max cutover time:" . $max_cutover_time . ". Please file a burt if this is unexpected."); } } last if $done; if ($scanner_status =~ /destroyed/i) { $destroy_sub->($od_state->last_failure_reason()); # If reached here, then no exception thrown, so # fail_on_destroy=0, so break out of the loop. last; } elsif ($scanner_status =~ qr/^Paused_Error$/i) { $Log->exit() if $may_exit; NACL::Exceptions::UnexpectedState->throw( "Job: $job_uuid; index: $file_index is in Paused_Error", monitoring_fields => ['scanner_status'], scanner_status => $od_state->scanner_status(), ); } if ( time() > $end_time ) { # Throw timeout error $Log->exit() if $may_exit; NACL::APISet::Exceptions::TimeoutException->throw( "Timeout of " . $opts{'method-timeout'} . " seconds " . "expired waiting for ondemand job to finish" ); } ## end if ( time() > $end_time) Tharn::sleep($polling_interval); } else { # On-Demand operation entry no longer exists, so has completed. # We need to check EMS for whether it was successful or ended # up in the destroy state. # (The entry would no longer exist if the status is queried for # more than 120s after the operation has completed) my @events = NACL::CS::EventLog->fetch( %opts, # No direct way to filter for the exact EMS message for this # job, so filter for all jobs of this type (wafl.ondemand.exception) # and then iterate through till we find the one for this job. filter => { messagename => 'wafl.ondemand.exception' }, requested_fields => ['event'], allow_empty => 1, ); # Create a regex that includes a list of the EMS messages # that should be excluded from the destroy state check because # they are transient and do not indicate that the job has completed. my $exclude_regex = _create_ems_exclusion_regex(); # Message will look like this: # "wafl.ondemand.exception: Cannot create destination file. No # space is left on volume with DSID 1033. File copy job # 73c40d0e-cf86-4462-8a2f-39e4c575f576-0." # The last portion is - (job_uuid followed # by hyphen followed by file_index) foreach my $ems_obj (@events) { my $event = $ems_obj->event(); if ($event =~ /file.*job.*$job_uuid\-$file_index/i && $event !~ $exclude_regex) { $destroy_sub->($event); } } # If we've reached here, then either the job succeeded, or it # failed and nacltask_fail_on_destroy was set to 0. Break out # of the while loop since we're done either way. last; } } ## end while (1) $Log->exit() if $may_exit; } sub _is_standard_args_call { $Log->enter() if $may_enter; my $self = shift; my $return = 0; eval { Params::Validate::validate( @_, { command_interface => { required => 1 }, 'source-paths' => { required => 1 }, 'destination-paths' => { required => 1 }, 'reference-path' => { type => SCALAR|OBJECT , optional => 1 }, } ); $return = 1; }; $Log->exit() if $may_exit; return $return; } ## end sub _is_standard_args_call sub _is_enhanced_args_call { $Log->enter() if $may_enter; my $self = shift; my $return = 0; eval { Params::Validate::validate( @_, { source_volume => { required => 1 }, destination_volume => { required => 1 }, source_filenames => { required => 1 }, } ); $return = 1; }; $Log->exit() if $may_exit; return $return; } ## end sub _is_enhanced_args_call sub _create_ems_exclusion_regex { # Build a regular expression of EMS messages # that should be excluded when determining if job # is in a destroy state - these are transient EMS # messages that don't imply a terminal state # These messages are defined in # /dev/mgmtgateway/src/zapis/ErrorCodeDefs.cc my @exclusions = ( ".*Waiting for lock revocation.*", ".*Data scanner delayed.*", ".*Job delayed.*", ".*Waiting for available space.*", ".*Waiting for .* to complete.*", ".*Waiting for the network.*", ".*Waiting for snapshot create to complete.*", ".*Destroying remaining hard links.*", ".*Need to write allocation map.*", ".*Waiting for the associated reference transfer to cut over.*", ".*Waiting for associated incremental transfers to complete their " . "allocation phases.*", ".*Waiting for rename operations.*", ".*Waiting for snapshot restore to complete.*", ".*Waiting for data to be flushed to disk.*", ".*Waiting for related jobs to complete their preparation phases.*", ".*Waiting for an IO fence to be released.*", ".*Not ready.*", ".*Volume with DSID .* is offline.*", ); my $regex = join('|', @exclusions); return qr/$regex/; } ## ens sub _create_ems_exclusion_regex sub _extract_sfod_path { $Log->enter() if $may_enter; my ($pkg_or_obj, $file) = @_; my $file_to_return; if (blessed ($file)) { if ($file->isa('NACL::C::VolumeFile')) { $file_to_return = $file->get_sfod_path(); } else { $Log->exit() if $may_exit; NATE::Exceptions::Argument->throw('The file should have been ' . 'either a path or an object of type NACL::C::VolumeFile ' . "but what was provided was of type '" . ref ($file) . "'" ); } } else { $file_to_return = $file; } $Log->exit() if $may_exit; return $file_to_return; } ## end sub _extract_sfod_path 1;