# # Copyright (c) 2001-2014 NetApp, Inc., All Rights Reserved # Any use, modification, or distribution is prohibited # without prior written consent from NetApp, Inc. # ## @summary Cleanup Manager ## @author dl-nacl-dev@netapp.com ## @status shared ## @pod here =head1 NAME NACL::Cleanup =head1 DESCRIPTION Cleanup subsystem to manage NACL Resources. It is derived from NACL::ResourceList, and provides a mechanism to cleanup NACL Resources. The Cleanup Manager is an object that manages a Resource List. It provides methods to run registered cleanup methods, purge the database. The table Test_Execution_Details will have the execution log name along with the path. =cut package NACL::Cleanup::Database; use strict; use warnings; use Params::Validate qw( validate validate_with :types); use NATE::BaseException qw(:try); use Tharn qw( param); use NATE::Log qw(log_global); my $Log = log_global(); my $may_enter = $Log->may_enter(); my $may_exit = $Log->may_exit(); use NACL::ResourceList::Schema; use NACL::ComponentUtils qw(Dumper); use NATE::ParamSet qw(param_global); use NACL::Cleanup::Database::MySQL; use NACL::Cleanup::Database::SQLite; use Storable qw(nfreeze thaw); use File::Spec; use File::Basename; use NATE::ParamSet qw(param_global); my $nate_params = NATE::ParamSet::param_global; use NATE::Events qw(call_on_fork_add); #use Scalar::Util qw(weaken); call_on_fork_add( { callback => sub {my $self = shift; $self->{command_interfaces} = ()}, keep_on_fork => 1, }, ); use Class::MethodMaker [ scalar => [ { '-static' => 1, '-default_ctor' => sub { return NATE::ParamSet::param_global->get('NATE_DB_DRIVER'); } }, 'nate_db_driver' ], scalar => [ { '-static' => 1, '-default_ctor' => sub { return NATE::ParamSet::param_global->get('NACL_CLEANUP_DRIVER'); } }, 'nacl_cleanup_driver' ], scalar => [ { '-static' => 1, '-default_ctor' => sub { return NATE::ParamSet::param_global->get('NATE_DB_NAME'); } }, 'nate_db_name' ], scalar => [ { '-static' => 1, '-default_ctor' => sub { return NATE::ParamSet::param_global->get('TOP_LOGDIR') || NATE::ParamSet::param_global->get("NATE_DB_DIR_TEMP"); } }, 'nate_log_dir' ], scalar => [ { '-static' => 1, '-default_ctor' => sub { $_[0]->nate_db_driver() =~ /mysql/i }, }, 'is_mysql' ], scalar => [ { '-static' => 1, '-default_ctor' => sub { if ($_[0]->nacl_cleanup_driver) { $_[0]->nacl_cleanup_driver =~ /sqlite/i; } else { $_[0]->nate_db_driver() =~ /sqlite/i; } } }, 'is_sqlite' ], scalar => 'deploy_schema', scalar => 'schema', scalar => 'dsn', scalar => 'db_name', scalar => 'user', scalar => 'password', scalar => 'random_db', ]; sub new { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { db_name => { type => SCALAR }, deploy_schema => { type => SCALAR, optional => 1, default => 1 } }, allow_extra => 1 ); my $db_object = $self->get_db_object( %opts ); # i believe this is due to a mysql limitation of 64 character db names if(length(basename($db_object->{db_name})) > 64) { my $exception_text = "size of db_name $db_object->{db_name} is more than 64.please modify the db_name"; NATE::BaseException->throw($exception_text); } $db_object->create_schema(); my @package_attributes_to_set = qw ( dsn deploy_schema ); foreach my $attribute (@package_attributes_to_set){ $db_object->$attribute($opts{$attribute}) if(defined $opts{$attribute}); } if($self->deploy_schema()){ $self->_update_test_execution_info(); } $db_object->{command_interfaces} = (); $Log->exit() if $may_exit; return $db_object; } sub _dbi_call_wrapper { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { code_ref => { type => CODEREF }, report_error_as => { type => SCALAR, default => "exception" }, } ); my $code_ref = delete $opts{'code_ref'}; my $report_error_as = lc( delete $opts{'report_error_as'} ); eval { &$code_ref; }; if ($@) { my $error = "Error in db transaction: " . Dumper($@); $Log->exit() if $may_exit; if ( $report_error_as eq "exception" ) { NATE::BaseException->throw($error); } elsif ( $report_error_as eq "debug" ) { $Log->debug($error) if ( $Log->may_debug() ); } elsif ( $report_error_as eq "comment" ) { $Log->comment($error); } else { $Log->comment( "report_error_as : $report_error_as is unknown so throwing an exception." ); NATE::BaseException->throw($error); } } $Log->exit() if $may_exit; } sub create_schema { $Log->enter() if $may_enter; my ( $self, %opts ) = @_; my @conn_params = ( $self->dsn(), $self->user(), $self->password() ); my $deploy_schema = $self->should_schema_be_deployed(); my $schema; $self->_dbi_call_wrapper( 'code_ref' => sub { $schema = NACL::ResourceList::Schema->connect( @conn_params, { AutoCommit => 1, auto_savepoint => 1, RaiseError => 1 }, ); } ); if ($deploy_schema) { $self->_dbi_call_wrapper( 'code_ref' => sub { $schema->deploy( { add_drop_table => 1, }, ); } ); $self->deploy_schema($deploy_schema); } $Log->exit() if $may_exit; $self->schema($schema); } sub _get_command_interface { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { element_id => { type => SCALAR }, } ); my $element_id = $opts{'element_id'}; my $schema = $self->schema(); my %host = $schema->resultset('Host')->search( { 'elements.id' => $element_id }, { join => 'elements', distinct => 1, order_by => { -desc => ' me.id ' } } )->first->get_columns(); my $name = $host{'name'}; my $ci = $self->{command_interfaces}->{$name}; if (!defined $ci) { $ci = thaw( $host{'cmd_interface'} ); if(!defined $ci) { require NACL::C::Node; $ci = NACL::C::Node->new(node => $name); $Log->warn("The filer ontap Version is: " . $ci->get_version_manager()->get_version_attribute(attribute => 'release')); } $self->{command_interfaces}->{$name} = $ci; # weaken $command_interfaces{$name}; } $Log->exit() if $may_exit; return $ci; } sub _get_package { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { 'element_id' => { type => SCALAR }, } ); my $id = delete $opts{'element_id'}; my $schema = $self->schema(); my $rs; $self->_dbi_call_wrapper( 'code_ref' => sub { $rs = $schema->resultset('Element')->search( { 'me.id' => $id }, { prefetch => 'package', , columns => [ { 'package' => 'package.package' }, ], } ); } ); my $result_obj = $rs->single(); my $package = $result_obj->get_column('package'); $Log->exit() if $may_exit; return $package; } sub _get_element_ids_to_cleanup { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate( @args, { objects => { type => ARRAYREF }, }, ); my $schema = $self->schema(); my @objects = ( @{ $opts{'objects'} } ); my %elements_to_cleanup; foreach my $obj (@objects) { my %primary_keys_hash = $obj->_get_primary_keys_w_or_wo_ci_from_pkg_or_obj( want_ci => 1 ); my $ci = delete $primary_keys_hash{'command_interface'}; my $package = ref $obj; my $host_id = $self->_get_host_id( 'name' => $ci->name() ); my $element_id = $self->_get_element_id( 'query' => \%primary_keys_hash, 'package' => $package, 'host_id' => $host_id ); if ($element_id) { my %host; $self->_dbi_call_wrapper( 'code_ref' => sub { %host = $schema->resultset('Host')->search( { 'elements.id' => $element_id }, { join => 'elements', distinct => 1, order_by => { -desc => ' me.id ' } } )->first->get_columns(); } ); if ( $ci->name() eq $host{'name'} ) { $elements_to_cleanup{$element_id} = 1; } } else { $Log->warn( "following object passed, is not registered for cleanup." . Dumper($obj) ); } } $Log->exit() if $may_exit; return \%elements_to_cleanup; } sub _get_element_id { my ( $self, @args ) = @_; my %opts = validate( @args, { query => { type => HASHREF }, host_id => { type => SCALAR }, package => { type => SCALAR }, }, ); my $num = scalar keys %{ $opts{'query'} }; my $package = delete $opts{'package'}; my $host_id = delete $opts{'host_id'}; my $schema = $self->schema(); my $package_rs = $self->fetch( 'table' => 'Package', 'where' => { 'me.package' => $package }, 'select' => ['id'] ); my $package_id = $package_rs->get_column('id')->single; my ( $rs, $rs2 ); $self->_dbi_call_wrapper( 'code_ref' => sub { $rs = $schema->resultset('PrimaryKey')->search( {}, { group_by => [qw/ me.element_id /], select => [ 'me.element_id', { count => 'me.element_id', -as => 'countx' } ], having => "countx = $num" } ); } ); my ( %where, $count ); for ( $count = 1; my ( $key, $value ) = each %{ $opts{query} }; $count++ ) { my $tag = ( $count == 1 ) ? "primary_keys" : "primary_keys_$count"; $where{"${tag}.attribute"} = $key; $where{"${tag}.value"} = $value; $where{"${tag}.package_id"} = $package_id; } $where{'element.host_id'} = $host_id; $self->_dbi_call_wrapper( 'code_ref' => sub { $rs2 = $rs->search_related( 'element', \%where, { join => [ ("primary_keys") x $count ], distinct => 1, columns => [qw/element.id/], order_by => { -desc => 'element.id' }, } ); } ); my $id = $rs2->get_column('id')->single; return $id ? $id : 0; } sub fetch { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { table => { type => SCALAR }, where => { type => HASHREF, default => {} }, select => { type => ARRAYREF, default => [] }, } ); my $schema = $self->schema(); my $where = $opts{'where'}; my $table = $opts{'table'}; my $select = $opts{'select'}; my $rs; $self->_dbi_call_wrapper( 'code_ref' => sub { $rs = $schema->resultset($table) ->search( $where, { 'select' => $select, order_by => { -desc => 'me.id' } } ); } ); $Log->exit() if $may_exit; return $rs ? $rs : 0; } sub _insert_into_Element { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { package_id => { type => SCALAR }, group_id => { type => SCALAR, optional => 1 }, host_id => { type => SCALAR }, }, ); my %record = ( table => 'Element', row_info => \%opts ); return $self->_do_insert_txn( %record, insert_method => "new" ); } sub _do_insert_txn { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { table => { type => SCALAR }, row_info => { type => HASHREF }, insert_method => { type => SCALAR, default => "find_or_new" }, no_id => { type => BOOLEAN, default => 0, optional => 1 }, }, ); my $table = delete $opts{'table'}; my $no_id = delete $opts{'no_id'}; my %row_info = %{ delete $opts{'row_info'} }; my $insert_method = delete $opts{'insert_method'}; my $schema = $self->schema(); my $row; eval { $row = $schema->resultset($table)->$insert_method( {%row_info} ) }; if($@) { my $error = "Error in db transaction: " . Dumper($@); NATE::BaseException->throw($error); } $self->_dbi_call_wrapper( 'code_ref' => sub { $row = $schema->resultset($table)->$insert_method( {%row_info} ); my $in_storage = $row->in_storage(); eval { $row->insert }; if ($@) { my $error = $@; if( (! defined $in_storage || $in_storage == 0 ) && $error =~ /Duplicate entry/i){ $Log->debug("Duplicate Entry Error in db transaction : $error"); $row = $schema->resultset($table)->$insert_method( {%row_info} ); } else { die $error; } }} ); return $row->get_column('id') if(! $no_id); } sub _insert_into_command_history { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { element_id => { type => SCALAR }, command_id => { type => SCALAR }, }, ); my %record = ( table => 'CommandHistory', row_info => \%opts ); return $self->_do_insert_txn( %record, insert_method => "new" ); } sub _insert_into_primary_key { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { element_id => { type => SCALAR }, package_id => { type => SCALAR }, attributes => { type => HASHREF }, }, ); my $element_id = delete $opts{'element_id'}; my $package_id = delete $opts{'package_id'}; my %attributes = %{ $opts{'attributes'} }; while ( my ( $attribute, $value ) = each %attributes ) { my %record = ( table => 'PrimaryKey', row_info => { element_id => $element_id, package_id => $package_id, attribute => $attribute, value => $value } ); $self->_do_insert_txn(%record); } } sub _insert_into_argument { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { command_seq_id => { type => SCALAR }, arguments => { type => HASHREF }, }, ); my $command_seq_id = delete $opts{'command_seq_id'}; my %arguments = %{ $opts{'arguments'} }; while ( my ( $argument, $value ) = each %arguments ) { my $frozen_form; if (ref $value) { $frozen_form = nfreeze ($value); } else { # Needs to be a reference for nfreeze() $frozen_form = nfreeze (\$value); } my %record = ( table => 'Argument', row_info => { command_seq_id => $command_seq_id, argument => $argument, value => $frozen_form, } ); $self->_do_insert_txn(%record); } } sub _insert_into_group { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { group => { type => SCALAR }, }, allow_extra => 1, ); my $group = delete $opts{'group'}; my %record = ( table => 'Group', row_info => { name => $group } ); return $self->_do_insert_txn(%record); } sub _insert_into_command { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { command => { type => SCALAR }, }, allow_extra => 1, ); my $command = delete $opts{'command'}; my %record = ( table => 'Command', row_info => { command => $command } ); return $self->_do_insert_txn(%record); } sub _insert_into_package { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { package => { type => SCALAR }, }, allow_extra => 1, ); my $package = delete $opts{'package'}; my %record = ( table => 'Package', row_info => { package => $package } ); return $self->_do_insert_txn(%record); } sub _insert_into_host { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { name => { type => SCALAR }, 'cmd_interface' => { isa => 'NACL::C::CommandInterface', type => OBJECT }, }, allow_extra => 1, ); my $name = delete $opts{'name'}; my $cmd_interface = delete $opts{'cmd_interface'}; # _disconnect('command_interface' => $cmd_interface); my $frozen_ci = nfreeze($cmd_interface); my %record = ( table => 'Host', row_info => { name => $name, 'cmd_interface' => $frozen_ci } ); return $self->_do_insert_txn(%record); } sub update_element { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { command => { type => SCALAR }, element_id => { type => SCALAR }, arguments => { type => HASHREF, default => {} } }, ); my $command = $opts{command}; my $element_id = $opts{'element_id'}; my %arguments = %{ $opts{'arguments'} }; if ( !scalar %{ $opts{'arguments'} } ) { $Log->debug( "No argument data is provided! Hence no arguments are registered for" . $command ) if ( $Log->may_debug() ); } my $command_id = $self->_insert_into_command( 'command' => $command ); my $command_seq_id = $self->_insert_into_command_history( 'element_id' => $element_id, 'command_id' => $command_id ); $self->_insert_into_argument( 'command_seq_id' => $command_seq_id, 'arguments' => \%arguments ); $Log->exit() if $may_exit; } sub _purge_a_element_history { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { command_seq_id => { type => SCALAR }, } ); my $command_seq_id = $opts{'command_seq_id'}; my $schema = $self->schema(); my ( $element_rs, $primary_keys_for_element_ID, $command_history_rs, $argument_rs ); # purge the data in CommandHistory table $self->_dbi_call_wrapper( 'code_ref' => sub { $command_history_rs = $schema->resultset('CommandHistory') ->search( { 'me.id' => $command_seq_id }, {} ); } ); # purge the data in Argument table $self->_dbi_call_wrapper( 'code_ref' => sub { $argument_rs = $schema->resultset('Argument') ->search( { 'me.command_seq_id' => $command_seq_id }, {} ); } ); $self->_delete_txn( 'result_set' => $command_history_rs, 'ignore_error' => 1 ); $self->_delete_txn( 'result_set' => $argument_rs, 'ignore_error' => 1 ); $Log->exit() if $may_exit; } sub _delete_txn { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { result_set => { isa => 'DBIx::Class::ResultSet' }, ignore_error => { type => SCALAR, default => 0 } } ); my $rs = delete $opts{'result_set'}; my $report_error_as = $opts{'ignore_error'} ? "debug" : "exception"; $self->_dbi_call_wrapper( 'code_ref' => sub { $rs->delete(); }, 'report_error_as' => $report_error_as ); $Log->exit() if $may_exit; } sub _purge_an_element { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { primary_keys => { type => HASHREF }, 'package' => { type => SCALAR }, 'host_id' => { type => SCALAR } } ); my $primary_keys = $opts{'primary_keys'}; my $package = $opts{'package'}; my $host_id = $opts{'host_id'}; my $schema = $self->schema(); my $element_id = $self->_get_element_id( 'query' => $primary_keys, 'package' => $package, 'host_id' => $host_id ); if ($element_id) { my ( $element_rs, $primary_keys_for_element_ID, $command_history_rs, $argument_rs ); # purge the data in Element table $self->_dbi_call_wrapper( 'code_ref' => sub { $element_rs = $schema->resultset('Element') ->search( { 'me.id' => $element_id }, {} ); } ); # purge the data in PrimaryKey table $self->_dbi_call_wrapper( 'code_ref' => sub { $primary_keys_for_element_ID = $schema->resultset('PrimaryKey') ->search( { 'me.element_id' => $element_id }, {} ); } ); $self->_delete_txn( 'result_set' => $element_rs, 'ignore_error' => 1 ); $self->_delete_txn( 'result_set' => $primary_keys_for_element_ID, 'ignore_error' => 1 ); } $Log->exit() if $may_exit; } sub _do_update_txn { my ( $self, @args ) = @_; $Log->enter() if $may_enter; my %opts = validate_with( params => \@args, spec => { table => { type => SCALAR }, data => { type => HASHREF }, where => { type => HASHREF }, }, ); my $table = $opts{'table'}; my %data = %{ $opts{'data'} }; my %where = %{ $opts{'where'} }; my $schema = $self->schema(); while ( my ( $key, $val ) = each %data ) { $self->_dbi_call_wrapper( 'code_ref' => sub { my $row = $schema->resultset($table)->find( \%where ); if($row){ $row->$key($val); $row->update(); } } ); } $Log->exit() if $may_exit; } sub _get_host_id { $Log->enter() if $may_enter; my ( $self, @args ) = @_; my %opts = validate( @args, { name => { type => SCALAR }, }, ); my $name = $opts{'name'}; my $schema = $self->schema(); my $rs; $self->_dbi_call_wrapper( 'code_ref' => sub { $rs = $schema->resultset('Host') ->search( { 'me.name' => $name } ); }, ); my $host_id = 0; if ( $rs->count() ) { $host_id = $rs->first->get_column('id'); } $Log->exit() if $may_exit; return $host_id; } sub _get_table_data { my ( $self, @args ) = @_; my %opts = validate_with( params => \@args, spec => { table => { type => SCALAR }, where => { type => HASHREF, default => {} }, select => { type => ARRAYREF, default => [] }, }, ); my @table_data; my $packages_rs = $self->fetch(%opts); while ( my $row = $packages_rs->next() ) { my %data = $row->get_columns(); push @table_data, \%data; } $Log->exit() if $may_exit; return \@table_data; } sub drop_database { $Log->enter() if $may_enter; my ( $self ) = @_; if ( $self->is_mysql() || $self->is_sqlite() ) { $self->_drop_database(); } else { &_throw_unsupported_db_exception($self->nate_db_driver()); } $Log->exit() if $may_exit; } sub get_db_object { $Log->enter() if $may_enter; my ( $self, %opts ) = @_; if ( $self->is_sqlite() ) { return NACL::Cleanup::Database::SQLite->new(%opts); } elsif ( $self->is_mysql() ) { return NACL::Cleanup::Database::MySQL->new(%opts); } else { &_throw_unsupported_db_exception($self->nate_db_driver()); } $Log->exit() if $may_exit; } sub extract_filename_from_path { $Log->enter() if $may_enter; my ( $self, $path_or_file ) = @_; my ( $volume, $directory, $file ) = File::Spec->splitpath($path_or_file); my $logdir = $self->nate_log_dir(); chop $directory; if ( $directory && $directory ne $logdir ) { $Log->warn( "directory provided ( $directory ) will be ignored and the " . "db will be created at $logdir/$file" ); } $Log->exit() if $may_exit; return $file; } sub _update_test_execution_info { my ($self) = @_; my $log_dir = $self->nate_log_dir(); my %data = ( subject => 'log directory path', detail => $log_dir, ); my %record = ( table => 'TestExecutionDetails', row_info => \%data, no_id => 1, ); return $self->_do_insert_txn( %record ); } sub _throw_unsupported_db_exception { my $nate_db_driver = shift; my $msg = "DB Driver: " . $nate_db_driver . " is not currently supported by NACL::Cleanup\n"; $msg .= "If this needs to be supported, please raise a RFE with type = NACL and sub_type = nacl_core."; NATE::BaseException->throw($msg); } sub _truncate_database { my ( $self ) = @_; my $schema = $self->schema(); ### "Element" should be after "Argument" and "CommandHistory" to prevent issues such as on burt 951266 my @tables = qw/PrimaryKey Argument CommandHistory Element Command Group Host Package /; foreach my $table (@tables) { my $rs = $self->fetch( 'table' => $table ); $rs->delete_all() if ($rs); } } sub _db_name_with_unique_id { my ($self, %opts ) = @_; $opts{name} =~ s/\.db$//; my $db_name = $opts{name}; my $nate_db_name = $self->nate_db_name(); my $uid = $nate_db_name; $uid =~ s/test//; $db_name =~ s/$uid//g; $nate_db_name =~ s/test/$db_name/; $db_name = $nate_db_name; $db_name .= '.db' if($self->is_sqlite()); return $db_name; } sub _get_nate_uuid { my ($self, %opts ) = @_; my $nate_db_name = $self->nate_db_name(); my $uid = $nate_db_name; $uid =~ s/test//; return $uid; } sub _get_ci_hash { my $self = shift; return $self->{command_interfaces}; } sub does_database_exists { my ( $self, %opts ) = @_; if ( $self->is_sqlite() ) { return NACL::Cleanup::Database::SQLite->_does_database_exists(%opts); } elsif ( $self->is_mysql() ) { return NACL::Cleanup::Database::MySQL->_does_database_exists(%opts); } else { &_throw_unsupported_db_exception($self->nate_db_driver()); } } sub clear_ci_hash { my $self = shift; $Log->enter() if $may_enter; foreach my $ci ( values %{$self->{command_interfaces}}){ _disconnect('command_interface' => $ci); } $self->{command_interfaces} = (); $Log->exit() if $may_exit; } sub _disconnect { my @args = @_; my %opts = validate_with( params => \@args, spec => { 'command_interface' => { type => OBJECT }, }, allow_extra => 1, ); my $command_interface = $opts{'command_interface'}; foreach my $apiset ( $command_interface->_apisets_values() ) { # disconnect only for CLI connections if ( $apiset->{interface} eq 'CLI' ) { $apiset->get_connection()->destroy(); } } } 1;