# $Id: //depot/prod/test/nacldev/lib/NACL/MTask/ShelfLogMonitoring.pm#1 $
#
#   Copyright (c) 2012 NetApp, Inc., All Rights Reserved
#   Any use, modification, or distribution is prohibited
#   without prior written consent from NetApp, Inc.
#

## @summary ShelfLogMonitoring Task Module
## @author sangavai.p@netapp.com, dl-nacl-dev@netapp.com
## @status shared
## @pod here

package NACL::MTask::ShelfLogMonitoring;

use strict;
use warnings;

use base qw(NACL::MTask::MTask);

use NATE::Log qw(log_global);
my $Log = log_global();
my $may_enter = $Log->may_enter();
my $may_exit  = $Log->may_exit();
use Params::Validate qw(SCALAR ARRAYREF HASHREF BOOLEAN validate_with);
use NATE::BaseException qw(:try);
use NACL::C::CommandInterface;
use NATE::Time qw(timeout2time);
use NACL::APISet;
use Carp qw(croak);
use NACL::ComponentUtils qw(Dumper _ontap_ci_validate_spec);
use NACL::APISet::Response::CLI::ParserUtils qw(snip_only_required_output);
use NACL::APISet::Exceptions::UnexpectedOutputException;
use NATE::Exceptions::Argument;
use NACL::Exceptions::ElementNotFoundInShelfLog;
use NACL::C::Client::File;
use NACL::CS::Client::File;


=head1 NAME

NACL::MTask::ShelfLogMonitoring

=head1 DESCRIPTION

NACL::MTask::ShelfLogMonitoring provides a well-defined but
potentially complex or multi-step method related to monitoring of Shelf Logs in
ONTAP.

The steps performed are as follows:
1. The log file is fetched based on the shelf type.
2. The start & end marker for the file is determined.
3. The log is parsed for the header & other details.

=head1 ATTRIBUTES

=head2 command_interface

(Required) Represents NACL::C::CommandInterface::ONTAP on the filer.

=head1 METHODS

=head2 log_monitor


(Class method) This method is used to monitor the logs based on the parameters passed.

=over

=item Options

=over

=item C<< nacltask_shelf_type => $shelf_type >>

(Optional) Shelf type. Default is sas.
If this parameter 'nacltask_shelf_type' is not specified,
then the parameter nacltask_log_path must be specified.

=item C<< nacltask_log_path => $shelflog_path >>

(Optional) Shelf log path.
If the parameter 'nacltask_shelf_type' is not specified,
then this parameter nacltask_log_path must be specified.
Eg : /etc/log/shelflog/shelflog_sas.0

=item C<< nacltask_start_marker => $start_marker >>

(Optional) Start marker(line) of the log file.
Default is 0.

=item C<< nacltask_end_marker => $eventnd_marker >>

(Optional) End marker(line) of the log file.
Default is End of file line number.

=item C<< nacltask_default_strings => $boolean >>

(Optional) The default set of strings to be checked.
		   Default value is 0.
		   If this option is set to 1,
           then the strings/messages used by default are as follows:
		   [
            "PMC Firmware assert",
            "watchdog",
            "crash",
            "Failed to start Discovery service",
            "Failed to ping HAL watchdog",
            "Failed to configure drive phy",
            "DrvMgr service was unable to get CLI subsystem handle",
            "Unable to get number of PCM",
            "Customer region CRC error",
            "Failed to read from customer region",
            "Clock Fault",
            "Failed to get board ID, targeted download unavailable",
            "Download aborted. Bad data at offset",
            "Failed to get acpEnabled flag",
            "Failed to get acpmgr thread timeout",
            "Failed to get heatbeat info from HAL layer",
            "Fault processing failed",
            "Failed to get DrvMgr handle",
            "Failed to get HID handle",
            "Failed to get General handle",
            "Failed to get Expander Manager handle",
            "Failed to get HAL context",
            "Watchdog",
            "NMI_Type",
            "Temperature jump detected"
		]
These default strings/messages are events that are checked in the shelflog.

=item C<< nacltask_include_strings => \%include_strings >>

(Optional) Strings(events) to be checked.
"Events" to be checked in the shelf log and
these will be checked in addition to the default list of events
if nacltask_default_strings is set to true.
Eg: nacltask_default_strings => 1,
    nacltask_include_strings => [ 'Host Not Attatched'],

Along  with the default set of strings, 'Host Not Attached' will be
checked in the shelf log.

=item C<< nacltask_exclude_strings => \%exclude_strings >>

(Optional) "Events" to be deleted from the default list of events  to be checked.
if nacltask_default_strings is set to true.

Eg: nacltask_default_strings => 1,
    nacltask_include_strings => [ 'Host Not Attatched'],
    nacltask_exclude_strings => [ 'Download aborted. Bad data at offset'],
	Now the set of events to be checked will be
	set_strings = default_set_strings - exclude_strings + include_strings

String 'Download aborted. Bad data at offset' will be deleted from the default set
& 'Host Not Attatched' will be added to the default set of strings.
The set of strings will be checked in shelf log.

=item C<< nacltask_header => $boolean >>

(Optional) Default is 0. If set, the header is printed.

=item C<< nacltask_log_also => $boolean >>

(Optional) The parameter when set, prints the line both to the dump file and <stdout>.
		   By default this is set to 0.

=item C<< nacltask_dump_file => $dump_file >>

(Optional) The log file to be dumped with the logs.
Eg : /etc/log/shelflog/dump_file

=item C<< nacltask_log_records => \%log_records >>

(Optional) Set of records to be checked. The parameter included
in the records and their description are as follows:

	nacltask_log_records => {
        'channel' => [0c],
		'shelf' => [ 0],
		%additional_parameters,
    }

channel => $channels

(Optional) Adapter to be checked.
Eg:
       NACL::MTask::ShelfLogMonitoring->log_monitor (
	   command_interface => $ci,
	   nacltask_shelf_type => 'esh',
	   nacltask_module_serials => [ 'IMS698133122470' ],
	   nacltask_log_records   => { channel => [0c] },
	   );

The sample output from the log:
--------------------------------------------------------------
Channel: 0a Shelf: 5 Shelf Serial Number: OPS677342101450
Shelf product id: DS14-Mk4-FC
Module A Serial Number: IMS698133122470  Firmware rev: 14
Module B Serial Number: IMS6981331224B9  Firmware rev: 14
Controller A boot time: 21 days 4 hrs 35 mins 50 secs
Controller B boot time: 21 days 4 hrs 35 mins 37 secs
Power control state: ok
Last power control completion status: ok
Timestamp: Sat Mar 24 09:09:57 GMT 2012
--------------------------------------------------------------

Log have below parameters:
==========================
Module A Serial Number is IMS698133122470
Channel is 0a.

Parameter passed by the user:
=============================
nacltask_module_serials is IMS698133122470
Channel is 0c

For Module Serial Number 'IMS698133122470'
The parameter passed by the user i.e Expected value of channel is 0c
The parameter from the log i.e Obtained value of channel is 0a.

If the expected & obtained value are same there is a match,
else there is a mismatch.

The output of mismatch:
Module Serial Number: IMS6981331224B9
============================================
Expected Value of channel : 0c
Obtained Value of channel : 0a


Eg: If the parameter 'nacltask_module_serials' not passed,
nacltask_log_records => { channel => [ 0c] }, then
the channel 0c is searched for all the modules,
if match is found at record 1, the message is printed as
"Attribute channel with value 0c matches at record 1".

shelf => $shelf

(Optional) shelf id to be checked.

Eg: If nacltask_module_serials = [ 'IMS698133122470' ]

	 NACL::MTask::ShelfLogMonitoring->log_monitor (
	   command_interface => $ci,
	   nacltask_shelf_type => 'esh',
	   nacltask_module_serials => [ 'IMS698133122470' ],
	   nacltask_log_records   => { shelf => [0] },
	);

The sample output from the log:
--------------------------------------------------------------
Channel: 0a Shelf: 5 Shelf Serial Number: OPS677342101450
Shelf product id: DS14-Mk4-FC
Module A Serial Number: IMS698133122470  Firmware rev: 14
Module B Serial Number: IMS6981331224B9  Firmware rev: 14
Controller A boot time: 21 days 4 hrs 35 mins 50 secs
Controller B boot time: 21 days 4 hrs 35 mins 37 secs
Power control state: ok
Last power control completion status: ok
Timestamp: Sat Mar 24 09:09:57 GMT 2012
--------------------------------------------------------------

Log have below parameters:
==========================
Module A Serial Number is IMS698133122470
Shelf is 5.

Parameter passed by the user:
=============================
nacltask_module_serials is IMS698133122470
Shelf is 0.

For Module Serial Number 'IMS698133122470'
The parameter passed by the user i.e Expected value of shelf is 0
The parameter from the log i.e Obtained value of shelf is 5.

If the expected & obtained value are same there is a match,
else there is no match.

Eg: If the parameter 'nacltask_module_serials' not passed,

	NACL::MTask::ShelfLogMonitoring->log_monitor (
	   command_interface => $ci,
	   nacltask_shelf_type => 'esh',
	   nacltask_log_records   => { shelf => [0] },
	);

For nacltask_log_records => { shelf => [ 0 ] },
the shelf 0 is searched for all the modules.
If match is found for record 1, the message printed is
"Attribute shelf with value 0 matches for record 1".

The additional parameters are -

module => $module_name
The module name can be either A or B.

shelf_uid => $shelf_uid
The address represented in the hexadecimal format.
Eg: 50:05:0c:c0:02:10:5f:59

shelf_s/n => $shelf_s/n
The shelf serial number represented in shelf_type iom.
Eg: 6000047967

term_switch => $term_switch
Terminate switch status
It can be on or off.

shelf_state => $shelf_state
The shelf status.
online - Shelf is fully configured and operational.
offline - Contact was lost with shelf .
missing - Shelf was removed from the system entirely (all paths).
failed - Failure occurred on the shelf.

module_state => $module_state
The module state must be OK.

module_uptime => $module_uptime
The uptime of the module
Eg: 0 days 4 hrs 19 mins 48 secs

shelf_serial_number => $shelf_serial_number
Shelf serial number.
Eg: NTAPMIL-092504A525

shelf_product_id => $shelf_product_id
Product id of the shelf.
Eg: DS14-Mk4-FC

module_a_serial_number => $module_a_serial_number
Serial number of Module A.
Eg: IMS828990069B2B

module_b_serial_number => $module_b_serial_number
Serial number of Module B.

firmware_rev => $firmware_revision
Shelf Firmware revision.

module_type => $module_type
Type of module.
Eg: ESAS, AT-FCX

controller_a_boot_time => $controller_a_boot_time
Boot time of Controller A.
Eg: 21 days 4 hrs 35 mins 50 secs

controller_b_boot_time => $controller_a_boot_time
Boot time of Controller B.

power_control_state => $countower_control_state
The Power supply state to the shelf.
It can be ok, critical
Eg: critical - power control fault detected

last_power_control_completion_status => $last_command_completion_status
This can be ok or reasons that failed the power control.
Eg: inconsistent drive status

timestamp => $timestamp
The timestamp of the log details for each module.

=item C<< nacltask_stat_records	=> $stat_records >>

(Optional) Set of statistical records to be checked. The parameter included
in the records and their description are as follows:

	nacltask_stat_records => {
        'disk_id' => { 1 => [ 1,2], }
		'loop_id' => { 1 => ['0b.32', '0b.33'], 2 => ['0c.32]}
		%additional_parameters,
    }
 disk_id => $disk_ids

 (Optional) disk_ids to be searched.

 The sample output are as follows:

 Module 1:
===================START OF MODULE 1==========================================================
--------------------------------------------------------------------------------------
Timestamp: Mon Apr 16 16:33:12 GMT 2012
Shelf name:    0c.shelf0
Channel:       0c
Module:        B
Shelf id:      0
Shelf UID:     50:0c:0f:f0:0d:38:b4:3c
Shelf S/N:     N/A
Term switch:   N/A
Shelf state:   ONLINE
Module state:  OK


               Partial Path   Link    Invalid   Running     Loss    Phy       CRC     Phy
Disk    Port   Timeout        Rate     DWord    Disparity   Dword   Reset     Error   Change
Id     State   Value (ms)    (Gb/s)    Count    Count       Count   Problem   Count   Count
--------------------------------------------------------------------------------------------
[IN0 ] OK             7        3.0        0           0       0        0         0       8
[IN1 ] OK             7        3.0        0           0       0        0         0       8
[IN2 ] OK             7        3.0        0           0       0        0         0       8
[IN3 ] OK             7        3.0        0           0       0        0         0       8
[OUT0] UNUSED         0         NA        0           0       0        0         0       1
[OUT1] UNUSED         0         NA        0           0       0        0         0       1
[OUT2] UNUSED         0         NA        0           0       0        0         0       1
[OUT3] UNUSED         0         NA        0           0       0        0         0       1
[  0 ] OK             7        3.0        0           0       0        0         0       1
[  1 ] OK             7        3.0        0           0       0        0         0       1
[  2 ] OK             7        3.0        0           0       0        0         0       1
[  3 ] OK             7        3.0        0           0       0        0         0       1

--------------------------------------------------------------------------------------------
================END OF MODULE1===============================================================

 Eg:    NACL::MTask::ShelfLogMonitoring->log_monitor (
			command_interface => $ci,
			nacltask_stat_records => { 'disk_id' => { 1 => [ 1,2,30]}},
		);

Log contains statistical records
================================
Parameter of stats : Disk Id, Port State etc.
The value for Disk Id for this module: IN0, IN1, 0, 1, 2 etc.

Parameter passed by the user.
=============================
Parameter of nacltask_stat_records : disk_id
The value of disk_id for module 1 are 1, 2, 30.

For Module 1,
The parameter passed by the user i.e Expected value of disk_id are 1, 2, 30
The parameter from the log i.e Obtained value of disk_id are 0, 1, 2.

If the expected & obtained value are same, then there is a match.
If the expected & obtained value are different,
then there is no match.

 loop_id => $loop_ids

 (Optional) Similar to the disk_ids, the loop_ids are checked.

 Eg:  NACL::MTask::ShelfLogMonitoring->log_monitor (
			command_interface => $ci,
			nacltask_module_serials => [ 'IMS698133122470'],
			nacltask_stat_records => { loop_id => { 1 => [ '0a.80', '0a.81', '0b.2']}},
		);

Sample of output
--------------------------------------------------------------
Channel: 0a Shelf: 5 Shelf Serial Number: OPS677342101450
Shelf product id: DS14-Mk4-FC
Module A Serial Number: IMS698133122470  Firmware rev: 14
Module B Serial Number: IMS6981331224B9  Firmware rev: 14
Controller A boot time: 0 days 20 hrs 24 mins 2 secs
Controller B boot time: 0 days 20 hrs 20 mins 28 secs
Power control state: ok
Last power control completion status: ok
Timestamp: Fri Apr 20 11:26:39 GMT 2012
--------------------------------------------------------------

Loop                   Drive            Drive     Command     5V     12V        Drive     Reset        Over     CtlLine
 ID                  Present            Power     Request                     Powered     Fault     Current       Stuck
                                       Status      Status                        Down                 Fault        OFF
0a.80                     X                Ok           -     ON      ON           -         -           -           -
0a.81                     X                Ok           -     ON      ON           -         -           -           -

Log contains statistical records
================================
Parameter of stats : Loop Id, Drive Present etc.
The value for Loop Id for this module: 0a.80, 0a.81

Parameter passed by the user.
=============================
Parameter of nacltask_stat_records : loop_id
The value of loop_id for module serial number IMS698133122470 are 0a.80, 0a.81.

For Module Serial Number IMS698133122470,
The parameter passed by the user i.e Expected value of loop_id are 0a.80, 0a.81, 0b.2.
The parameter from the log i.e Obtained value of loop_id are 0a.80, 0a.81.

If module_serials is passed, the parameters mentioned for nacltask_stat_records is
checked from the module that contains the particular serial number.
If the expected & obtained value are same there is a match,
else there is no match.

The output is as follows:
=========================

Module Serial Number: IMS698133122470
=======================================
Attribute loop_id with value 0a.80 matches
Attribute loop_id with value 0a.81 matches
Attribute loop_id: with value 0b.2 does not exists in the log

 Similar to the disk_id, the additional parameters are checked

=item C<< nacltask_module_serials => $serial_numbers >>

(Optional) If the parameter is specified, then the module serial numbers are searched
		   If the module serial number found, then for each number found
		   if the nacltask_log_records specified, then related records are searched,
		   if the nacltask_stat_records specified, then statistical datum are searched.
		   if the nacltask_header_only specified, then the header is printed.
		   if the nacltask_log_also specified, then the log is dumped in the log file(nacltask_dump_file)
		   as well as STDOUT.
		   If the log file doesn't exist, then printed in the log of the current program.


=item Example 1: To obtain the log info with shelf type sas as default

my $output = NACL::MTask::ShelfLogMonitoring->log_monitor(
    command_interface  => $command_interface,
);

=item Example 2:  To obtain the log info with shelf type passed.

my $output = NACL::MTask::ShelfLogMonitoring->log_monitor(
    command_interface => $command_interface,
    nacltask_shelf_type => 'iom',
);

=item Example 3:  To obtain the log info with shelf log passed

my $output = NACL::MTask::ShelfLogMonitoring->download_and_verify(
    command_interface  => $command_interface,
    nacltask_shelf_log => '/etc/log/shelflog/shelflog_esh',
);

=item Example 4: To obtain the log info with start & end marker passed.
				 The log info will be captured between that lines.

my $output = NACL::MTask::ShelfLogMonitoring->log_monitor(
    command_interface => $command_interface,
    nacltask_shelf_type => 'iom',
	nacltask_start_marker => 12,
	nacltask_end_marker => 100,
);


=back

=back

=cut

sub log_monitor {
    $Log->enter() if $may_enter;
    my $pkg  = shift;
    my %opts = validate_with(
        params => \@_,
        spec   => {
            command_interface        => _ontap_ci_validate_spec(),
            nacltask_shelf_type      => { type => SCALAR, default => "sas", },
            nacltask_log_path        => { type => SCALAR, optional => 1 },
            nacltask_start_marker    => { type => SCALAR, default => 0 },
            nacltask_end_marker      => { type => SCALAR, optional => 1 },
            nacltask_default_strings => { type => BOOLEAN, default => 0 },
            nacltask_include_strings => { type => ARRAYREF, optional => 1 },
            nacltask_exclude_strings => { type => ARRAYREF, optional => 1 },
            nacltask_header_only     => { type => BOOLEAN, default => 0 },
            nacltask_log_also        => { type => BOOLEAN, default => 0 },
            nacltask_dump_file       => { type => SCALAR, optional => 1 },
            nacltask_log_records     => { type => HASHREF, optional => 1 },
            nacltask_stat_records    => { type => HASHREF, optional => 1 },
            nacltask_module_serials  => { type => ARRAYREF, optional => 1 },
        },
    );

    my $string_set;
    if ( $opts{nacltask_default_strings} ) {
        $string_set = [
            "PMC Firmware assert",
            "watchdog",
            "crash",
            "Failed to start Discovery service",
            "Failed to ping HAL watchdog",
            "Failed to configure drive phy",
            "DrvMgr service was unable to get CLI subsystem handle",
            "Unable to get number of PCM",
            "Customer region CRC error",
            "Failed to read from customer region",
            "Clock Fault",
            "Failed to get board ID, targeted download unavailable",
            "Download aborted. Bad data at offset",
            "Failed to get acpEnabled flag",
            "Failed to get acpmgr thread timeout",
            "Failed to get heatbeat info from HAL layer",
            "Fault processing failed",
            "Failed to get DrvMgr handle",
            "Failed to get HID handle",
            "Failed to get General handle",
            "Failed to get Expander Manager handle",
            "Failed to get HAL context",
            "Watchdog",
            "NMI_Type",
            "Temperature jump detected"
        ];
    }

    my %exclude_strings
        = map { $_ => 1 } @{ $opts{nacltask_exclude_strings} };
    my @valid_strings;
    foreach my $string ( @{$string_set} ) {
        push @valid_strings, $string
            if ( !exists $exclude_strings{$string} );
    }
    $string_set = [@valid_strings];

    my @events = map( $_ =~ /^\w/ ? $_ : (), @$string_set );
    if ( $opts{nacltask_include_strings} ) {
        foreach ( @{ $opts{nacltask_include_strings} } ) {
            push( @events, $_ );
        }
    }

    my $command_interface = $opts{command_interface};
    my $path;
    delete $opts{nacltask_shelf_type} if ( $opts{nacltask_log_path} );
    if ( $opts{nacltask_shelf_type} ) {
        $opts{nacltask_shelf_type} = lc( $opts{nacltask_shelf_type} );
        if ( $opts{nacltask_shelf_type} =~ /at-fcx/ ) {
            $opts{nacltask_shelf_type} = 'ata';
        }
        $path = "/etc/log/shelflog/shelflog_" . $opts{nacltask_shelf_type};
    } else {
        $path = $opts{nacltask_log_path};
    }
    if (   ( $opts{nacltask_log_path} )
        && ( $path =~ /\/etc\/log\/shelflog\/shelflog_/ ) ) {
        $opts{nacltask_shelf_type} = $';
    }
    if ( !$path ) {
        NATE::BaseException->throw("Log Path not found");
    } else {
        $Log->comment("The log path is $path");
    }

    my @file_name = NACL::MTask::SCP->copy_to_localhost(
        source_command_interface => $opts{command_interface},
        'files_to_copy'          => ["/mroot/$path"],
    );

    my $file_obj;
    my $raw_output;

    $file_obj = NACL::C::Client::File->open(
        file_name   => $file_name[0],
        access_mode => 'read',
        open_flags  => 'open-existing'
    );

    try {
        my $file_state_obj = NACL::CS::Client::File->fetch(
            file_name        => "$file_name[0]",
            requested_fields => ['size'],
        );

        my $file_size = $file_state_obj->size();
        $raw_output = $file_obj->read(num_bytes => $file_size);
    } finally {

        #Cleanup. Delete the local copy of the file.
        $file_obj->delete();
    };

    my @lines      = get_lines('raw_output' => $raw_output);

    my $apiset = $command_interface->get_7m_or_nodescope_apiset();
    if ( $opts{nacltask_dump_file} ) {
        $apiset->mkfile(
            filename => $opts{nacltask_dump_file},
            size     => 1024
        );
    }


    my $total_lines = $#lines + 1;
    $opts{nacltask_end_marker} = $total_lines
        if ( !$opts{nacltask_end_marker} );
    if ( $opts{nacltask_start_marker} > $opts{nacltask_end_marker} ) {
        NATE::Exceptions::Argument->throw("Input for file  marking is wrong");
    } elsif ( $total_lines == 0 ) {
        NATE::BaseException->throw("File is empty");
    } else {
        $Log->comment("Number of lines in file : $total_lines");
    }
    @lines = splice(
        @lines,
        $opts{nacltask_start_marker},
        $opts{nacltask_end_marker}
    );
    my $tot_lines_required = @lines;
    $Log->comment("Number of lines to be parsed : $tot_lines_required");
    my $output = _parse_log_path(@lines);

    my @matched_events;
    my $file                    = delete $opts{nacltask_dump_file};
    my $count                   = 1;
    my $match_found_log_records = 0;
    my $log_records             = delete $opts{nacltask_log_records};
    my $stat_records            = delete $opts{nacltask_stat_records};
    my @ret;
    foreach my $var (@$output) {
        @matched_events = ();
        if ( $opts{nacltask_default_strings} ) {
            if ( $var->{'event'} ) {
                my $all_events = @{ $var->{'event'} };
                for ( my $i = 0; $i < $all_events; $i++ ) {
                    my $curr_event = $var->{'event'}->[$i]->{'event_details'};
                    foreach my $user_event (@events) {
                        if ( $curr_event =~ /$user_event/ ) {
                            push( @matched_events, $var->{'event'}->[$i] );
                        }
                    }
                }
                push( @ret, @matched_events );
                $Log->comment("Event occured at Timestamp $var->{timestamp}");
                $Log->comment( "Matched Events\n" . Dumper(@matched_events) );
            }
        }
        if ($log_records) {
            my $header_content;
            if ( !$opts{nacltask_module_serials} ) {
                foreach my $key ( keys(%$log_records) ) {
                    foreach my $val ( @{ $log_records->{$key} } ) {
                        if ( $var->{$key} && $var->{$key} =~ /$val/ ) {
                            $Log->comment(
                                "Attribute $key with value $val matches at module $count"
                            );
                            $header_content = "$key : $val";
                            push( @ret, $header_content );
                            $match_found_log_records = 1;
                        }
                    }
                    if ( !$match_found_log_records ) {
                        $Log->comment(
                            "Attribute $key doesn't match any value at module $count"
                        );
                        $Log->debug(
                            "Obtained value for $key at module $count : $var->{$key}"
                        );
                    }
                }
            } else {
                for (
                    my $rec_no = 0;
                    $rec_no < @{ $opts{nacltask_module_serials} };
                    $rec_no++
                    ) {
                    my $mod_no = $opts{nacltask_module_serials}->[$rec_no];
                    my $mod_a  = $var->{module_a_serial_number};
                    my $mod_b  = $var->{module_b_serial_number};
                    if (   ( $mod_a && $mod_a =~ /$mod_no/i )
                        || ( $mod_b && $mod_b =~ /$mod_no/i ) ) {
                        $Log->comment("Module Serial Number: $mod_no");
                        $Log->comment(
                            "============================================");
                        foreach my $key ( keys(%$log_records) ) {
                            if (   $var->{$key}
                                && $var->{$key}
                                =~ /$log_records->{$key}->[$rec_no]/ ) {
                                $Log->comment(
                                    "Attribute $key with value $var->{$key} matches"
                                );
                                $header_content = "$key : $var->{$key}";
                                push( @ret, $header_content );
                                $match_found_log_records = 1;
                            } else {
                                $Log->comment(
                                    "Expected Value of $key : $log_records->{$key}->[$rec_no]"
                                );
                                $Log->comment(
                                    "Obtained Value of $key : $var->{$key}");
                            }
                        }
                    }
                }
            }
        }
        if ($stat_records) {
            if ( $var->{'stat_details'} ) {
                my $stat_content;
                if ( $opts{nacltask_module_serials} ) {
                    for (
                        my $rec_no = 0;
                        $rec_no < @{ $opts{nacltask_module_serials} };
                        $rec_no++
                        ) {
                        my $mod_no
                            = $opts{nacltask_module_serials}->[$rec_no];
                        my $mod_a = $var->{'module_a_serial_number'};
                        my $mod_b = $var->{'module_b_serial_number'};
                        if (   ( $mod_a && $mod_a =~ /$mod_no/i )
                            || ( $mod_b && $mod_b =~ /$mod_no/i ) ) {
                            $Log->comment("Module Serial Number: $mod_no");
                            $Log->comment(
                                "=======================================");
                            foreach my $key ( keys(%$stat_records) ) {
                                if ( $var->{'stat_details'}->[0]->{$key} ) {
                                    my $arr_stats = $stat_records->{$key}
                                        ->{ $rec_no + 1 };
                                    for (
                                        my $each_val = 0;
                                        $each_val < @$arr_stats;
                                        $each_val++
                                        ) {
                                        my $match = 0;
                                        my $log_stats
                                            = $var->{'stat_details'};
                                        for (
                                            my $eventach_rec = 0;
                                            $eventach_rec < @$log_stats;
                                            $eventach_rec++
                                            ) {
                                            if ($log_stats->[$eventach_rec]
                                                ->{$key}
                                                && ($arr_stats->[$each_val] eq
                                                    $log_stats
                                                    ->[$eventach_rec]
                                                    ->{$key} )
                                                ) {
                                                $Log->comment(
                                                    "Attribute $key with value $arr_stats->[$each_val] matches"
                                                );
                                                $stat_content
                                                    = "$key : $arr_stats->[$each_val]";
                                                push( @ret, $stat_content );
                                                $match = 1;
                                            }
                                        }
                                        if ( !$match ) {
                                            NACL::Exceptions::ElementNotFoundInShelfLog
                                                ->throw(
                                                "Attribute $key: with value $arr_stats->[$each_val] does not exists in the log"
                                                );
                                        }
                                    }
                                } else {
                                    NACL::Exceptions::ElementNotFoundInShelfLog
                                        ->throw(
                                        "Attribute $key not present in the log"
                                        );
                                }
                            }
                        }
                    }
                } else {
                    $Log->comment("Module $count");
                    $Log->comment("=============");
                    foreach my $key ( keys(%$stat_records) ) {
                        if ( $var->{'stat_details'}->[0]->{$key} ) {
                            if ( $stat_records->{$key}->{$count} ) {
                                my $user_stats
                                    = $stat_records->{$key}->{$count};
                                my $log_stats = $var->{'stat_details'};
                                for (
                                    my $val = 0;
                                    $val < @$user_stats;
                                    $val++
                                    ) {
                                    my $match = 0;
                                    for (
                                        my $each_val = 0;
                                        $each_val < @$log_stats;
                                        $each_val++
                                        ) {
                                        if ( $user_stats->[$val] eq
                                            $log_stats->[$each_val]->{$key} )
                                        {
                                            my $record = $each_val + 1;
                                            $Log->comment(
                                                "Attribute $key with value $user_stats->[$val] matches at record $record"
                                            );
                                            $stat_content
                                                = "$key : $user_stats->[$val]";
                                            push( @ret, $stat_content );
                                            $match = 1;
                                        }
                                    }
                                    if ( !$match ) {
                                        NACL::Exceptions::ElementNotFoundInShelfLog
                                            ->throw(
                                            "Attribute $key: with value $user_stats->[$val] does not exists in the log"
                                            );
                                    }
                                }
                            }
                        } else {
                            NACL::Exceptions::ElementNotFoundInShelfLog
                                ->throw(
                                "The attribute $key not present in the log");
                        }
                    }
                }
            }
        }
        if ( $opts{nacltask_header_only} ) {
            my $temp_stat = delete $var->{'stat_details'}
                if ( $var->{'stat_details'} );
            my $temp_event = delete $var->{'event'} if ( $var->{'event'} );
            if ( !$opts{nacltask_module_serials} ) {
                if ( ($file) || $opts{nacltask_log_also} ) {
                    $Log->comment("Started writing the file");
                    foreach my $key ( keys(%$var) ) {
                        my $content = "$key : $var->{$key}";
                        $apiset->wrfile(
                            file    => $file,
                            content => $content,
                            append  => 1
                        );
                    }
                }
                if ( ( !$file ) || $opts{nacltask_log_also} ) {
                    $Log->comment(
                        "Header for module $count\n==================\n"
                            . Dumper($var) );
                }
            } else {
                foreach my $mod_no ( @{ $opts{nacltask_module_serials} } ) {
                    my $mod_a = $var->{module_a_serial_number};
                    my $mod_b = $var->{module_b_serial_number};
                    if (   ( $mod_a && $mod_a =~ /$mod_no/i )
                        || ( $mod_b && $mod_b =~ /$mod_no/i ) ) {
                        if ( ($file) || $opts{nacltask_log_also} ) {
                            $Log->comment("Started writing the file");
                            foreach my $key ( keys(%$var) ) {
                                my $content = "$key : $var->{$key}";
                                $apiset->wrfile(
                                    file    => $file,
                                    content => $content,
                                    append  => 1
                                );
                            }
                        }
                        if ( ( !$file ) || $opts{nacltask_log_also} ) {
                            $Log->comment(
                                "Header for Module with Module number $mod_no\n"
                                    . Dumper($var) );
                        }
                    }
                }
            }
            $var->{'stat_details'} = $temp_stat  if ($temp_stat);
            $var->{'event'}        = $temp_event if ($temp_event);
        }
        $count++;
    }
    if ( ( !$match_found_log_records ) && ($log_records) ) {
        NACL::Exceptions::ElementNotFoundInShelfLog->throw(
            "No matching log records");
    }
    $Log->exit() if $may_exit;
    if (@ret) {
        return \@ret;
    } else {
        return $output;
    }

} ## end sub log_monitor
1;

########################################################################
# Method: _parse_log_path
# Objective: To parse the shelf log
# Description:
# When the attribute defined in the header_set is present in the
# log the corresponding value is retrieved from the log, then the header
# is populated as $result_array[first_module]->{header_attribute} = value
# For each header, the statistical data attributes are checked
# with the log, when found then the attribute & its value populated as
# $result_array[first_module]->{stat_details}->[record 1]->{attribute} = value
# $result_array[first_module]->{event}->[0]->{event_details} = value
# The above line represents the first_module, the first event_details
# stored.
# Options:
#   lines - required - number of lines to be parsed
# Output: Returns the array reference contains the attributes & values.
########################################################################

sub _parse_log_path {
    my (@lines) = @_;

    #In the actual hash for all the keys spaces will be replaced with '-'

    my ( @header_set1, @header_set2, @all_fields, $count );
    my @result_array = ();

    my $stat  = -1;
    my $head  = -1;
    my $event = -1;

    for ( my $i = 0; $i <= $#lines; $i++ ) {
        my $head_pos;
        if (   ( $lines[$i] =~ /Timestamp:/ )
            && ( $lines[ $i + 1 ] && $lines[ $i + 1 ] =~ /Shelf name:/ ) ) {
            $head_pos = $i;
            while ( $lines[$head_pos] =~ /:/ ) {
                my @header_values = split( /:\s+/, $lines[$head_pos] );
                push( @header_set1, $header_values[0] );
                last if ( $lines[$head_pos] !~ /:/ );
                $head_pos++;
            }
        }
        if ( $lines[$i] =~ /Channel: ([0-9]+[a-f]) Shelf:/ ) {
            $head_pos = $i;
            while ( $lines[$head_pos] =~ /:/ ) {
                my @header_values = split( /:\s+/, $lines[$head_pos] );
                if ( $#header_values == 1 ) {
                    push( @header_set2, $header_values[0] );
                } else {
                    my @temp_arr = split( /:\s+\w+\s*/, $lines[$head_pos] );
                    for ( $count = 0; $count <= $#temp_arr; $count++ ) {
                        if ( $temp_arr[$count] =~ /-[A-Z]+\s+/ ) {
                            $temp_arr[$count] =~ s/-[A-Z]+\s+//g;
                        }
                    }
                    push( @header_set2, @temp_arr );
                }
                last if ( $lines[$head_pos] !~ /:/ );
                $head_pos++;
            }
        }
        my @words = split( /:\s+/, $lines[$i] );
        my ( @temp, @temp_header );
        if ( $#words == 1 ) {
            @header_set1 = grep {$_} @header_set1;
            @header_set2 = grep {$_} @header_set2;
            @temp = map( $_ eq $words[0] ? $_ : (), @header_set1 );
            @temp_header = map ( $_ eq $words[0] ? $_ : (), @header_set2 );
        } elsif ( $#words > 1 ) {
            @words = ();
            my $pos      = 0;
            my $word_pos = 0;
            foreach (@header_set2) {
                if ( $lines[$i] =~ /$_\: (\w+-*\w*)/ ) {
                    chomp($_);
                    $temp_header[$pos] = $_;
                    $words[$pos]       = $1;
                    $temp_header[$pos] =~ s/(\s+)/_/g;
                    $temp_header[$pos] = lc( $temp_header[$pos] );
                    $pos++;
                }
            }
            if ( $lines[$i]
                =~ /Module (.) Serial Number: (\w*)  Firmware rev: (\d*)/ ) {
                $temp_header[$pos] = "Module $1 Serial Number";
                $temp_header[$pos] =~ s/(\s+)/_/g;
                $temp_header[$pos]       = lc( $temp_header[$pos] );
                $words[$pos]             = $2;
                $temp_header[ $pos + 1 ] = "Firmware rev $1";
                $temp_header[ $pos + 1 ] =~ s/(\s+)/_/g;
                $temp_header[ $pos + 1 ] = lc( $temp_header[$pos] );
                $words[ $pos + 1 ]       = $3;
                $pos                     = $pos + 2;
            }
        }
        if (   ( $lines[$i] =~ /Disk\s+/ )
            && ( $lines[ $i + 1 ] && $lines[ $i + 1 ] =~ /Id\s+/ )
            && ( $lines[ $i - 1 ] =~ /Clock/ ) ) {
            @all_fields = _extract_stat_fields(
                $lines[ $i - 1 ],
                $lines[$i],
                $lines[ $i + 1 ]
            );
        } elsif ( ( $lines[$i] =~ /Disk\s+/ )
            && ( $lines[ $i + 1 ] && $lines[ $i + 1 ] =~ /Id\s+/ )
            && ( $lines[ $i - 1 ] =~ /Partial Path/ ) ) {
            @all_fields = _extract_stat_fields(
                $lines[ $i - 1 ],
                $lines[$i],
                $lines[ $i + 1 ]
            );
        } elsif ( ( $lines[$i] =~ /Disk\s+/ )
            && ( $lines[ $i + 1 ] && $lines[ $i + 1 ] =~ /Id\s+/ )
            && ( $lines[ $i - 1 ] =~ /Oscillation/ ) ) {
            @all_fields = _extract_stat_fields(
                $lines[ $i - 1 ],
                $lines[$i],
                $lines[ $i + 1 ]
            );
        } elsif ( ( $lines[$i] =~ /Loop\s+/ )
            && ( $lines[ $i + 1 ] && $lines[ $i + 1 ] =~ /ID\s+/ ) ) {
            @all_fields = _extract_stat_fields(
                $lines[$i],
                $lines[ $i + 1 ],
                $lines[ $i + 2 ]
            );
        } elsif ( ( $lines[$i] =~ /Disk\s+/ )
            && ( $lines[ $i + 1 ] && $lines[ $i + 1 ] =~ /ID\s+/ )
            && ( $lines[ $i - 1 ] =~ /ATA/ ) ) {
            @all_fields = _extract_stat_fields(
                $lines[ $i - 1 ],
                $lines[$i],
                $lines[ $i + 1 ]
            );
        } elsif ( ( $lines[$i] =~ /Link\s+/ )
            && ( $lines[ $i + 1 ] && $lines[ $i + 1 ] =~ /Errors\s+/ ) ) {
            @all_fields = _extract_stat_fields(
                $lines[ $i - 1 ],
                $lines[$i],
                $lines[ $i + 1 ]
            );
        } elsif ( ( $lines[$i] =~ /Timestamp:/ )
            && ( $lines[ $i + 1 ] && $lines[ $i + 1 ] =~ /Shelf name:/ ) ) {

            #start a new hash for the new shelf
            $head++;
            $stat = $event = -1;
            chomp( $words[1] );
            my @ar = split( /:\s+/, $lines[ $i + 1 ] );
            $result_array[$head]->{timestamp} = $words[1];
            $result_array[$head]->{'shelf_name'} = $ar[1];
        } elsif ( $lines[$i] =~ /Channel: ([0-9]+[a-f]) Shelf:/ ) {
            $head++;
            $stat = $event = -1;
            $result_array[$head]->{channel} = $1;

            if (@temp_header) {
                for ( $count = 0; $count <= $#temp_header; $count++ ) {
                    if ( $words[$count] ) {
                        chomp( $words[$count] );
                        $temp_header[$count] =~ s/(\s+)/_/g;
                        $temp_header[$count] = lc( $temp_header[$count] );
                        $result_array[$head]->{ $temp_header[$count] }
                            = $words[$count];
                    }
                }
            }
        } elsif (@temp) {
            $temp[0] =~ s/(\s+)/_/g;
            $temp[0] = lc( $temp[0] );
            chomp( $words[1] );
            $result_array[$head]->{ $temp[0] } = $words[1];
        } elsif ( @temp_header && $#words == 1 ) {
            $temp_header[0] =~ s/(\s+)/_/g;
            $temp_header[0] = lc( $temp_header[0] );
            chomp( $words[1] );
            $result_array[$head]->{ $temp_header[0] } = $words[1];

        } elsif ( @temp_header && $#words > 1 ) {
            for ( $count = 0; $count <= $#temp_header; $count++ ) {
                if ( $words[$count] ) {
                    chomp( $words[$count] );
                    $temp_header[$count] =~ s/(\s+)/_/g;
                    $temp_header[$count] = lc( $temp_header[$count] );
                    $result_array[$head]->{ $temp_header[$count] }
                        = $words[$count];
                }
            }
        } elsif ( $lines[$i] =~ /^\[\s*(\S+)\s*\]\s+/ ) {
            $stat++;
            my $disk_id   = $1;
            my $rest_line = $';
            my @cur_line_values;
            $result_array[$head]->{'stat_details'}->[$stat]->{'disk_id'}
                = $disk_id;
            my $start = 1;
            push( @cur_line_values, $disk_id );
            if (   ( $disk_id =~ /[A-Z]+/ )
                && ( $all_fields[1] =~ /disk_bay/ ) ) {
                my $disk_bay = "-";
                $result_array[$head]->{'stat_details'}->[$stat]->{'disk_bay'}
                    = $disk_bay;
                push( @cur_line_values, $disk_bay );
                $start = 2;
            }

            if (   ( $rest_line =~ /\s*NO SIGNAL/ )
                && ( $all_fields[1] =~ /port_state/ ) ) {
                my $port_state = $1;
                $result_array[$head]->{'stat_details'}->[$stat]
                    ->{'port_state'} = $port_state;
                push( @cur_line_values, $port_state );
                $start = 2;
            }
            my @rem_line_values = split( /\s+/, $rest_line );

            for ( $count = 0; $count < $start; $count++ ) {
                $result_array[$head]->{'stat_details'}->[$stat]
                    ->{ $all_fields[$count] } = $cur_line_values[$count];
            }
            for ( $count = $start; $count <= $#rem_line_values; $count++ ) {
                $result_array[$head]->{'stat_details'}->[$stat]
                    ->{ $all_fields[$count] } = $rem_line_values[$count];
            }
        } elsif ( $lines[$i] =~ /^[0-9]+[a-f]\.[0-9]+(.*)\s+/ ) {
            my @cur_line_values = split( /\s+/, $lines[$i] );
            $stat++;
            for ( $count = 0; $count <= $#cur_line_values; $count++ ) {
                $result_array[$head]->{'stat_details'}->[$stat]
                    ->{ $all_fields[$count] } = $cur_line_values[$count];
            }
        } elsif ( $lines[$i] =~ /^BZR:/ ) {
            my @keys = (
                "BZR",         "COS",
                "DoubleOpens", "ForcedSwitch",
                "LipSys",      "BadLim",
                "BadRouterTable"
            );
            my @words1 = split( /\s+/, $lines[$i] );
            foreach (@words1) {
                my @words2 = split( /:/, $_ );
                my @temp1 = map( $_ =~ /^$words2[0]/ ? $_ : (), @keys );
                $temp1[0] =~ s/(\s+)/_/g;
                $temp1[0] = lc( $temp1[0] );
                chomp( $words2[1] );
                $result_array[$head]->{ $temp1[0] } = $words2[1];
            }
        } elsif ( $lines[$i] =~ /^(\d+)\s+(\dd\d+h\d+m\d+s\d+ms)\s+(.*)/ ) {
            $event++;
            $result_array[$head]->{'event'}->[$event]->{'event_id'} = $1;
            $result_array[$head]->{'event'}->[$event]->{'event_timestamp'}
                = $2;
            $result_array[$head]->{'event'}->[$event]->{'event_details'} = $3;
        } elsif ( $lines[$i] =~ /(\d+\+\d+:\d+:\d+.\d+); (\w+); (.*)/ ) {
            $event++;
            $result_array[$head]->{'event'}->[$event]->{'event_timestamp'}
                = $1;
            $result_array[$head]->{'event'}->[$event]->{'event_id'}      = $2;
            $result_array[$head]->{'event'}->[$event]->{'event_details'} = $3;
        }
        if ( $lines[$i] =~ /(HI\/HO)|(A->B)/ ) {
            if ($1) {
                $stat = 0;
            } else {
                $stat = 1;
            }
            my @cur_line_values = split( /\s+/, $lines[$i] );
            for ( $count = 0; $count <= $#cur_line_values; $count++ ) {
                $all_fields[$count] =~ s/(\s+)/_/g;
                $all_fields[$count] = lc( $all_fields[$count] );
                $result_array[$head]->{'stat_details'}->[$stat]
                    ->{ $all_fields[$count] } = $cur_line_values[$count];
            }
        }
    }
    return \@result_array;
} ## end sub _parse_log_path

####################################################################################################
# Method:    _extract_stat_fields
# Objective: To return an array of statiscal fields name for each module.
# Description:
# Eg:               Partial Path   Link    Invalid   Running     Loss    Phy       CRC     Phy\n
#    Disk    Port   Timeout        Rate     DWord    Disparity   Dword   Reset     Error   Change\n
#    Id     State   Value (ms)    (Gb/s)    Count    Count       Count   Problem   Count   Count\n
#    --------------------------------------------------------------------------------------------
#  In the above example, Field1: Disk Id, Field2: Port State, Field3: Partial Path Timeout Value (ms)
#  The fields are aligned accordingly to be placed in three lines to give a tabular format.
#  The field extracted would be all_fields[0], i.e Field1 will be disk_id
#  similarly all_fields[1] would contain the field2 i.e port_state.
#  Concatenating the words in different line with the '_' & converting to lower case.
#########################################################################################################
sub _extract_stat_fields {
    my @stat_lines = @_;
    my @fields1    = split( "", $stat_lines[0] );
    my @fields2    = split( "", $stat_lines[1] );
    my @fields3    = split( "", $stat_lines[2] );
    my ( $greater_words, $word1, $word2, $word3, $field, @all_fields );
    my $tot_chars
        = @fields1
        > ( $greater_words = ( @fields2 > @fields3 ) ? @fields2 : @fields3 )
        ? @fields1
        : $greater_words;
    for ( my $c = 0; $c <= $tot_chars; $c++ ) {
        if (   $fields1[$c] && $fields1[$c] =~ /[0-9]|[A-Z]|[a-z]/
            || $fields2[$c] && $fields2[$c] =~ /[0-9]|[A-Z]|[a-z]/
            || $fields3[$c] && $fields3[$c] =~ /[0-9]|[A-Z]|[a-z]/ ) {
            $word1 = $word1 . $fields1[$c] if ( $fields1[$c] );
            $word2 = $word2 . $fields2[$c] if ( $fields2[$c] );
            $word3 = $word3 . $fields3[$c] if ( $fields3[$c] );
        } else {
            $word1 =~ s/\s+//g;
            $word2 =~ s/\s+//g;
            $word3 =~ s/\s+//g;
            $word1 = $word1 . "_" if ( $word1 && $word2 );
            $word1 = lc($word1);
            $word2 = $word2 . "_" if ( $word2 && $word3 );
            $word2 = lc($word2);
            $word3 = lc($word3);
            $field = $word1 . $word2 . $word3;
            push( @all_fields, $field ) if ($field);
            $word1 = $word2 = $word3 = "";
        }
    }
    return @all_fields;
} ## end sub _extract_stat_fields

###############################################################################
# Method:    get_lines
# Objective: To return an array of lines from the beginning of output excluding
#            any blank lines
################################################################################

sub get_lines(%) {
    my %args       = @_;
    my $raw_output = $args{'raw_output'};
    my $first_line = $args{'first_line'};
    my $can_throw  = $args{'can_throw'};
    my @lines;

    if ($raw_output = snip_only_required_output(
            'raw_output' => $raw_output,
            'delimiter'  => $first_line
        )
        ) {
        @lines = split( /\r*\n+/, $raw_output );
        map( s/\s+$//, @lines );
        map( s/\cM//g, @lines );
        @lines = grep ( /\S+/, @lines );
    } elsif ($can_throw) {
        my $response_object = $args{'response_object'};
        NACL::APISet::Exceptions::UnexpectedOutputException->throw(
            text            => "output is not in the expected format",
            response_object => $response_object
        );
    }
    return @lines;
} ## end sub get_lines(%)