#*******************************************************************************
# EMS log parser
# ------------------------------------------------------------------------------
# Author       : Chris White ( NetApp HDD Engineering )
# Created      : 7/15/11
# Last updated : 7/20/11
#
# Theory of Operation: 
# --------------------
# The EMS log is scanned for predefined ems messages. Once they are found the
# subsequent data is collected and then post-line processed. Several of the post-
# processed data values are also filtered in an attempt to make the parsed final 
# post output more readable.
# The final parsed file is currently structured as follows:
#   -------------------------------------------------------
#   1) a summary of all filtered ems messages by count
#   2) a summary of scsi sense codes by drive by count
#   3) details of each filtered ems error message
#   4) a summary of ALL un-filtered ems messages by count ( this is a kinda debug
#      that allows you to make sure no key error mesages were missed by the filter.
#
#*******************************************************************************
use Storage::Common_Lib;
eval{

  use vars qw(
  $FILER
  $FILERA
  $FILERB
  $TEST_CONFIG
  $LOGDIR
  );

param('FILER',          '-mesg',    'Filer');
param("FILERA",         '-default', '');
param("FILERB",         '-default', '');
param('TEST_CONFIG',    '-default', 'D');
param('LOGDIR',         '-default', '');
};

my @EMS_FILES,@EMS_DIR;
my $syslog_dir;
#print "the value of $LOGDIR ";
$logpath="$LOGDIR/../../../";
opendir( DIR_LOG, $logpath );
    my @files_DIR_LOG = readdir(DIR_LOG);
    foreach my $file (@files_DIR_LOG) {
        if ( $file =~ /System_logs/ ) {
            push( @EMS_FILES, $file );
        }
    }

my $i=0;
foreach my $path (@EMS_FILES){
#	print "the value of --- $LOGDIR/../../../$path";
	$EMS_DIR[$i]="$LOGDIR/../../../$path";
	$EMS_FILES[$i]="$LOGDIR/../../../$path/ems";
$i++}
#print "the value ==== $EMS_FILES[0]\n,i------ $EMS_DIR[0]\n --------";

#$EMS_FILES[0] = "$LOGDIR/../../../*System_logs*$FILER/ems";
#$EMS_FILES[1] = "$LOGDIR/../../../*System_logs*$FILERB/ems" if ($TEST_CONFIG eq "C");
#$EMS_DIR[0] = "$LOGDIR/../../../*System_logs*$FILER";
#$EMS_DIR[1] = "$LOGDIR/../../../*System_logs*$FILERB" if ($TEST_CONFIG eq "C");
DispDirFiles();
#--------------------------------------------------------------------
#   wait for keyboard input
# *** USER INPUT - NOT NEEDED FOR NDATE ***
#--------------------------------------------------------------------
sub GetUser()
{ ## $getkey=<STDIN>;
}

#--------------------------------------------------------------------
#   Display Directory Files
# *** USER INPUT - NOT NEEDED FOR NDATE ***
#--------------------------------------------------------------------
sub DispDirFiles
{
  dump_system_logs();
  create_ems_parsed_log();
}

#--------------------------------------------------------------------
# all variables must be cleared before each new valid ems entry
#--------------------------------------------------------------------
sub init_all()
{
    $cmd              = "";
    $lba              = "";
    $blksz            = "";
    $sSensekey        = "";
    $sSenseCode       = "";
    $scsicode         = "";
    $iSenseKey        = "";
    $iASC             = "";
    $iASCQ            = "";
    $iFRU             = "";
    $DTime            = "";
    $adapterName      = "";
    $ha_status        = "";
    $retryCount       = "";
    $errorno          = "";
    $errortext        = "";
    $diskname         = "";
    $serialno         = "";
    $disk_info        = "";
    $error_str        = "";
    $instanceFile     = "";
    $vendor           = "";
    $model            = "";
    $firmware_revision= "";
    $serialno         = "";
    $disk_type        = "";
    $disk_rpm         = "";
    $failure_reason   = "";
    $port             = "";
    $chan             = "";
    $bay              = "";
    $prodid           = "";
    $fwrev            = "";
    $serno            = "";
    $drvid            = "";

}

#--------------------------------------------------------------------
# Get the ems entry timestamp - this occurs 1x at each new entry 
# you can determine delta times relative to each new timestamp 
#--------------------------------------------------------------------
sub get_newentry
{
    # get time info for every occurance of new entry
    if( $line =~ /<LR/ )
    {
        if( $line =~ /t="(\d+)"/ )
        {
            $timestamp=$1;
            $lineno=$.;
        }
    }
        if( $line =~ /p="(\d+)"/ ){ $priority=$1; }
}

#--------------------------------------------------------------------
# This is the post line processing function
# Each valid (parsed) line entry is written to a tmp file
# the tmp file will be later copied to the final summary file.
#--------------------------------------------------------------------
sub save_data_tofile()
{
    if( $first_time==0 )
    {
        print F_TMP "\nP field : 0=Emergency, 1 ALERT, 2=CRITICAL, 3=error, 4=warning, 5=notice, 6=info, 7=debug";
        print F_TMP "\n\nlineno  timestamp  P    drive ID   cmd       lba          blksz     scsi descritpion        sense code      time               error message desription                  rty           error text                sSenseCode          diskname         serialno        err string   vendor        model         fw    failinfo      disk info";
        print F_TMP "  \n------  ---------- -  ------------ --- ---------------- --------  --------------------   ----------------- -------  --------------------------------------------------   ---   -----------------------------  --------------------  ----------  --------------------  ----------  --------  ----------------  ----  ---------- -------------------------------------------";
        $first_time=1;
    }


    printf F_TMP "\n%6s",$lineno;
    printf F_TMP " %8s ",$timestamp;
    printf F_TMP " %d ",$priority;

    if( $system_type==1)
    {
        printf F_TMP " [%3s",$port;
        printf F_TMP " %3s",$chan;
        printf F_TMP " %2s]",$bay;
    }
    else
    {
        printf F_TMP " [ %6s",$port;
        printf F_TMP " %2s]",$bay;
    }

    $empty="-";

    if( $cmd               ne "" ){ printf F_TMP "  %2s", $cmd              ; }else{ printf F_TMP "  %2s", $empty;  }
    if( $lba               ne "" ){ printf F_TMP " %16s", $lba              ; }else{ printf F_TMP " %16s", $empty;  }
    if( $blksz             ne "" ){ printf F_TMP " %8s", $blksz             ; }else{ printf F_TMP " %8s", $empty;   }
    if( $sSensekey         ne "" ){ printf F_TMP "  %20s", $sSensekey       ; }else{ printf F_TMP "  %20s", $empty; }

    if( $iSenseKey         ne "" ){$scsicode = $iSenseKey." ".$iASC." ".$iASCQ ." ".$iFRU; }
    if( $scsicode          ne "" ){ printf F_TMP "  %18s", $scsicode        ; }else{ printf F_TMP "  %18s", $empty; }

    if( $DTime             ne "" ){ printf F_TMP "  %6s ", $DTime           ; }else{ printf F_TMP "  %6s ",$empty;  }

    printf F_TMP " %-52s",$ems_msgtype;

    if( $retryCount        ne "" ){ printf F_TMP " %3s ", $retryCount       ; }else{ printf F_TMP " %3s ",$empty;  }

    if( $errortext         ne "" ){ printf F_TMP " %30s ", $errortext       ; }else{ printf F_TMP " %30s ",$empty; }
    if( $sSenseCode        ne "" ){ printf F_TMP " %20s", $sSenseCode        ; }else{ printf F_TMP " %20s", $empty; }
    if( $diskname          ne "" ){ printf F_TMP " %10s ", $diskname        ; }else{ printf F_TMP " %10s ",$empty; }
    if( $serialno          ne "" ){ printf F_TMP " %20s ", $serialno        ; }else{ printf F_TMP " %20s ",$empty; }
    if( $error_str         ne "" ){ printf F_TMP " %10s ", $error_str       ; }else{ printf F_TMP " %10s ",$empty; }
    if( $vendor            ne "" ){ printf F_TMP " %8s ", $vendor           ; }else{ printf F_TMP " %8s ",$empty;  }
    if( $model             ne "" ){ printf F_TMP " %16s ", $model           ; }else{ printf F_TMP " %16s ",$empty; }
    if( $firmware_revision ne "" ){ printf F_TMP " %4s ", $firmware_revision; }else{ printf F_TMP " %4s ",$empty;  }
    if( $failure_reason    ne "" ){ printf F_TMP " %10s ", $failure_reason  ; }else{ printf F_TMP " %10s ",$empty; }
    if( $disk_info         ne "" ){ printf F_TMP " %20s ", $disk_info        ; }else{ printf F_TMP " %20s ",$empty;  }

    # these values are commented out and can be added as needed
    #    if( $errorno           ne "" ){ printf F_TMP " %3s ", $errorno          ; }else{ printf F_TMP " %3s ",$empty;  }
    #    if( $adapterName       ne "" ){ printf F_TMP " %2s ", $adapterName      ; }else{ printf F_TMP " %2s ",$empty;  }
    #    if( $ha_status         ne "" ){ printf F_TMP " %3s ", $ha_status        ; }else{ printf F_TMP " %3s ",$empty;  }
    #    if( $disk_type         ne "" ){ printf F_TMP " %2s ", $disk_type        ; }else{ printf F_TMP " %2s ",$empty;  }
    #    if( $disk_rpm          ne "" ){ printf F_TMP " %4s ", $disk_rpm         ; }else{ printf F_TMP " %4s ",$empty;  }
    #    if( $instanceFile      ne "" ){ printf F_TMP " %2s ", $instanceFile     ; }else{ printf F_TMP " %2s ",$empty;  }

}

#--------------------------------------------------------------------
# DO ANALYSIS
#   create summary hashes for use in post processing
#--------------------------------------------------------------------
sub do_analysis()
{
    $errmsg_byid_hash{$ems_msgtype}{$drvid}++;

    if( $scsicode ne "" )
    {
        $scsicode_byid_hash{$scsicode}{$ems_msgtype}{$drvid}++;
    }
}

#--------------------------------------------------------------------
# for development use
#--------------------------------------------------------------------
sub ems_dgb1()
{
    # #*****************************************
    # #           <<< DEBUG >>>
    # #*****************************************
    # # raw dislpay - display all lines
    # #*****************************************
    # if( $line =~ /<\/LR/ ){ $raw_valid=0; }
    # if( $line =~ /<LR/ )  { $raw_valid=1; }
    # elsif( $raw_valid==1 )
    # {
    #     $raw_valid=2;
    #     $rcnt=0;
    #     printf F_ALL "\n%9s : %52s ", $timestamp, $ems_msgtype;
    # }
    # elsif( $raw_valid==2 )
    # {
    #     $rcnt++;
    #     print F_ALL " - $line";
    # }
}

#--------------------------------------------------------------------
# EMS_FILTER_SEARCH
#   This is the main filter routine. Each EMS message is checked and if
#   valid then the entry data will be collected. When the end of the entry
#   is detected then the data is sent to a post line processing function
#   to be saved.
#--------------------------------------------------------------------
sub ems_filter_search()
{
    # test filters
    # don't worry about NULL entries
    if( $line !~ /s="NULL"/ )
    {
           if ( $line =~ /shm_adapter_offline /i )                         { $valid_entry=1; }
        elsif ( $line =~ /shm_disk_.+/i )                                  { $valid_entry=1; }
        elsif ( $line =~ /shm_fab_.+/i )                                   { $valid_entry=1; }
        elsif ( $line =~ /shm_failureBytes_fullError/i )                   { $valid_entry=1; }
        elsif ( $line =~ /shm_reformat_startMsg/i )                        { $valid_entry=1; }
        elsif ( $line =~ /shm_threshold_.+/i )                             { $valid_entry=1; }
        elsif ( $line =~ /disk_rewriteData/i )                             { $valid_entry=1; }
        elsif ( $line =~ /disk_rewriteDataSuccessful/i )                   { $valid_entry=1; }
        elsif ( $line =~ /disk_senseError/i )                              { $valid_entry=1; }
        elsif ( $line =~ /disk_write_failure/i )                           { $valid_entry=1; }
        elsif ( $line =~ /esh_bypass_.+/i )                                { $valid_entry=1; }
        elsif ( $line =~ /raid_disk_io_.+/i )                              { $valid_entry=1; }
        elsif ( $line =~ /scsi_cmd_.+/i )                                  { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_adapter_reset/i )                    { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_bad/i )                              { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_bus_reset/i )                        { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_bus_scan_failed/i )                  { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_bus_scan_timeout/i )                 { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_command_timeout/i )                  { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_detected_unexpected_bus_phase/i )    { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_device_reset/i )                     { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_device_reset_failed/i )              { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_device_timeout/i )                   { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_devRequestFail/i )                   { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_error/i )                            { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_failed/i )                           { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_firmware_download/i )                { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_firmware_fault/i )                   { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_firmware_update_failed/i )           { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_inq_cmd_failed/i )                   { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_inq_cmd_success/i )                  { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_inq_cmd_timeout/i )                  { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_io_channel_failed/i )                { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_not_ready/i )                        { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_offline/i )                          { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_offline_fail/i )                     { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_offlining/i )                        { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_online/i )                           { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_onlining/i )                         { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_timeoutd_quiescing/i )               { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_timeoutd_unquiescing/i )             { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_bus_scan_timeout/i )                 { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_command_timeout/i )                  { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_device_timeout/i )                   { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_inq_cmd_timeout/i )                  { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_timeoutd_quiescing/i )               { $valid_entry=1; }
        elsif ( $line =~ /scsiAdapter_timeoutd_unquiescing/i )             { $valid_entry=1; }
        elsif ( $line =~ /scsitarget_command_hung/i )                      { $valid_entry=1; }
        elsif ( $line =~ /scsitarget_write_failure/i )                     { $valid_entry=1; }
        elsif ( $line =~ /scsitarget_write_failureUnitAttention/i )        { $valid_entry=1; }
        elsif ( $line =~ /bge_PhyTimeout/i )                               { $valid_entry=1; }
        elsif ( $line =~ /fci_device_.+/i )                                { $valid_entry=1; }
        elsif ( $line =~ /sas_device_.+/i )                                { $valid_entry=1; }
        elsif ( $line =~ /fci_discovery_timeout/i )                        { $valid_entry=1; }
        elsif ( $line =~ /raid_disk_timeout_.+/i )                         { $valid_entry=1; }
        elsif ( $line =~ /sas_device_timeout/i )                           { $valid_entry=1; }
        elsif ( $line =~ /sfu_partnerUpdateTimeout/i )                     { $valid_entry=1; }
        elsif ( $line =~ /shm_disk_ioTimeout/i )                           { $valid_entry=1; }
        elsif ( $line =~ /shm_threshold_consecutiveTimeouts/i )            { $valid_entry=1; }
        elsif ( $line =~ /monitor_raid_brokenDisk/i )                      { $valid_entry=1; }
        elsif ( $line =~ /monitor_raid_reconstruct/i )                     { $valid_entry=1; }
        elsif ( $line =~ /monitor_raidDiskFail/i )                         { $valid_entry=1; }
        elsif ( $line =~ /monitor_raiddp_vol_singleDegraded/i )            { $valid_entry=1; }
        elsif ( $line =~ /raid_assim_noDisks/i )                           { $valid_entry=1; }
        elsif ( $line =~ /raid_cksum_bad_file_block/i )                    { $valid_entry=1; }
        elsif ( $line =~ /raid_cksum_verify_error/i )                      { $valid_entry=1; }
        elsif ( $line =~ /raid_cksum_wc_blkErr/i )                         { $valid_entry=1; }
        elsif ( $line =~ /raid_cksum_wc_recFail/i )                        { $valid_entry=1; }
        elsif ( $line =~ /raid_cksum_zeroed_blkErr/i )                     { $valid_entry=1; }
        elsif ( $line =~ /raid_cksum_zeroed_sblkErr/i )                    { $valid_entry=1; }
        elsif ( $line =~ /raid_data_lw_blkErr/i )                          { $valid_entry=1; }
        elsif ( $line =~ /raid_data_lw_recFail/i )                         { $valid_entry=1; }
        elsif ( $line =~ /raid_data_ws_blkErr/i )                          { $valid_entry=1; }
        elsif ( $line =~ /raid_disk_missing/i )                            { $valid_entry=1; }
        elsif ( $line =~ /raid_disk_online_fail/i )                        { $valid_entry=1; }
        elsif ( $line =~ /raid_disk_predictiveFailure/i )                  { $valid_entry=1; }
        elsif ( $line =~ /raid_disk_pulled/i )                             { $valid_entry=1; }
        elsif ( $line =~ /raid_disk_removed/i )                            { $valid_entry=1; }
        elsif ( $line =~ /raid_disk_timeout_.+/i )                         { $valid_entry=1; }
        elsif ( $line =~ /raid_label_io_.+/i )                             { $valid_entry=1; }
        elsif ( $line =~ /raid_multierr_bad_block/i )                      { $valid_entry=1; }
        elsif ( $line =~ /raid_read_cksum_err/i )                          { $valid_entry=1; }
        elsif ( $line =~ /raid_read_media_err/i )                          { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_media_scrub_cksum_blk_err_bypass/i )     { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_recons.+/i )                             { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_recons_multidisk_bypass/i )              { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_scrub_media_err_bypass/i )               { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_disk_reconstruction_failed/i )           { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_media_scrub_media_err/i )                { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_media_scrub_media_errs/i )               { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_readerr_wc_blkErr/i )                    { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_recons_start/i )                         { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_scrub_cksum_err/i )                      { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_scrub_cksum_errs/i )                     { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_scrub_cksum_zero/i )                     { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_scrub_media_err/i )                      { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_scrub_media_err_bypass/i )               { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_scrub_media_errs/i )                     { $valid_entry=1; }
        elsif ( $line =~ /raid_rg_scrub_pi/i )                             { $valid_entry=1; }
        elsif ( $line =~ /raid_spares_media_scrub_media_err/i )            { $valid_entry=1; }
    }

    ###################################################################
    # always clean up the EMS Message info, whether it is used or not
    ###################################################################
    if( $line =~ /<LR / ){ $msgline_flg=1; }
    elsif( $msgline_flg==1 )
    {
        $msgline_flg=0;
        $ems_msgtype = $line;
        $ems_msgtype =~ s/<//;
        # EMS currently has 7 levels of message entries - check the wiki for details
        if($ems_msgtype =~ /_1/ ){ $var1 = "_1"; }
        if($ems_msgtype =~ /_2/ ){ $var1 = "_2"; }
        if($ems_msgtype =~ /_3/ ){ $var1 = "_3"; }
        if($ems_msgtype =~ /_4/ ){ $var1 = "_4"; }
        if($ems_msgtype =~ /_5/ ){ $var1 = "_5"; }
        if($ems_msgtype =~ /_6/ ){ $var1 = "_6"; }
        if($ems_msgtype =~ /_7/ ){ $var1 = "_7"; }
        $ems_msgtype =~ s/$var1//;
        $var1="/>";
        $ems_msgtype =~ s/$var1//;
        # save a master list of all errmsgs - and display this at the end of the file
        $ems_msg_all_summ{$ems_msgtype}++;
    }

    ###################################################################
    # look for END-OF-ENTRY and turn off valid entry flag
    ###################################################################
    if(( $line =~ /<\/LR>/ )&&( $valid_entry==2 ))
    {
        $valid_entry=0;
        save_data_tofile();
        do_analysis();
        init_all();
    }

    ###################################################################
    # Valid entry is detected - save EMS message for summary display 
    # and transition to the next state
    ###################################################################
    if( $valid_entry==1 )
    {
        $valid_entry=2;
        # printf F_SUMM "\n%9s : %52s ", $timestamp, $ems_msgtype;
        # collect summary of valid error msgs
        $ems_msg_summ{$ems_msgtype}++;
    }
    elsif( $valid_entry==2 )
    {
        # collect the entry data for post line processing
        # THIS MAY NEED UPDATING AS NEW DATA VARIABLES ARE DETECTED
        #
        if( $line =~ /sSenseKey="(.+)"/i )          { $sSensekey        = $1; }
        if( $line =~ /sSenseCode="(.+)"/i )         { $sSenseCode       = $1; }
        if( $line =~ /iSenseKey="(.+)"/i )          { $iSenseKey        = $1; }
        if( $line =~ /iASC="(.+)"/i )               { $iASC             = $1; }
        if( $line =~ /iASCQ="(.+)"/i )              { $iASCQ            = $1; }
        if( $line =~ /iFRU="(.+)"/i )               { $iFRU             = $1; }
        if( $line =~ /DTime="(.+)"/i )              { $DTime            = $1; }
        if( $line =~ /adapterName="(.+)"/i )        { $adapterName      = $1; }
        if( $line =~ /ha_status="(.+)"/i )          { $ha_status        = $1; }
        if( $line =~ /retryCount="(.+)"/i )         { $retryCount       = $1; }
        if( $line =~ /errorno="(.+)"/i )            { $errorno          = $1; }
        if( $line =~ /errortext="(.+)"/i )          { $errortext        = $1; }
        if( $line =~ /diskname="(.+)"/i )           { $diskname         = $1; }
        if( $line =~ /serialno="(.+)"/i )           { $serialno         = $1; }
        if( $line =~ /disk_info="(.+)"/i )          { $disk_info        = $1; }
        if( $line =~ /error_str="(.+)"/i )          { $error_str        = $1; }
        if( $line =~ /instanceFile="(.+)"/i )       { $instanceFile     = $1; }
        if( $line =~ /vendor="(.+)"/i )             { $vendor           = $1; }
        if( $line =~ /model="(.+)"/i )              { $model            = $1; }
        if( $line =~ /firmware_revision="(.+)"/i )  { $firmware_revision= $1; }
        if( $line =~ /disk_type="(.+)"/i )          { $disk_type        = $1; }
        if( $line =~ /disk_rpm="(.+)"/i )           { $disk_rpm         = $1; }
        if( $line =~ /failure_reason="(.+)"/i )     { $failure_reason   = $1; }
    
        if( $line =~ /diskName="(\w+)\.(\w+)"/i )       { $port=$1;           $bay=$2; $drvid=$1." ".$2;         $system_type=2; $t1=1; }
        if( $line =~ /diskName="(\w+)\.(\w+)\.(\w+)"/i ){ $port=$1; $chan=$2; $bay=$3; $drvid=$1." ".$2." ".$3;  $system_type=1; $t1=1; }

        if( $line =~ /disk="(\w+)\.(\w+)"/i )       { $port=$1;           $bay=$2; $drvid=$1." ".$2;         $system_type=2; $t1=1; }
        if( $line =~ /disk="(\w+)\.(\w+)\.(\w+)"/i ){ $port=$1; $chan=$2; $bay=$3; $drvid=$1." ".$2." ".$3;  $system_type=1; $t1=1; }

        if( $line =~ /disk_info="Disk / )
        {
            if( $line =~ /(\w+)\.(\w+) Shelf/i )       { $port=$1;           $bay=$2; $drvid=$1." ".$2;         $system_type=2; $t1=2; }
            if( $line =~ /(\w+)\.(\w+)\.(\w+) Shelf/i ){ $port=$1; $chan=$2; $bay=$3; $drvid=$1." ".$2." ".$3;  $system_type=1; $t1=2; }
        }

        if( $line =~ /deviceName="(\w+)\.(\w+)"?\s?/i )       { $port=$1;           $bay=$2; $drvid=$1." ".$2;         $system_type=2; $t1=3; }
        if( $line =~ /deviceName="(\w+)\.(\w+)\.(\w+)"?\s?/i ){ $port=$1; $chan=$2; $bay=$3; $drvid=$1." ".$2." ".$3;  $system_type=1; $t1=3; }

        if( $line =~ /cdb="0x(\w+):?(\w*?):?(\w*?)"/ )      { $cmd=$1; $lba=$2; $blksz=$3; }
        
        if( $t1>0 ){ print FD "\n$drvid --$system_type --- $t1 $."; print FD " - $line"; }
        $t1=0;
        
    }

}

#--------------------------------------------------------------------
# Serial Number search - this routine looks for all occuracnes of SN's
# in the log file. The data is collected and used to make the output
# more verbose.
#--------------------------------------------------------------------
sub ems_driveinfo_search()
{
    # look for start and end of entrys
    if( $line =~ /<\/LR/ ){ $active_flg=0; }
    if( $line =~ /<LR/ )  { $active_flg=1; }

    # when we find and entry, look for drive info we can add to a database
    if( $active_flg==1 )
    {
        if( $line =~ /disk_info="Disk / )
        {
            if( $line =~ /(\w+)\.(\w+)\.?(\w*?) Shelf/i )
            {
                if( $3 eq "" )
                {   $t_system_type=2;
                    $t_port       =$1;
                    $t_bay        =$2;
                    $t_drvid      =$2;
                }else
                {   $t_system_type=1;
                    $t_port       =$1;
                    $t_chan       =$2;
                    $t_bay        =$3;
                    $t_drvid      =$2." ".$3;
                }
            }

            if( $line =~ /\[NETAPP\s+(\w+)\s+(\w+)\] /i )
            {
                $t_prodid=$1;
                $t_fwrev =$2;
            }
            if( $line =~ /S\/N \[(.+)\]"/i )
            {
                $t_serno=$1;
            }

            # ---------------------------------------
            # build database
            # ---------------------------------------
            %storage_info=(
                            PORT   => $t_port,
                            CHAN   => $t_chan,
                            BAY    => $t_bay,
                            PRODID => $t_prodid,
                            FWREV  => $t_fwrev,
                            SN     => $t_serno,
                            );
            # hash info is unique to the drive ID
            $ems_disk_info{ $t_drvid }={ %storage_info };
        }
    }
}

#--------------------------------------------------------------------
# call this routine with valid $drvid
#--------------------------------------------------------------------
sub disp_drive_info()
{
    #print F_SUMM "  <- $ems_disk_info{$drvid}->{PORT}";
    #print F_SUMM "   $ems_disk_info{$drvid}->{CHAN}";
    #print F_SUMM "   $ems_disk_info{$drvid}->{BAY}";
    printf F_SUMM "  %16s", $ems_disk_info{$drvid}->{PRODID};

    $len=length($ems_disk_info{$drvid}->{PRODID});
    $dongle_type=substr($ems_disk_info{$drvid}->{PRODID}, ($len-3));
    if( $dongle_type eq "SDB" ){ print F_SUMM "  Marvell "; }
    if( $dongle_type eq "SSM" ){ print F_SUMM "  Acorn   "; }
    if( $dongle_type eq "SSA" ){ print F_SUMM "  Filbert "; }
    else                       { print F_SUMM "          "; }

    printf F_SUMM " %4s ", $ems_disk_info{$drvid}->{FWREV};
    printf F_SUMM " %20s", $ems_disk_info{$drvid}->{SN};
}

#--------------------------------------------------------------------
# POST - Processing
#   
#--------------------------------------------------------------------
sub post
{
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # The following code segment requires NDATE LOG parser to run first
    # which generates the cwpar_sysconfig.txt file". This log contains more
    # drive info that will be added to this output to make more readable.
    #
    $sysconf_flg=0;
    open(F1, "<cwpar_sysconf.log")|| ($sysconf_flg=1);
    # if file found then read in and process
    #
    if( $sysconf_flg==0 )
    {
        while(<F1>)
        {
            chomp($line = $_);
            #print "\n$line";
        }
    }
#    unlink("cwpar_sysconf.log");

    printf F_SUMM "\n###########################################################################################################";
    printf F_SUMM "\n\n     EMS (Event Management System) SUMMARY LOG PARSED FILE    <<< NetApp Hdd Eng Group >>>   ver1.0\n";
    printf F_SUMM "\n###########################################################################################################";

    printf F_SUMM "\n\n===========================================================================================================";
    printf F_SUMM "  \n##### Summary of Filtered EMS Error Messages #####";
    printf F_SUMM "  \n===========================================================================================================";
    foreach $emsg (sort keys %ems_msg_summ)
    {
        printf F_SUMM "\n- %-52s  = %6d ", $emsg, $ems_msg_summ{$emsg};
    }

    printf F_SUMM "\n\n\n===========================================================================================================";
    printf F_SUMM "    \n##### Count of SCSI Sense Error Codes by Drive #####";
    printf F_SUMM "    \n===========================================================================================================";
    print F_SUMM "\n\n drive ID     sensecode    cnt         prodid        dongle    fw          serno                       Error Description";
    print F_SUMM "  \n-----------   ----------  -----    ----------------  -------  ----  --------------------   ------------------------------------------------";

    foreach $ecode (keys %scsicode_byid_hash)
    {
        foreach $emsg (keys %{$scsicode_byid_hash{$ecode}})
        {
            foreach $id (keys %{$scsicode_byid_hash{$ecode}{$emsg}})
            {
                $tmp=$ecode;
                $tmp =~ s/0x//g;
                @pecodes = split " ",$tmp;
                @pid = split " ",$id;
                if( $system_type==1){
                    $drvid = $pid[1]." ".$pid[2];
                    printf F_SUMM "\n[%3s%3s%3s]  ",$pid[0],$pid[1],$pid[2];
                }
                else{
                    $drvid = $pid[1];
                    printf F_SUMM "\n[%3s%3s]     ",$pid[0],$pid[1];
                }
                printf F_SUMM " %.2x", hex($pecodes[0]);
                printf F_SUMM " %.2x%.2x", hex($pecodes[1]), hex($pecodes[2]);
                printf F_SUMM " %.2x", hex($pecodes[3]);
                printf F_SUMM "  =%4d  ", $scsicode_byid_hash{$ecode}{$emsg}->{$id};

                disp_drive_info();

                printf F_SUMM "   %s ",$emsg;
            }
        }
    }

    printf F_SUMM "\n\n\n===========================================================================================================";
    printf F_SUMM "    \n##### Count of Filtered EMS Error Messages by Drive #####";
    printf F_SUMM "    \n===========================================================================================================";
    foreach $emsg (sort keys %errmsg_byid_hash)
    {
        printf F_SUMM "\n\n$emsg:";
        $cnt=length($emsg);
        print F_SUMM "\n"; print F_SUMM '_' x $cnt;
        foreach $id (keys %{$errmsg_byid_hash{$emsg}})
        {
            @pid = split " ",$id;
            if( $system_type==1){
                $drvid = $pid[1]." ".$pid[2];
                printf F_SUMM "\nid=[%3s%3s%3s]    = %4d  ",$pid[0],$pid[1],$pid[2], $errmsg_byid_hash{$emsg}->{$id};
            }
            else{
                $drvid = $pid[1];
                printf F_SUMM "\nid=[%3s%3s]       = %4d  ",$pid[0],$pid[1], $errmsg_byid_hash{$emsg}->{$id};
            }

            disp_drive_info();
        }
    }

    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # copy line entry details to summary file
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    open(FIN, "<$syslog_dir/cw_ems_tmp.txt")|| die "Could not open $filename...";
    printf F_SUMM "\n\n\n===========================================================================================================";
    printf F_SUMM "    \n##### EMS Error Message Line Details #####";
    printf F_SUMM "    \n===========================================================================================================";
    while(<FIN>){ chomp($line = $_); print F_SUMM "\n$line"; }
    close(FIN);
    unlink("$syslog_dir/cw_ems_tmp.txt");

    printf F_SUMM "\n\n\n===========================================================================================================";
    printf F_SUMM "    \n##### Summary of ALL EMS Messages < Non-Filtered > #####";
    printf F_SUMM "    \n===========================================================================================================";
    foreach $emscode (sort keys %ems_msg_all_summ)
    {
        printf F_SUMM "\n-> %-52s  = %6d", $emscode, $ems_msg_all_summ{$emscode};
    }
}

#====================================================================
#   MAIN
#====================================================================
sub create_ems_parsed_log{
  foreach my $ems_file(@EMS_FILES)
  {
		foreach my $ems_dir(@EMS_DIR){
#	print  "the value  \n \n $ems_dir\n \n ------ \n";
	open(FIN, "<$ems_file")|| die "Could not open $filename...";
	#print " the value  \n \n $ems_file \n \nyyyyyyy     $ems_dir ";
	$syslog_dir = $ems_dir;
	open(F_SUMM,">$syslog_dir/cw_ems_parsed.log")|| die "Could not open $filename...";
	#open(F_ALL, ">cw_ems_all_parsed.txt")|| die "Could not open $filename...";
	open(F_TMP, ">$syslog_dir/cw_ems_tmp.txt")|| die "Could not open $filename...";
	#open(FD, ">dbg1.txt");

	while(<FIN>)
	{
	    chomp($line = $_);
	    $var1="/>";
	    $line =~ s/$var1//;
	    get_newentry();
	    ems_filter_search();
	    ems_driveinfo_search();
	#    ems_dgb1();

   	}
	close(FIN);
	#close(F_ALL);
	close(F_TMP);
	#close(FD);

	post();

	# summary log is closed after post processing
	close(F_SUMM);

	print "\n\n- Processing complete...";
  }
}
	print "Calling new simple parser - ems file\n";
	print "Main Log dir - $logpath\n";
	$temp_log_path;
	opendir( DIR_LOG, $logpath );
    my @files_DIR_LOG = readdir(DIR_LOG);
    foreach my $file (@files_DIR_LOG) {
        if ($file =~ /SYSTEM_LOGS/){
            print "Directory found - $file\n";
            $temp_log_path = "$logpath"."$file";
            opendir(SYS_LOG, $temp_log_path);
            my @files = readdir(SYS_LOG);
            foreach my $dfile (@files) {
                if ( $dfile !~ /\./ ){
                    print "Temp Log path - $temp_log_path\n";
                    opendir(RD_EMS, "$temp_log_path"."/"."$dfile");
                    print "Temp Log path - $temp_log_path"."/"."$dfile\n";
                    my @ems_files = readdir(RD_EMS);
                    foreach my $chk_ems_file (@ems_files) {
                        if ($chk_ems_file =~ /^ems$/){
                            print "EMS file to parse = $chk_ems_file\n";
                            my $par_ems_file = "$temp_log_path"."/"."$dfile"."/"."$chk_ems_file";
                            parse_simple_ems($par_ems_file);
                            print "Parsing done \n";
                            big_ems_all($par_ems_file);
			    print "EMS Parsing done\n";
			}
                    }
                }
            }
        }
    }
}

sub parse_simple_ems {
    my $file = shift;
    my $new_log;
    my $filename;
    print "File to parse - $file \n";
    if ($file =~ /(\S+)\/(.*)\/ems/) {
         $new_log = $1;
         $filename = $2;
    }
    print "New log - $new_log \n";
    open(FIN, "<$file")|| die "Could not open $file...";
    my $outputfile = "$filename"."_EMS.log";
    $outputfile = "$new_log"."/"."$filename"."/"."$outputfile";
    print "Output file - $outputfile\n";
    open(FOUT, ">$outputfile") || die "Could not open $file";
    while(<FIN>)
    {
        chomp($line=$_);
        if( $line =~ /\/LR/ ){
        # for future filtering options
        if(( $title =~ /dummy/ )or
           ( $title =~ /dummy/ )or
           ( $title =~ /dummy/ )){
           # filters - do nothing
        }
        else{
            printf FOUT "\n";
            printf FOUT " %s", $date;
            printf FOUT " %s", $time;
            printf FOUT " %s", $name;
            printf FOUT " %u", $tsec;
            printf FOUT "%50s ",$title;

            foreach $info (@data){
                printf FOUT " $info";
            }
        }
    }

    if( $line =~ /<LR d="(\S+) (\S+)" n="(\S+)" t="(\d+)"/ ){
        $date = $1;
        $time = $2;
        $name = $3;
        $tsec = $4;
        @data = ();
    }

    if( $line =~ /^\<(\S+)_\S+/ ){
        $title = $1;
    }

    if( $line =~ /^\s+\S+="(\S+)"/ ){
       push @data, $1;
    }
  }
}

sub dump_system_logs
{
	my $host;
	my $hostB;
	my $filer;
	my $filerB;
	my @filers_rsh;
	my $filer_con_rsh;
	my $filer_conB_rsh;
	#Set filer param based on system type, C for cluster and D for dual setup
	if ( "$TEST_CONFIG" eq "C" ) {
	    $host       = host($FILERA);
		$hostB      = host($FILERB);
		$filer_con_rsh = connect("$FILERA#rsh");
		$filer_conB_rsh= connect("$FILERB#rsh");
		push(@filers_rsh, $filer_con_rsh);
	} elsif("$TEST_CONFIG" eq "D") { 
		$host      = host($FILER);
		$filer_con_rsh = connect("$FILER#rsh");
		push(@filers_rsh, $filer_con_rsh);
	}
	
	my $fatal_error = 0;
	### Check for FATAL error in log file.
	if ($LOGDIR =~ /(\S+)HDD(\S+)/){
        my $MAIN = $1;
        chomp($MAIN);
        opendir(DIR_LOG, $MAIN);
        my @files_DIR_LOG = readdir(DIR_LOG);
        foreach my $file (@files_DIR_LOG){
                if ( $file =~ /(.*_NADQ02.*)\.log/ ) {
                        @ary = `tail -n 10 $MAIN/$file`;
                        foreach my $line(@ary){
                                if ($line =~ /FATAL/i){
                                    $fatal_error = 1;
                                }
                        }
                }
        }
	}

	print "The fata value is $fatal_error\n";
	
	###Generate the system logs in case of fatal failure
	if ( $fatal_error == 1 ) {
		print "Generating the system logs";
		foreach $Filer_con (@filers_rsh) {
		$FILER_C = $FILER;
		$FILER_C = $FILERB if ( $Filer_con eq $filer_conB_rsh );
		$Host    = $host;
		$Host    = $hostB if ( $Filer_con eq $filer_conB_rsh );
	
		my $output;
		my $logdir;
		$logdir = "$LOGDIR/../../../System_Logs_".$FILER_C;
		my $f_ip;
		$f_ip = $Host->default_ip();
		print "The directory containing the system logs is: $logdir";
		print "The filer IP is: $f_ip";
	
		print "$FILER_C - Dumping system log\n";

		$output = get_system_logs(
			host       => $Host,
			logdir     => $logdir,
			filerip    => $f_ip,
			);
		}
	}
}

sub big_ems_all{
	my $file = shift;
   	my $new_log;
	my $filename;
	print "File to parse - $file \n";
    	if ($file =~ /(\S+)\/(.*)\/ems/) {
        	$new_log = $1;
		$filename = $2;
    	}
    	print "New log - $new_log \n";
    	open(FIN, "<$file")|| die "Could not open $file...";
    	my $outputfile = "$filename"."_EMS_ALL.log";
    	$outputfile = "$new_log"."/"."$filename"."/"."$outputfile";
    	print "Output file - $outputfile\n";
    	open(FOUT, ">$outputfile") || die "Could not open $file";
    	while(<FIN>){
		chomp($line=$_);
    		if( $line =~ /<LR d="(\S+) (\S+)" n="(\S+)" t="(\d+)"/ ){
        		$date=$1;
        		$time=$2;
        		$name=$3;
        		$tsec=$4;
		        printf FOUT "\n$date";
		        printf FOUT " $time";
		        printf FOUT " $name";
		        printf FOUT " $tsec";
		        $dsec = $tsec - $ptsec;
		        printf FOUT " %6u ",$dsec;
		        $ptsec = $tsec;
	    	}
		elsif( $line =~ /<(\S+)_\d+/ ){
	        $ems_msg=$1;
        	printf FOUT " %40s : ", $ems_msg;
	    	}
		elsif( $line =~ /"(.*?)"/ ){
	        	$data=$1;
		        $data =~ s/\/>//;
      
	        printf FOUT " $data";
	    	}
	}
	close(FIN);
	close(FOUT);
	print "\n\nprocessing complete";
}