#!/usr/software/bin/python2.7 -utt

import sys

sys.path.append('/u/rathishk/s2t_concurrent/lib/python2.7/site-packages/futures-2.1.3-py2.7.egg')
lib_dir = "/u/rathishk/p4/ncov/depot/prod/rats/tools/ncov_tool/tools/"
sys.path.append(lib_dir)
gat_dir = "/x/eng/webgrps/gat/"
sys.path.append(gat_dir)

import math
from optparse import OptionParser
import os
import commands
import datetime
import concurrent.futures
import scriptUtil
import gcovUtil
from struct import *
import re
from operator import itemgetter
import shutil
import string

def print_file(file, msg):

#################################################################################
# Sub routine to print the data in the file                                     #
#################################################################################

  log_f=open(file, "a")
  if (msg != "*"):
    msg = "%s\n" % msg
  log_f.write("%s" % msg)
  log_f.close()

def read_gcno_data(fh, big_endian, length, type, reset_pos):

#################################################################################
# Read gcno string/value/word from the gcno file depending the type it is called#
#################################################################################

  if (type == "string"):
    length = read_gcno_data(fh, big_endian, -1, "value", False)
    if length is None:
      return None
    elif length == 0:
      return ""
    else:
      length = length * 4
    string = fh.read(length)
    if string is None:
      return None
    return string.replace("\0","")
  elif (type == "value"):
    word = read_gcno_data(fh, big_endian, -1, "word", reset_pos)
    if word is None:
      return None
    if (big_endian == 1):
      return unpack("!L", word)[0]
    else:
      return unpack("=L", word)[0]
  elif (type == "word"):
    if(reset_pos):
      pos = fh.tell()
    data = fh.read(4)
    if(reset_pos):
      seek = fh.seek(pos, 0)
    return data
  elif (type == "skip"):
    return fh.read(length)

def is_func_checksum(fh, big_endian, length, log_file_name):

#################################################################################
# Determine if there is checksum present in function data                       #
#################################################################################
  
  cmd = "/usr/software/build/compilers/i386-unknown-freebsd7-gcc-4.2.1_p11/bin/i386-unknown-freebsd7-gcov -v"
  (status, output) = commands.getstatusoutput(cmd)
  if(status != 0):
    print_file(log_file_name, "Get GCOV version using 'gcov -v' : FAILED")
    return False
  match = re.search("(\d+)\.(\d+)\.?(\d+)?", output)
  version = match.group().split(".")
  if(int(version[0]) >= 4 and int(version[1]) >= 7):
    return True
  str_len = read_gcno_data(fh, big_endian, -1, "value", True)
  if (str_len * 4) >= (length - 12): 
    return True
  return False

def normalize_path(file_name, base):

#################################################################################
# Normalize the Source File with respect to the build root location             #
#################################################################################
 
  global buildroot
  result = base
  build_name = buildroot.rstrip("/").split("/")[-1]
  if(not os.path.isabs(base)):
    result = "%s/%s/%s" % (file_name.split(build_name)[0], build_name, base)

  result = os.path.normpath(result)
  match = re.search("^/x/eng/rlse(.+)/(.*?)/nightly/", result)
  if match:
    pattern = "/x/eng/rlse%s/%s/nightly/" % (match.group(1), match.group(2))
    replace = "/x/eng/rlse/%s/" % (match.group(2))
    result = result.replace(pattern, replace)
  match = re.search("^/x/eng/nrlse(.+)/(.*?)/nightly/", result)
  if match:
    pattern = "/x/eng/nrlse%s/%s/nightly/" % (match.group(1), match.group(2))
    replace = "/x/eng/rlse/%s/" % (match.group(2))
    result = result.replace(pattern, replace)

  return result

def read_gcno_func_data(fh, big_endian, length, log_file_name):

#################################################################################
# Read gcno function data from the gcno file                                    #
#################################################################################

#################################################################################
# Skip 8 bytes of Function Ident and Checksum                                   #
#################################################################################

  read_gcno_data(fh, big_endian, 8, "skip", False)
  if(is_func_checksum(fh, big_endian, length, log_file_name)):

#################################################################################
# Skip 4 bytes Function Cfg Checksum                                            #
#################################################################################

    read_gcno_data(fh, big_endian, 4, "skip", False)
  function = read_gcno_data(fh, big_endian, -1, "string", False)
  file = read_gcno_data(fh, big_endian, -1, "string", False)
  line_num = read_gcno_data(fh, big_endian, -1, "value", False)
  if (not (function is None) and not (file is None) and not (line_num is None)):
    return (file, function, line_num)
  else:
    return (None, None, None)

def read_gcno_line_data(fh, big_endian, file_name, function, log_file_name):

#################################################################################
# Read gcno line data from gcno file                                            #
#################################################################################
 
  line_nums = []
  empty_data = []

#################################################################################
# Skip 4 bytes of basic block index                                             #
#################################################################################

  read_gcno_data(fh, big_endian, 4, "skip", False)
  while (1):
    line_num = read_gcno_data(fh, big_endian, -1, "value", False)
    if line_num is None:
      return (None, empty_data)
    if line_num == 0:
       string = read_gcno_data(fh, big_endian, -1, "string", False)
       if string is None:
         return (None, empty_data)
       if string == "":
         return (file_name, line_nums)
       file_name = string
       continue

    if file_name is None:
      print_file(log_file_name, "WARNING: unassigned line number")
      continue

    line_nums.append(line_num)

def read_gcno_file(filename, log_file_name):

#################################################################################
# Read gcno file                                                                #
#################################################################################

  big_endian_check = 0x67636e6f
  tag_function = 0x01000000
  tag_lines = 0x01450000
  file_name = None
  function = None
  file_data = {}
  empty_data =  {}

  fh = open(filename, 'rb')

#################################################################################
# First 4 Bytes File Magic data                                                 #
#################################################################################

  data = read_gcno_data(fh, -1, -1, "word", False)

  if (unpack("!L", data)[0] == big_endian_check):
    big_endian = 1
  elif (unpack("=L", data)[0] == big_endian_check):
    big_endian = 0
  else:
    print_file(log_file_name, "found unrecognized gcno file magic")
    return empty_data

#################################################################################
# Skip 8 bytes of version and stamp data                                        #
#################################################################################

  read_gcno_data(fh, big_endian, 8, "skip", False)
  base = read_gcno_data(fh, big_endian, -1, "string", False)
  base = normalize_path(filename, base)

  while (fh.tell() != os.fstat(fh.fileno()).st_size):
    tag = read_gcno_data(fh, big_endian, -1, "value", False)
    if (tag is None):
      print_file(log_file_name, "Unexpected end of file")
      return empty_data
    length = read_gcno_data(fh, big_endian, -1, "value", False)
    if (length is None):
      print_file(log_file_name, "Unexpected end of file")
      return empty_data
    length = length * 4
    next_pos = fh.tell()
    if next_pos == -1:
      print_file(log_file_name, "Could not determine the file position")
      return empty_data
    next_pos = next_pos + length
    if (tag == tag_function):
      (file_name, function, line_num) = read_gcno_func_data(fh, big_endian, length, log_file_name)
      abs_file_name = "%s/%s" % (base, file_name)
      if abs_file_name not in file_data:
        file_data[abs_file_name] = {}
        file_data[abs_file_name]['function'] = {}
        file_data[abs_file_name]['line'] = []
      if function is None:
        print_file(log_file_name, "Unexpected end of file")
        return empty_data
      file_data[abs_file_name]['function'][function] = line_num
    elif (tag == tag_lines):
      (file_name, line_num) = read_gcno_line_data(fh, big_endian, file_name, function, log_file_name)
      abs_file_name = "%s/%s" % (base, file_name)
      if abs_file_name not in file_data:
        file_data[abs_file_name] = {}
        file_data[abs_file_name]['function'] = {}
        file_data[abs_file_name]['line'] = []
      if file_name is None:
        print_file(log_file_name, "Unexpected end of file")
        return empty_data
      file_data[abs_file_name]['line'].extend(line_num)
    else:
      read_gcno_data(fh, big_endian, length, "skip", False)
    curr_pos = fh.tell()
    if curr_pos == -1:
      print_file(log_file_name, "Could not determine the file position")
      return empty_data
    if curr_pos == next_pos:
      continue
    if curr_pos > next_pos:
      print_file(log_file_name, "found unrecognized gcno file format")
      return empty_data
    read_gcno_data(fh, big_endian, next_pos - curr_pos, "skip", False)

  fh.close()
  return file_data

def create_stub_info(file_name, info_file, log_file_name):

#################################################################################
# Create stub info for the given gcno file by parsing the data in it            #
#################################################################################

  global buildroot

  try:
    file_data = read_gcno_file(file_name, log_file_name)

    if(len(file_data) == 0):
      print_file(log_file_name, "Length file_data = %d" % len(file_data))
      return False

    info_f=open(info_file, "a")
  
    for file in file_data.keys():
      if(not os.path.isfile(file)):
        continue

      info_f.write("TN:\n")
      info_f.write("SF:%s\n" % file)

      sorted_func_data =  sorted(file_data[file]['function'].iteritems(), key=itemgetter(1))
      for func_det in sorted_func_data:
        info_f.write("FN:%s,%s\n" % (func_det[1], func_det[0]))

      for func_det in sorted_func_data:
        info_f.write("FNDA:0,%s\n" % (func_det[0]))
        print_file(log_file_name, "*")

      info_f.write("FNF:%d\n" % len(file_data[file]['function']))
      info_f.write("FNH:0\n")

      for i in file_data[file]['line']:
        info_f.write("DA:%d,0\n" % i)
        print_file(log_file_name, "*")

      info_f.write("LF:%s\n" % len(file_data[file]['line']))
      info_f.write("LH:0\n")
      info_f.write("end_of_record\n")
      print_file(log_file_name, "*")

    info_f.write("# %s\n" % buildroot)
    info_f.close()
    print_file(log_file_name, "\n")
  #except KeyError:
  except:
    print_file(log_file_name, "Unexpected error: %s %s" %  (sys.exc_info()[0], sys.exc_info()[1]))
    os.remove(info_file)
    return False

  return True

def stub_info_process(file_name):

#################################################################################
# Function which calls the function to create stub info for the given gcno file #
#################################################################################

#################################################################################
# log files and directories are prefixed with the date-time stamp               #
#################################################################################

  now = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")
  log_file_name = "%s/stub_info_logs/Job%s.log" % (output_dir, now)
  info_file_name = "%s/stub_info_files/Job%s.info" % (output_dir, now)

  print_file(logfile, "\nGCNO File: %s\nInfo File: %s\nLog File: %s\n" % (file_name, info_file_name, log_file_name))

  return (create_stub_info(file_name, info_file_name, log_file_name))

def merge_info_process(cmd):

#################################################################################
# Function which issues the merge command and create the single info file       # 
#################################################################################

  (status, output) = commands.getstatusoutput(cmd)
  if(status != 0):
    return False
  return True

def merge_stub_info(jobs):

#################################################################################
# Function which merges the created info files and create the single info file  #
#################################################################################

  info_dir = "%s/stub_info_files" % (output_dir)
  tmp_dir = "%s/tmp" % (output_dir)
  tmp1_dir = "%s/tmp1" % (output_dir)

  if(os.path.isdir(tmp_dir)):
    os.rmdir(tmp_dir)
  os.mkdir(tmp_dir, 0777)

  while (1):
    files = []
    files.extend(gcovUtil.getFileList(info_dir,extension='info',add_db=False))
    no_of_files = len(files)
    if (no_of_files == 1):
      info_file = tmp1_dir + "/" + os.listdir(tmp1_dir)[0]
      final_info_file = "%s/kernel.info" % (output_dir)
      shutil.copy2(info_file, final_info_file)
      os.remove(info_file)
      os.rmdir(tmp_dir)
      os.rmdir(tmp1_dir)
      return True
    no_of_iter = int(math.ceil(float(no_of_files)/float(jobs)))
    start = 0
    end = jobs
    cmds = []
    
    for i in range(no_of_iter):
      now = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")
      log_file = "%s/stub_info_logs/merge%s.log" % (output_dir, now)
      cmd = "lcov -a "
      cmd = cmd + string.join(files[start:end], " -a ")
      cmd = cmd + " -o %s/Job%s.info >& %s" % (tmp_dir, now, log_file)
      cmds.append(cmd)
      #print cmds
      start = start + jobs
      end = end + jobs

    merge_failed = 0
    with concurrent.futures.ProcessPoolExecutor(max_workers=jobs) as executor: 
      for merge_cmd, completed in zip(cmds, executor.map(merge_info_process, cmds)):
        if(completed):
          print_file(logfile, '\nMerge of the stub info files in following command completed successfully\nCmd: %s\n' % (merge_cmd))
        else:
          print_file(logfile, '\nERROR: Merge of the stub info files in following command failed\nCmd: %s\n' % (merge_cmd))
          merge_failed = merge_failed + 1
    if(merge_failed > 0):
      return False
    if(os.path.isdir(tmp1_dir)):
      shutil.rmtree(tmp1_dir)
    os.rename(tmp_dir, tmp1_dir)
    os.mkdir(tmp_dir, 0777)
    info_dir = tmp1_dir

def main():

  usage = """
  %prog -i <file_name> -o <output_dir> -b <build_root> -j <no_of_concurrent_jobs>

  This tool generates the stub info file, for the list of gcno files provided as the input
  """

  parser = OptionParser(usage)
  parser.add_option("-i", "--infile", dest='infile',  help='File which has the list of gcno files for which the stub info to be created')
  parser.add_option("-o", "--outputdir", dest='outputdir',  help='Output directory where stub info file will be created')
  parser.add_option("-b", "--buildroot", dest='buildroot',  help='Build root for which the stub info file will be created')
  parser.add_option("-j", "--jobs", dest='jobs',  help='Concurrent jobs for speed up. Default: 10')
  (options, args) = parser.parse_args()

  global output_dir, infile, buildroot, logfile

#################################################################################
# If the input file is not specified by user then print parser error and exit   #
#################################################################################

  if not options.infile:
    parser.error('File with list of source files is not specified')
  infile = options.infile

#################################################################################
# If the buildroot is not specified by user then print parser error and exit    #
#################################################################################

  if not options.buildroot:
    parser.error('Target buildroot is not specified')
  buildroot = options.buildroot
  buildroot = normalize_path("", buildroot)

#################################################################################
# If the output is not specified by user then the output directory is current   #
# working directory else that will be the value specified by user               #
#################################################################################

  if not options.outputdir:
    output_dir = os.getcwd()
  else:
    output_dir = options.outputdir

  if(not os.path.isabs(output_dir)):
    output_dir = "%s/%s" % (os.getcwd().rstrip("/"), output_dir.rstrip("/"))

#################################################################################
# If number of jobs is specified consider that value for generating info file   #
#################################################################################

  jobs = 10
  if options.jobs:
    jobs = int(options.jobs)

#################################################################################
# log file will be generated with the date-time stamp                           #
#################################################################################

  logprefix = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")
  logfile = "%s/log%s.log" % (output_dir, logprefix)

  if(not os.path.isdir(output_dir)):
    os.mkdir(output_dir, 0777)
  dir_name = "%s/stub_info_logs" % (output_dir)
  if(not os.path.isdir(dir_name)):
    os.mkdir(dir_name, 0777)
  dir_name = "%s/stub_info_files" % (output_dir)
  if(not os.path.isdir(dir_name)):
    os.mkdir(dir_name, 0777)

  print "Log File: %s " % logfile

  if(not os.path.isfile(infile)):
    parser.error('Input file %s does not exist' % infile)

  args = [line.strip() for line in open(infile)]
  
  file_err = 0
  file_list = []
  for arg in args:
    if(os.path.isdir(arg)) :

#################################################################################
# If the given input is a directory, then get the list of gcno files in that    #
# directory.                                                                    #
#################################################################################

      file_list.extend(gcovUtil.getFileList(arg,extension='gcno',add_db=False))
    elif(os.path.isfile(arg)):

#################################################################################
# If the given input is a file, then check if it is a gcno file. If file is gcno#
# file, then proceed else display warning in the log.                           #
#################################################################################

      if(arg.endswith(".gcno")):
        file_list.append(arg)
      else:
        print_file(logfile, "Warning: %s is not a valid gcno file\n" % arg)
        file_err = file_err + 1
    else:

#################################################################################
# If the given input is neither a valid file nor a valid directory then display #
# a warning in the log.                                                         #
#################################################################################

      print_file(logfile, "Warning: %s is a neither valid file nor a valid directory\n" % arg)
      file_err = file_err + 1

  failed_proc = 0

  #stub_info_process("/x/eng/nrlse43/DOT/nightly/mainN_130126_2059/adminapi/bedrock/private/x86_64.debug.gcov/ulibso/ZapiIf_aggr.gcno")
  #sys.exit(0)

#################################################################################
# With the list of source files create stub info file for each file in parallel #
# (max of 10 jobs at a time) using concurrent.futures. Print the execution      #
# status of the process in the log file                                         #
#################################################################################

  with concurrent.futures.ProcessPoolExecutor(max_workers=jobs) as executor:
    for file_data, completed in zip(file_list, executor.map(stub_info_process, file_list)):
      if(completed):
        print_file(logfile, 'Creating stub info file for the gcno file %s completed successfully\n' % (file_data))
      else:
        print_file(logfile, 'ERROR: Creating stub info file for the gcno file %s failed\n' % (file_data))
        failed_proc = failed_proc + 1

#################################################################################
# If there are any invalid input file/directories then display warning in the   #
# STDOUT.                                                                       #
#################################################################################

  if(file_err > 0):
    print "WARNING: Few Input Files/Directories are invalid. Please check the log"

#################################################################################
# If any of the process forked out to create stub info file in parallel fails   #
# then display error in the STDOUT.                                             #
#################################################################################

  if(failed_proc > 0):
    print "WARNING: Creating stub info file for few gcno files failed. Please check the log"
  else:
    print "Creating stub info file for the input gcno files completed successfully. Proceeding with merging the info files"

  merge_status = merge_stub_info(jobs)
  if(merge_status):
    print "Generated info file '%s/kernel.info'" % (output_dir)
    print "Finished .info-file creation"
  else:
    print "ERROR: Merging the stub info file failed. Please check the log"

if __name__ == '__main__':
  main()