""" A Cython plugin for coverage.py Requires the coverage package at least in version 4.0 (which added the plugin API). """ from __future__ import absolute_import import re import os.path import sys from collections import defaultdict from coverage.plugin import CoveragePlugin, FileTracer, FileReporter # requires coverage.py 4.0+ from coverage.files import canonical_filename from .Utils import find_root_package_dir, is_package_dir, open_source_file from . import __version__ def _find_c_source(base_path): if os.path.exists(base_path + '.c'): c_file = base_path + '.c' elif os.path.exists(base_path + '.cpp'): c_file = base_path + '.cpp' else: c_file = None return c_file def _find_dep_file_path(main_file, file_path): abs_path = os.path.abspath(file_path) if file_path.endswith('.pxi') and not os.path.exists(abs_path): # include files are looked up relative to the main source file pxi_file_path = os.path.join(os.path.dirname(main_file), file_path) if os.path.exists(pxi_file_path): abs_path = os.path.abspath(pxi_file_path) # search sys.path for external locations if a valid file hasn't been found if not os.path.exists(abs_path): for sys_path in sys.path: test_path = os.path.realpath(os.path.join(sys_path, file_path)) if os.path.exists(test_path): return canonical_filename(test_path) return canonical_filename(abs_path) class Plugin(CoveragePlugin): # map from traced file paths to absolute file paths _file_path_map = None # map from traced file paths to corresponding C files _c_files_map = None # map from parsed C files to their content _parsed_c_files = None def sys_info(self): return [('Cython version', __version__)] def file_tracer(self, filename): """ Try to find a C source file for a file path found by the tracer. """ if filename.startswith('<') or filename.startswith('memory:'): return None c_file = py_file = None filename = canonical_filename(os.path.abspath(filename)) if self._c_files_map and filename in self._c_files_map: c_file = self._c_files_map[filename][0] if c_file is None: c_file, py_file = self._find_source_files(filename) if not c_file: return None # parse all source file paths and lines from C file # to learn about all relevant source files right away (pyx/pxi/pxd) # FIXME: this might already be too late if the first executed line # is not from the main .pyx file but a file with a different # name than the .c file (which prevents us from finding the # .c file) self._parse_lines(c_file, filename) if self._file_path_map is None: self._file_path_map = {} return CythonModuleTracer(filename, py_file, c_file, self._c_files_map, self._file_path_map) def file_reporter(self, filename): # TODO: let coverage.py handle .py files itself #ext = os.path.splitext(filename)[1].lower() #if ext == '.py': # from coverage.python import PythonFileReporter # return PythonFileReporter(filename) filename = canonical_filename(os.path.abspath(filename)) if self._c_files_map and filename in self._c_files_map: c_file, rel_file_path, code = self._c_files_map[filename] else: c_file, _ = self._find_source_files(filename) if not c_file: return None # unknown file rel_file_path, code = self._parse_lines(c_file, filename) if code is None: return None # no source found return CythonModuleReporter(c_file, filename, rel_file_path, code) def _find_source_files(self, filename): basename, ext = os.path.splitext(filename) ext = ext.lower() if ext in ('.py', '.pyx', '.pxd', '.c', '.cpp'): pass elif ext == '.pyd': # Windows extension module platform_suffix = re.search(r'[.]cp[0-9]+-win[_a-z0-9]*$', basename, re.I) if platform_suffix: basename = basename[:platform_suffix.start()] elif ext == '.so': # Linux/Unix/Mac extension module platform_suffix = re.search(r'[.](?:cpython|pypy)-[0-9]+[-_a-z0-9]*$', basename, re.I) if platform_suffix: basename = basename[:platform_suffix.start()] elif ext == '.pxi': # if we get here, it means that the first traced line of a Cython module was # not in the main module but in an include file, so try a little harder to # find the main source file self._find_c_source_files(os.path.dirname(filename), filename) if filename in self._c_files_map: return self._c_files_map[filename][0], None else: # none of our business return None, None c_file = filename if ext in ('.c', '.cpp') else _find_c_source(basename) if c_file is None: # a module "pkg/mod.so" can have a source file "pkg/pkg.mod.c" package_root = find_root_package_dir.uncached(filename) package_path = os.path.relpath(basename, package_root).split(os.path.sep) if len(package_path) > 1: test_basepath = os.path.join(os.path.dirname(filename), '.'.join(package_path)) c_file = _find_c_source(test_basepath) py_source_file = None if c_file: py_source_file = os.path.splitext(c_file)[0] + '.py' if not os.path.exists(py_source_file): py_source_file = None try: with open(c_file, 'rb') as f: if b'/* Generated by Cython ' not in f.read(30): return None, None # not a Cython file except (IOError, OSError): c_file = None return c_file, py_source_file def _find_c_source_files(self, dir_path, source_file): """ Desperately parse all C files in the directory or its package parents (not re-descending) to find the (included) source file in one of them. """ if not os.path.isdir(dir_path): return splitext = os.path.splitext for filename in os.listdir(dir_path): ext = splitext(filename)[1].lower() if ext in ('.c', '.cpp'): self._parse_lines(os.path.join(dir_path, filename), source_file) if source_file in self._c_files_map: return # not found? then try one package up if is_package_dir(dir_path): self._find_c_source_files(os.path.dirname(dir_path), source_file) def _parse_lines(self, c_file, sourcefile): """ Parse a Cython generated C/C++ source file and find the executable lines. Each executable line starts with a comment header that states source file and line number, as well as the surrounding range of source code lines. """ if self._parsed_c_files is None: self._parsed_c_files = {} if c_file in self._parsed_c_files: code_lines = self._parsed_c_files[c_file] else: match_source_path_line = re.compile(r' */[*] +"(.*)":([0-9]+)$').match match_current_code_line = re.compile(r' *[*] (.*) # <<<<<<+$').match match_comment_end = re.compile(r' *[*]/$').match not_executable = re.compile( r'\s*c(?:type)?def\s+' r'(?:(?:public|external)\s+)?' r'(?:struct|union|enum|class)' r'(\s+[^:]+|)\s*:' ).match code_lines = defaultdict(dict) filenames = set() with open(c_file) as lines: lines = iter(lines) for line in lines: match = match_source_path_line(line) if not match: continue filename, lineno = match.groups() filenames.add(filename) lineno = int(lineno) for comment_line in lines: match = match_current_code_line(comment_line) if match: code_line = match.group(1).rstrip() if not_executable(code_line): break code_lines[filename][lineno] = code_line break elif match_comment_end(comment_line): # unexpected comment format - false positive? break self._parsed_c_files[c_file] = code_lines if self._c_files_map is None: self._c_files_map = {} for filename, code in code_lines.items(): abs_path = _find_dep_file_path(c_file, filename) self._c_files_map[abs_path] = (c_file, filename, code) if sourcefile not in self._c_files_map: return (None,) * 2 # e.g. shared library file return self._c_files_map[sourcefile][1:] class CythonModuleTracer(FileTracer): """ Find the Python/Cython source file for a Cython module. """ def __init__(self, module_file, py_file, c_file, c_files_map, file_path_map): super(CythonModuleTracer, self).__init__() self.module_file = module_file self.py_file = py_file self.c_file = c_file self._c_files_map = c_files_map self._file_path_map = file_path_map def has_dynamic_source_filename(self): return True def dynamic_source_filename(self, filename, frame): """ Determine source file path. Called by the function call tracer. """ source_file = frame.f_code.co_filename try: return self._file_path_map[source_file] except KeyError: pass abs_path = _find_dep_file_path(filename, source_file) if self.py_file and source_file[-3:].lower() == '.py': # always let coverage.py handle this case itself self._file_path_map[source_file] = self.py_file return self.py_file assert self._c_files_map is not None if abs_path not in self._c_files_map: self._c_files_map[abs_path] = (self.c_file, source_file, None) self._file_path_map[source_file] = abs_path return abs_path class CythonModuleReporter(FileReporter): """ Provide detailed trace information for one source file to coverage.py. """ def __init__(self, c_file, source_file, rel_file_path, code): super(CythonModuleReporter, self).__init__(source_file) self.name = rel_file_path self.c_file = c_file self._code = code def lines(self): """ Return set of line numbers that are possibly executable. """ return set(self._code) def _iter_source_tokens(self): current_line = 1 for line_no, code_line in sorted(self._code.items()): while line_no > current_line: yield [] current_line += 1 yield [('txt', code_line)] current_line += 1 def source(self): """ Return the source code of the file as a string. """ if os.path.exists(self.filename): with open_source_file(self.filename) as f: return f.read() else: return '\n'.join( (tokens[0][1] if tokens else '') for tokens in self._iter_source_tokens()) def source_token_lines(self): """ Iterate over the source code tokens. """ if os.path.exists(self.filename): with open_source_file(self.filename) as f: for line in f: yield [('txt', line.rstrip('\n'))] else: for line in self._iter_source_tokens(): yield [('txt', line)] def coverage_init(reg, options): reg.add_file_tracer(Plugin())