mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-09 19:35:51 +00:00
Bug 1402539 - Generate preprocessor information for lcov rewriting at build time. r=chmanchester
--HG-- extra : rebase_source : bad1d80838a930bd66aa88394065f9ea955ab237
This commit is contained in:
Marco Castelluccio
parent
78d2a3c1db
commit
a12e141983
@ -5,6 +5,7 @@
|
||||
from collections import defaultdict
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urlparse
|
||||
|
||||
from mach.config import ConfigSettings
|
||||
@ -58,6 +59,37 @@ class ChromeManifestHandler(object):
|
||||
for e in parse_manifest(None, entry.path):
|
||||
self.handle_manifest_entry(e)
|
||||
|
||||
_line_comment_re = re.compile('^//@line (\d+) "(.+)"$')
|
||||
def generate_pp_info(path):
|
||||
with open(path) as fh:
|
||||
# (start, end) -> (included_source, start)
|
||||
section_info = dict()
|
||||
|
||||
this_section = None
|
||||
|
||||
def finish_section(pp_end):
|
||||
pp_start, inc_source, inc_start = this_section
|
||||
section_info[str(pp_start) + ',' + str(pp_end)] = inc_source, inc_start
|
||||
|
||||
for count, line in enumerate(fh):
|
||||
# Regex are quite slow, so bail out early.
|
||||
if not line.startswith('//@line'):
|
||||
continue
|
||||
m = re.match(_line_comment_re, line)
|
||||
if m:
|
||||
if this_section:
|
||||
finish_section(count + 1)
|
||||
inc_start, inc_source = m.groups()
|
||||
pp_start = count + 2
|
||||
this_section = pp_start, inc_source, int(inc_start)
|
||||
|
||||
if this_section:
|
||||
finish_section(count + 2)
|
||||
|
||||
return section_info
|
||||
|
||||
# This build backend is assuming the build to have happened already, as it is parsing
|
||||
# built preprocessed files to generate data to map them to the original sources.
|
||||
class ChromeMapBackend(CommonBackend):
|
||||
def _init(self):
|
||||
CommonBackend._init(self)
|
||||
@ -82,18 +114,25 @@ class ChromeMapBackend(CommonBackend):
|
||||
for path, files in obj.files.walk():
|
||||
for f in files:
|
||||
dest = mozpath.join(obj.install_target, path, f.target_basename)
|
||||
is_pp = isinstance(obj,
|
||||
FinalTargetPreprocessedFiles)
|
||||
self._install_mapping[dest] = f.full_path, is_pp
|
||||
is_pp = isinstance(obj, FinalTargetPreprocessedFiles)
|
||||
obj_path = mozpath.join(self.environment.topobjdir, dest)
|
||||
if obj_path.endswith('.in'):
|
||||
obj_path = obj_path[:-3]
|
||||
if is_pp:
|
||||
assert os.path.exists(obj_path), '%s should exist' % obj_path
|
||||
pp_info = generate_pp_info(obj_path)
|
||||
else:
|
||||
pp_info = None
|
||||
self._install_mapping[dest] = f.full_path, pp_info
|
||||
|
||||
def consume_finished(self):
|
||||
# Our result has three parts:
|
||||
# A map from url prefixes to objdir directories:
|
||||
# { "chrome://mozapps/content/": [ "dist/bin/chrome/toolkit/content/mozapps" ], ... }
|
||||
# A map of overrides.
|
||||
# A map from objdir paths to sourcedir paths, and a flag for whether the source was preprocessed:
|
||||
# A map from objdir paths to sourcedir paths, and an object storing mapping information for preprocessed files:
|
||||
# { "dist/bin/browser/chrome/browser/content/browser/aboutSessionRestore.js":
|
||||
# [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", false ], ... }
|
||||
# [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", {} ], ... }
|
||||
outputfile = os.path.join(self.environment.topobjdir, 'chrome-map.json')
|
||||
with self._write_file(outputfile) as fh:
|
||||
chrome_mapping = self.manifest_handler.chrome_mapping
|
||||
|
||||
@ -5,10 +5,7 @@
|
||||
from argparse import ArgumentParser
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urlparse
|
||||
from collections import defaultdict
|
||||
|
||||
from mozpack.copier import FileRegistry
|
||||
from mozpack.files import PreprocessedFile
|
||||
@ -18,7 +15,6 @@ import mozpack.path as mozpath
|
||||
from chrome_map import ChromeManifestHandler
|
||||
|
||||
import buildconfig
|
||||
import mozpack.path as mozpath
|
||||
|
||||
class LcovRecord(object):
|
||||
__slots__ = ("test_name",
|
||||
@ -79,39 +75,7 @@ class RecordRewriter(object):
|
||||
# Helper class for rewriting/spliting individual lcov records according
|
||||
# to what the preprocessor did.
|
||||
def __init__(self):
|
||||
self.pp_info = {}
|
||||
self._ranges = None
|
||||
self._line_comment_re = re.compile('^//@line (\d+) "(.+)"$')
|
||||
|
||||
def has_pp_info(self, src_path):
|
||||
return src_path in self.pp_info
|
||||
|
||||
def populate_pp_info(self, fh, src_path):
|
||||
# (start, end) -> (included_source, start)
|
||||
section_info = dict()
|
||||
|
||||
this_section = None
|
||||
|
||||
def finish_section(pp_end):
|
||||
pp_start, inc_source, inc_start = this_section
|
||||
section_info[(pp_start, pp_end)] = inc_source, inc_start
|
||||
|
||||
for count, line in enumerate(fh):
|
||||
# Regex are quite slow, so bail out early.
|
||||
if not line.startswith('//@line'):
|
||||
continue
|
||||
m = re.match(self._line_comment_re, line)
|
||||
if m:
|
||||
if this_section:
|
||||
finish_section(count + 1)
|
||||
inc_start, inc_source = m.groups()
|
||||
pp_start = count + 2
|
||||
this_section = pp_start, inc_source, int(inc_start)
|
||||
|
||||
if this_section:
|
||||
finish_section(count + 2)
|
||||
|
||||
self.pp_info[src_path] = section_info
|
||||
|
||||
def _get_range(self, line):
|
||||
for start, end in self._ranges:
|
||||
@ -196,10 +160,10 @@ class RecordRewriter(object):
|
||||
|
||||
record.branches = rewritten_branches
|
||||
|
||||
def rewrite_record(self, record):
|
||||
def rewrite_record(self, record, pp_info):
|
||||
# Rewrite the lines in the given record according to preprocessor info
|
||||
# and split to additional records when pp_info has included file info.
|
||||
self._current_pp_info = self.pp_info[record.source_file]
|
||||
self._current_pp_info = dict([(tuple([int(l) for l in k.split(',')]), v) for k, v in pp_info.items()])
|
||||
self._ranges = sorted(self._current_pp_info.keys())
|
||||
self._additions = {}
|
||||
self._rewrite_lines(record)
|
||||
@ -250,7 +214,7 @@ class LcovFile(object):
|
||||
|
||||
def iterate_records(self, rewrite_source=None):
|
||||
current_source_file = None
|
||||
current_preprocessed = False
|
||||
current_pp_info = None
|
||||
current_lines = []
|
||||
for lcov_path in self.lcov_paths:
|
||||
with open(lcov_path) as lcov_fh:
|
||||
@ -262,10 +226,10 @@ class LcovFile(object):
|
||||
if line == 'end_of_record':
|
||||
# We skip records that we couldn't rewrite, that is records for which
|
||||
# rewrite_url returns None.
|
||||
if current_source_file != None:
|
||||
yield (current_source_file, current_preprocessed, current_lines)
|
||||
if current_source_file is not None:
|
||||
yield (current_source_file, current_pp_info, current_lines)
|
||||
current_source_file = None
|
||||
current_preprocessed = False
|
||||
current_pp_info = None
|
||||
current_lines = []
|
||||
continue
|
||||
|
||||
@ -274,11 +238,11 @@ class LcovFile(object):
|
||||
|
||||
if prefix == 'SF':
|
||||
sf = line[(colon + 1):]
|
||||
res = rewrite_source(sf) if rewrite_source is not None else (sf, False)
|
||||
res = rewrite_source(sf) if rewrite_source is not None else (sf, None)
|
||||
if res is None:
|
||||
current_lines.append(line)
|
||||
else:
|
||||
current_source_file, current_preprocessed = res
|
||||
current_source_file, current_pp_info = res
|
||||
current_lines.append('SF:' + current_source_file)
|
||||
else:
|
||||
current_lines.append(line)
|
||||
@ -325,10 +289,10 @@ class LcovFile(object):
|
||||
return ret
|
||||
|
||||
def print_file(self, fh, rewrite_source, rewrite_record):
|
||||
for source_file, preprocessed, record_content in self.iterate_records(rewrite_source):
|
||||
if preprocessed:
|
||||
for source_file, pp_info, record_content in self.iterate_records(rewrite_source):
|
||||
if pp_info is not None:
|
||||
record = self.parse_record(record_content)
|
||||
for r in rewrite_record(record):
|
||||
for r in rewrite_record(record, pp_info):
|
||||
fh.write(self.format_record(r))
|
||||
fh.write(self.format_record(record))
|
||||
else:
|
||||
@ -433,7 +397,7 @@ class LcovFile(object):
|
||||
self.current_record.covered_line_count = covered_line_count
|
||||
|
||||
def parse_LF(self, line_count):
|
||||
self.current_record.line_count = line_count
|
||||
self.current_record.line_count = line_count
|
||||
|
||||
|
||||
class UrlFinderError(Exception):
|
||||
@ -557,15 +521,15 @@ class UrlFinder(object):
|
||||
return res
|
||||
|
||||
def find_files(self, url):
|
||||
# Returns a tuple of (source file, objdir file, preprocessed)
|
||||
# Returns a tuple of (source file, pp_info)
|
||||
# for the given "resource:", "chrome:", or "file:" uri.
|
||||
term = url
|
||||
if term in self._url_overrides:
|
||||
term = self._url_overrides[term]
|
||||
|
||||
if os.path.isabs(term) and term.startswith(self.topobjdir):
|
||||
source_path, preprocessed = self._abs_objdir_install_info(term)
|
||||
return source_path, term, preprocessed
|
||||
source_path, pp_info = self._abs_objdir_install_info(term)
|
||||
return source_path, pp_info
|
||||
|
||||
objdir_path = None
|
||||
for prefix, dests in self._url_prefixes.iteritems():
|
||||
@ -593,11 +557,11 @@ class UrlFinder(object):
|
||||
objdir_path = objdir_path[1:]
|
||||
|
||||
if os.path.isabs(objdir_path) and objdir_path.startswith(self.topobjdir):
|
||||
source_path, preprocessed = self._abs_objdir_install_info(objdir_path)
|
||||
return source_path, term, preprocessed
|
||||
source_path, pp_info = self._abs_objdir_install_info(objdir_path)
|
||||
return source_path, pp_info
|
||||
|
||||
src_path, preprocessed = self._install_info(objdir_path)
|
||||
return mozpath.normpath(src_path), objdir_path, preprocessed
|
||||
src_path, pp_info = self._install_info(objdir_path)
|
||||
return mozpath.normpath(src_path), pp_info
|
||||
|
||||
def rewrite_url(self, url):
|
||||
# This applies one-off rules and returns None for urls that we aren't
|
||||
@ -610,7 +574,7 @@ class UrlFinder(object):
|
||||
if url.endswith('> Function'):
|
||||
return None
|
||||
if ' -> ' in url:
|
||||
url = url.split(' -> ')[1].rstrip()
|
||||
url = url.split(' -> ')[1].rstrip()
|
||||
|
||||
url_obj = urlparse.urlparse(url)
|
||||
if url_obj.scheme == 'jar':
|
||||
@ -627,7 +591,7 @@ class UrlFinder(object):
|
||||
else:
|
||||
# We don't know how to handle this jar: path, so return it to the
|
||||
# caller to make it print a warning.
|
||||
return url_obj.path, None, False
|
||||
return url_obj.path, None
|
||||
|
||||
dir_parts = parts[0].rsplit(app_name + '/', 1)
|
||||
url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', 'bin', dir_parts[1].lstrip('/'), parts[1].lstrip('/')))
|
||||
@ -649,7 +613,7 @@ class UrlFinder(object):
|
||||
# longer exists.
|
||||
return None
|
||||
if not path.startswith(self.topobjdir):
|
||||
return path, None, False
|
||||
return path, None
|
||||
url = url_obj.path
|
||||
elif url_obj.scheme in ('http', 'https', 'javascript', 'data', 'about'):
|
||||
return None
|
||||
@ -662,13 +626,12 @@ class LcovFileRewriter(object):
|
||||
# Class for partial parses of LCOV format and rewriting to resolve urls
|
||||
# and preprocessed file lines.
|
||||
def __init__(self, appdir, gredir, extra_chrome_manifests):
|
||||
self.topobjdir = buildconfig.topobjdir
|
||||
self.url_finder = UrlFinder(appdir, gredir, extra_chrome_manifests)
|
||||
self.pp_rewriter = RecordRewriter()
|
||||
|
||||
def rewrite_files(self, in_paths, output_file, output_suffix):
|
||||
unknowns = set()
|
||||
found_valid = False
|
||||
found_valid = [False]
|
||||
|
||||
def rewrite_source(url):
|
||||
try:
|
||||
@ -682,16 +645,12 @@ class LcovFileRewriter(object):
|
||||
unknowns.add(url)
|
||||
return None
|
||||
|
||||
source_file, objdir_file, preprocessed = res
|
||||
source_file, pp_info = res
|
||||
assert os.path.isfile(source_file), "Couldn't find mapped source file %s at %s!" % (url, source_file)
|
||||
if preprocessed and not self.pp_rewriter.has_pp_info(source_file):
|
||||
obj_path = os.path.join(self.topobjdir, objdir_file)
|
||||
with open(obj_path) as fh:
|
||||
self.pp_rewriter.populate_pp_info(fh, source_file)
|
||||
|
||||
found_valid = True
|
||||
found_valid[0] = True
|
||||
|
||||
return source_file, preprocessed
|
||||
return source_file, pp_info
|
||||
|
||||
in_paths = [os.path.abspath(in_path) for in_path in in_paths]
|
||||
|
||||
@ -705,7 +664,7 @@ class LcovFileRewriter(object):
|
||||
with open(in_path + output_suffix, 'w+') as out_fh:
|
||||
lcov_file.print_file(out_fh, rewrite_source, self.pp_rewriter.rewrite_record)
|
||||
|
||||
if not found_valid:
|
||||
if not found_valid[0]:
|
||||
print("WARNING: No valid records found in %s" % in_path)
|
||||
return
|
||||
|
||||
|
||||
@ -10,6 +10,7 @@ from StringIO import StringIO
|
||||
import json
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
from mozbuild.codecoverage import chrome_map
|
||||
from mozbuild.codecoverage import lcov_rewriter
|
||||
import buildconfig
|
||||
import mozunit
|
||||
@ -119,7 +120,7 @@ class TestLcovParser(unittest.TestCase):
|
||||
with TempFile(lcov_string) as fname:
|
||||
file_obj = lcov_rewriter.LcovFile([fname])
|
||||
out = StringIO()
|
||||
file_obj.print_file(out, lambda s: (s, False), lambda x: x)
|
||||
file_obj.print_file(out, lambda s: (s, None), lambda x, pp: x)
|
||||
|
||||
return out.getvalue()
|
||||
|
||||
@ -174,44 +175,41 @@ class TestLineRemapping(unittest.TestCase):
|
||||
shutil.move(self._old_chrome_info_file, self._chrome_map_file)
|
||||
|
||||
def test_map_multiple_included(self):
|
||||
self.pp_rewriter.populate_pp_info(StringIO(multiple_included_files),
|
||||
'')
|
||||
actual = self.pp_rewriter.pp_info['']
|
||||
with TempFile(multiple_included_files) as fname:
|
||||
actual = chrome_map.generate_pp_info(fname)
|
||||
expected = {
|
||||
(2, 3): ('foo.js', 1),
|
||||
(4, 5): ('bar.js', 2),
|
||||
(6, 7): ('foo.js', 3),
|
||||
(8, 9): ('bar.js', 2),
|
||||
(10, 11): ('test.js', 3),
|
||||
(12, 13): ('baz.js', 1),
|
||||
(14, 15): ('f.js', 6),
|
||||
"2,3": ('foo.js', 1),
|
||||
"4,5": ('bar.js', 2),
|
||||
"6,7": ('foo.js', 3),
|
||||
"8,9": ('bar.js', 2),
|
||||
"10,11": ('test.js', 3),
|
||||
"12,13": ('baz.js', 1),
|
||||
"14,15": ('f.js', 6),
|
||||
}
|
||||
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_remap_lcov(self):
|
||||
pp_remap = {
|
||||
(1941, 2158): ('dropPreview.js', 6),
|
||||
(2159, 2331): ('updater.js', 6),
|
||||
(2584, 2674): ('intro.js', 6),
|
||||
(2332, 2443): ('undo.js', 6),
|
||||
(864, 985): ('cells.js', 6),
|
||||
(2444, 2454): ('search.js', 6),
|
||||
(1567, 1712): ('drop.js', 6),
|
||||
(2455, 2583): ('customize.js', 6),
|
||||
(1713, 1940): ('dropTargetShim.js', 6),
|
||||
(1402, 1548): ('drag.js', 6),
|
||||
(1549, 1566): ('dragDataHelper.js', 6),
|
||||
(453, 602): ('page.js', 141),
|
||||
(2675, 2678): ('newTab.js', 70),
|
||||
(56, 321): ('transformations.js', 6),
|
||||
(603, 863): ('grid.js', 6),
|
||||
(322, 452): ('page.js', 6),
|
||||
(986, 1401): ('sites.js', 6)
|
||||
"1941,2158": ('dropPreview.js', 6),
|
||||
"2159,2331": ('updater.js', 6),
|
||||
"2584,2674": ('intro.js', 6),
|
||||
"2332,2443": ('undo.js', 6),
|
||||
"864,985": ('cells.js', 6),
|
||||
"2444,2454": ('search.js', 6),
|
||||
"1567,1712": ('drop.js', 6),
|
||||
"2455,2583": ('customize.js', 6),
|
||||
"1713,1940": ('dropTargetShim.js', 6),
|
||||
"1402,1548": ('drag.js', 6),
|
||||
"1549,1566": ('dragDataHelper.js', 6),
|
||||
"453,602": ('page.js', 141),
|
||||
"2675,2678": ('newTab.js', 70),
|
||||
"56,321": ('transformations.js', 6),
|
||||
"603,863": ('grid.js', 6),
|
||||
"322,452": ('page.js', 6),
|
||||
"986,1401": ('sites.js', 6)
|
||||
}
|
||||
|
||||
self.pp_rewriter.pp_info['lcov_test_newTab.js'] = pp_remap
|
||||
|
||||
fpath = os.path.join(here, 'sample_lcov.info')
|
||||
|
||||
# Read original records
|
||||
@ -234,7 +232,7 @@ class TestLineRemapping(unittest.TestCase):
|
||||
r_num = []
|
||||
def rewrite_source(s):
|
||||
r_num.append(1)
|
||||
return s, self.pp_rewriter.has_pp_info(s)
|
||||
return s, pp_remap
|
||||
|
||||
out = StringIO()
|
||||
lcov_file.print_file(out, rewrite_source, self.pp_rewriter.rewrite_record)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user