diff --git a/python/mozbuild/mozbuild/action/buildlist.py b/python/mozbuild/mozbuild/action/buildlist.py index 9d601d69a90f..d5546b5ae006 100644 --- a/python/mozbuild/mozbuild/action/buildlist.py +++ b/python/mozbuild/mozbuild/action/buildlist.py @@ -17,32 +17,33 @@ from mozbuild.util import ( lock_file, ) + def addEntriesToListFile(listFile, entries): - """Given a file |listFile| containing one entry per line, - add each entry in |entries| to the file, unless it is already - present.""" - ensureParentDir(listFile) - lock = lock_file(listFile + ".lck") - try: - if os.path.exists(listFile): - f = open(listFile) - existing = set(x.strip() for x in f.readlines()) - f.close() - else: - existing = set() - for e in entries: - if e not in existing: - existing.add(e) - with open(listFile, 'wb') as f: - f.write("\n".join(sorted(existing))+"\n") - finally: - lock = None + """Given a file |listFile| containing one entry per line, + add each entry in |entries| to the file, unless it is already + present.""" + ensureParentDir(listFile) + lock = lock_file(listFile + ".lck") + try: + if os.path.exists(listFile): + f = open(listFile) + existing = set(x.strip() for x in f.readlines()) + f.close() + else: + existing = set() + for e in entries: + if e not in existing: + existing.add(e) + with open(listFile, 'wb') as f: + f.write("\n".join(sorted(existing))+"\n") + finally: + lock = None def main(args): if len(args) < 2: print("Usage: buildlist.py [ ...]", - file=sys.stderr) + file=sys.stderr) return 1 return addEntriesToListFile(args[0], args[1:]) diff --git a/python/mozbuild/mozbuild/action/check_binary.py b/python/mozbuild/mozbuild/action/check_binary.py index b00e36d742ce..f04ac6cc7335 100644 --- a/python/mozbuild/mozbuild/action/check_binary.py +++ b/python/mozbuild/mozbuild/action/check_binary.py @@ -280,13 +280,14 @@ def check_networking(binary): s = 'TEST-UNEXPECTED-FAIL | check_networking | {} | Identified {} ' + \ 'networking function(s) being imported in the rust static library ({})' print(s.format(basename, len(bad_occurences_names), - ",".join(sorted(bad_occurences_names))), - file=sys.stderr) + ",".join(sorted(bad_occurences_names))), + file=sys.stderr) retcode = 1 elif buildconfig.substs.get('MOZ_AUTOMATION'): print('TEST-PASS | check_networking | {}'.format(basename)) return retcode + def checks(target, binary): # The clang-plugin is built as target but is really a host binary. # Cheat and pretend we were passed the right argument. @@ -345,7 +346,7 @@ def main(args): if options.networking and options.host: print('--networking is only valid with --target', - file=sys.stderr) + file=sys.stderr) return 1 if options.networking: diff --git a/python/mozbuild/mozbuild/action/dumpsymbols.py b/python/mozbuild/mozbuild/action/dumpsymbols.py index ef5f179331e4..c2a44898aed7 100644 --- a/python/mozbuild/mozbuild/action/dumpsymbols.py +++ b/python/mozbuild/mozbuild/action/dumpsymbols.py @@ -11,6 +11,7 @@ import shutil import sys import os + def dump_symbols(target, tracking_file, count_ctors=False): # Our tracking file, if present, will contain path(s) to the previously generated # symbols. Remove them in this case so we don't simply accumulate old symbols @@ -59,7 +60,7 @@ def dump_symbols(target, tracking_file, count_ctors=False): os.environ['OBJCOPY'] = objcopy args = ([buildconfig.substs['PYTHON'], os.path.join(buildconfig.topsrcdir, 'toolkit', - 'crashreporter', 'tools', 'symbolstore.py')] + + 'crashreporter', 'tools', 'symbolstore.py')] + sym_store_args + ['-s', buildconfig.topsrcdir, dump_syms_bin, os.path.join(buildconfig.topobjdir, 'dist', @@ -73,6 +74,7 @@ def dump_symbols(target, tracking_file, count_ctors=False): fh.write(out_files) fh.flush() + def main(argv): parser = argparse.ArgumentParser( usage="Usage: dumpsymbols.py ") diff --git a/python/mozbuild/mozbuild/action/exe_7z_archive.py b/python/mozbuild/mozbuild/action/exe_7z_archive.py index 361a4ecfde35..cb2c1058e887 100644 --- a/python/mozbuild/mozbuild/action/exe_7z_archive.py +++ b/python/mozbuild/mozbuild/action/exe_7z_archive.py @@ -13,6 +13,7 @@ import mozpack.path as mozpath import buildconfig from mozbuild.base import BuildEnvironmentNotFoundException + def archive_exe(pkg_dir, tagfile, sfx_package, package, use_upx): tmpdir = tempfile.mkdtemp(prefix='tmp') try: @@ -30,7 +31,8 @@ def archive_exe(pkg_dir, tagfile, sfx_package, package, use_upx): except BuildEnvironmentNotFoundException: # configure hasn't been run, just use the default sevenz = '7z' - subprocess.check_call([sevenz, 'a', '-r', '-t7z', mozpath.join(tmpdir, 'app.7z'), '-mx', '-m0=BCJ2', '-m1=LZMA:d25', '-m2=LZMA:d19', '-m3=LZMA:d19', '-mb0:1', '-mb0s1:2', '-mb0s2:3']) + subprocess.check_call([sevenz, 'a', '-r', '-t7z', mozpath.join(tmpdir, 'app.7z'), '-mx', + '-m0=BCJ2', '-m1=LZMA:d25', '-m2=LZMA:d19', '-m3=LZMA:d19', '-mb0:1', '-mb0s1:2', '-mb0s2:3']) with open(package, 'wb') as o: for i in [final_sfx, tagfile, mozpath.join(tmpdir, 'app.7z')]: @@ -41,6 +43,7 @@ def archive_exe(pkg_dir, tagfile, sfx_package, package, use_upx): shutil.move('core', pkg_dir) shutil.rmtree(tmpdir) + def main(args): if len(args) != 4: print('Usage: exe_7z_archive.py ', @@ -50,5 +53,6 @@ def main(args): archive_exe(args[0], args[1], args[2], args[3], args[4]) return 0 + if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/exe_7z_extract.py b/python/mozbuild/mozbuild/action/exe_7z_extract.py index db9a8fd59360..5c340fdfb5ab 100644 --- a/python/mozbuild/mozbuild/action/exe_7z_extract.py +++ b/python/mozbuild/mozbuild/action/exe_7z_extract.py @@ -8,10 +8,12 @@ import shutil import sys import subprocess + def extract_exe(package, target): subprocess.check_call(['7z', 'x', package, 'core']) shutil.move('core', target) + def main(args): if len(args) != 2: print('Usage: exe_7z_extract.py ', @@ -21,5 +23,6 @@ def main(args): extract_exe(args[0], args[1]) return 0 + if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/file_generate.py b/python/mozbuild/mozbuild/action/file_generate.py index bb52c1ca5d17..a4a1bae1ef7e 100644 --- a/python/mozbuild/mozbuild/action/file_generate.py +++ b/python/mozbuild/mozbuild/action/file_generate.py @@ -116,5 +116,6 @@ def main(argv): return 1 return ret + if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/generate_searchjson.py b/python/mozbuild/mozbuild/action/generate_searchjson.py index 8dd85552cf7e..fd048505bfa2 100644 --- a/python/mozbuild/mozbuild/action/generate_searchjson.py +++ b/python/mozbuild/mozbuild/action/generate_searchjson.py @@ -14,27 +14,29 @@ output_file = sys.argv[3] output = open(output_file, 'w') with open(sys.argv[1]) as f: - searchinfo = json.load(f) + searchinfo = json.load(f) # If we have a locale, use it, otherwise use the default if locale in searchinfo["locales"]: - localeSearchInfo = searchinfo["locales"][locale] + localeSearchInfo = searchinfo["locales"][locale] else: - localeSearchInfo = {} - localeSearchInfo["default"] = searchinfo["default"] + localeSearchInfo = {} + localeSearchInfo["default"] = searchinfo["default"] + def validateDefault(key): - if (not key in searchinfo["default"]): - print >>sys.stderr, "Error: Missing default %s in list.json" % (key) - sys.exit(1) + if (not key in searchinfo["default"]): + print >>sys.stderr, "Error: Missing default %s in list.json" % (key) + sys.exit(1) -validateDefault("searchDefault"); -validateDefault("visibleDefaultEngines"); + +validateDefault("searchDefault") +validateDefault("visibleDefaultEngines") # If the selected locale doesn't have a searchDefault, # use the global one. if not "searchDefault" in localeSearchInfo["default"]: - localeSearchInfo["default"]["searchDefault"] = searchinfo["default"]["searchDefault"] + localeSearchInfo["default"]["searchDefault"] = searchinfo["default"]["searchDefault"] # If the selected locale doesn't have a searchOrder, # use the global one if present. @@ -45,23 +47,24 @@ if not "searchOrder" in localeSearchInfo["default"] and "searchOrder" in searchi # If we have region overrides, enumerate through them # and add the additional regions to the locale information. if "regionOverrides" in searchinfo: - regionOverrides = searchinfo["regionOverrides"] + regionOverrides = searchinfo["regionOverrides"] - for region in regionOverrides: - # Only add a new engine list if there is an engine that is overridden - enginesToOverride = set(regionOverrides[region].keys()) - if region in localeSearchInfo and "visibleDefaultEngines" in localeSearchInfo[region]: - visibleDefaultEngines = localeSearchInfo[region]["visibleDefaultEngines"] - else: - visibleDefaultEngines = localeSearchInfo["default"]["visibleDefaultEngines"] - if set(visibleDefaultEngines) & enginesToOverride: - if region not in localeSearchInfo: - localeSearchInfo[region] = {} - localeSearchInfo[region]["visibleDefaultEngines"] = copy.deepcopy(visibleDefaultEngines) - for i, engine in enumerate(localeSearchInfo[region]["visibleDefaultEngines"]): - if engine in regionOverrides[region]: - localeSearchInfo[region]["visibleDefaultEngines"][i] = regionOverrides[region][engine] + for region in regionOverrides: + # Only add a new engine list if there is an engine that is overridden + enginesToOverride = set(regionOverrides[region].keys()) + if region in localeSearchInfo and "visibleDefaultEngines" in localeSearchInfo[region]: + visibleDefaultEngines = localeSearchInfo[region]["visibleDefaultEngines"] + else: + visibleDefaultEngines = localeSearchInfo["default"]["visibleDefaultEngines"] + if set(visibleDefaultEngines) & enginesToOverride: + if region not in localeSearchInfo: + localeSearchInfo[region] = {} + localeSearchInfo[region]["visibleDefaultEngines"] = copy.deepcopy( + visibleDefaultEngines) + for i, engine in enumerate(localeSearchInfo[region]["visibleDefaultEngines"]): + if engine in regionOverrides[region]: + localeSearchInfo[region]["visibleDefaultEngines"][i] = regionOverrides[region][engine] output.write(json.dumps(localeSearchInfo, ensure_ascii=False).encode('utf8')) -output.close(); +output.close() diff --git a/python/mozbuild/mozbuild/action/generate_suggestedsites.py b/python/mozbuild/mozbuild/action/generate_suggestedsites.py index 63300957805a..d1a949ee4451 100644 --- a/python/mozbuild/mozbuild/action/generate_suggestedsites.py +++ b/python/mozbuild/mozbuild/action/generate_suggestedsites.py @@ -102,7 +102,8 @@ def main(output, *args, **kwargs): def add_names(names, defaults={}): for name in names: site = copy.deepcopy(defaults) - site.update(properties.get_dict('browser.suggestedsites.{name}'.format(name=name), required_keys=('title', 'url', 'bgcolor'))) + site.update(properties.get_dict('browser.suggestedsites.{name}'.format( + name=name), required_keys=('title', 'url', 'bgcolor'))) site['imageurl'] = image_url_template.format(name=name) sites.append(site) @@ -116,7 +117,7 @@ def main(output, *args, **kwargs): matches = [p for p, _ in finder.find(drawables_template.format(name=name))] if not matches: raise Exception("Could not find drawable in '{resources}' for '{name}'" - .format(resources=resources, name=name)) + .format(resources=resources, name=name)) else: if opts.verbose: print("Found {len} drawables in '{resources}' for '{name}': {matches}" @@ -129,12 +130,14 @@ def main(output, *args, **kwargs): ('browser.suggestedsites.restricted.list', {'restricted': True}), ] if opts.verbose: - print('Reading {len} suggested site lists: {lists}'.format(len=len(lists), lists=[list_name for list_name, _ in lists])) + print('Reading {len} suggested site lists: {lists}'.format( + len=len(lists), lists=[list_name for list_name, _ in lists])) for (list_name, list_item_defaults) in lists: names = properties.get_list(list_name) if opts.verbose: - print('Reading {len} suggested sites from {list}: {names}'.format(len=len(names), list=list_name, names=names)) + print('Reading {len} suggested sites from {list}: {names}'.format( + len=len(names), list=list_name, names=names)) add_names(names, list_item_defaults) # We must define at least one site -- that's what the fallback is for. diff --git a/python/mozbuild/mozbuild/action/make_unzip.py b/python/mozbuild/mozbuild/action/make_unzip.py index 5d8ee36ca6d6..8bdae46daa24 100644 --- a/python/mozbuild/mozbuild/action/make_unzip.py +++ b/python/mozbuild/mozbuild/action/make_unzip.py @@ -7,9 +7,11 @@ from __future__ import print_function import sys import subprocess + def make_unzip(package): subprocess.check_call(['unzip', package]) + def main(args): if len(args) != 1: print('Usage: make_unzip.py ', @@ -19,5 +21,6 @@ def main(args): make_unzip(args[0]) return 0 + if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/make_zip.py b/python/mozbuild/mozbuild/action/make_zip.py index 13176afd5e9f..3063a1b86bfa 100644 --- a/python/mozbuild/mozbuild/action/make_zip.py +++ b/python/mozbuild/mozbuild/action/make_zip.py @@ -7,9 +7,11 @@ from __future__ import print_function import sys import subprocess + def make_zip(source, package): subprocess.check_call(['zip', '-r9D', package, source, '-x', '\*/.mkdir.done']) + def main(args): if len(args) != 2: print('Usage: make_zip.py ', @@ -19,5 +21,6 @@ def main(args): make_zip(args[0], args[1]) return 0 + if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/output_searchplugins_list.py b/python/mozbuild/mozbuild/action/output_searchplugins_list.py index a4767bbe40be..19846e1e2b14 100644 --- a/python/mozbuild/mozbuild/action/output_searchplugins_list.py +++ b/python/mozbuild/mozbuild/action/output_searchplugins_list.py @@ -10,24 +10,24 @@ engines = [] locale = sys.argv[2] with open(sys.argv[1]) as f: - searchinfo = json.load(f) + searchinfo = json.load(f) # Get a list of the engines from the locale or the default engines = set() if locale in searchinfo["locales"]: - for region, table in searchinfo["locales"][locale].iteritems(): - if "visibleDefaultEngines" in table: - engines.update(table["visibleDefaultEngines"]) + for region, table in searchinfo["locales"][locale].iteritems(): + if "visibleDefaultEngines" in table: + engines.update(table["visibleDefaultEngines"]) if not engines: - engines.update(searchinfo["default"]["visibleDefaultEngines"]) + engines.update(searchinfo["default"]["visibleDefaultEngines"]) # Get additional engines from regionOverrides for region, overrides in searchinfo["regionOverrides"].iteritems(): - for originalengine, replacement in overrides.iteritems(): - if originalengine in engines: - # We add the engine because we still need the original - engines.add(replacement) + for originalengine, replacement in overrides.iteritems(): + if originalengine in engines: + # We add the engine because we still need the original + engines.add(replacement) # join() will take an iterable, not just a list. print('\n'.join(engines)) diff --git a/python/mozbuild/mozbuild/action/package_fennec_apk.py b/python/mozbuild/mozbuild/action/package_fennec_apk.py index 93b2a59be77f..184bddf66786 100644 --- a/python/mozbuild/mozbuild/action/package_fennec_apk.py +++ b/python/mozbuild/mozbuild/action/package_fennec_apk.py @@ -70,7 +70,7 @@ def package_fennec_apk(inputs=[], omni_ja=None, if verbose: print('Packaging %s from %s' % (path, file.path)) if not os.path.exists(abspath): - raise ValueError('File %s not found (looked for %s)' % \ + raise ValueError('File %s not found (looked for %s)' % (file.path, abspath)) if jarrer.contains(path): jarrer.remove(path) diff --git a/python/mozbuild/mozbuild/action/package_generated_sources.py b/python/mozbuild/mozbuild/action/package_generated_sources.py index 1f4c35a65846..61e8a18a2ad7 100644 --- a/python/mozbuild/mozbuild/action/package_generated_sources.py +++ b/python/mozbuild/mozbuild/action/package_generated_sources.py @@ -23,6 +23,7 @@ def main(argv): args = parser.parse_args(argv) objdir_abspath = mozpath.abspath(buildconfig.topobjdir) + def is_valid_entry(entry): if isinstance(entry[1], BaseFile): entry_abspath = mozpath.abspath(entry[1].path) diff --git a/python/mozbuild/mozbuild/action/preprocessor.py b/python/mozbuild/mozbuild/action/preprocessor.py index 78f86ed6e82e..10820116b1a3 100644 --- a/python/mozbuild/mozbuild/action/preprocessor.py +++ b/python/mozbuild/mozbuild/action/preprocessor.py @@ -15,10 +15,11 @@ def generate(output, *args): pp.handleCommandLine(list(args), True) return set(pp.includes) + def main(args): pp = Preprocessor() pp.handleCommandLine(args, True) if __name__ == "__main__": - main(sys.argv[1:]) + main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/process_define_files.py b/python/mozbuild/mozbuild/action/process_define_files.py index f66bc68ad1a0..a388e1279555 100644 --- a/python/mozbuild/mozbuild/action/process_define_files.py +++ b/python/mozbuild/mozbuild/action/process_define_files.py @@ -55,6 +55,7 @@ def process_define_file(output, input): # via the command line, which raises a mass of macro # redefinition warnings. Just handle those macros # specially here. + def define_for_name(name, val): define = "#define {name} {val}".format(name=name, val=val) if name in ('WINVER', '_WIN32_WINNT'): diff --git a/python/mozbuild/mozbuild/action/process_install_manifest.py b/python/mozbuild/mozbuild/action/process_install_manifest.py index 3159d391b943..16254fe8e681 100644 --- a/python/mozbuild/mozbuild/action/process_install_manifest.py +++ b/python/mozbuild/mozbuild/action/process_install_manifest.py @@ -29,8 +29,8 @@ COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \ def process_manifest(destdir, paths, track, - no_symlinks=False, - defines={}): + no_symlinks=False, + defines={}): if os.path.exists(track): # We use the same format as install manifests for the tracking @@ -44,15 +44,15 @@ def process_manifest(destdir, paths, track, for p, f in finder.find(dest): remove_unaccounted.add(p, dummy_file) - remove_empty_directories=True - remove_all_directory_symlinks=True + remove_empty_directories = True + remove_all_directory_symlinks = True else: # If tracking is enabled and there is no file, we don't want to # be removing anything. remove_unaccounted = False - remove_empty_directories=False - remove_all_directory_symlinks=False + remove_empty_directories = False + remove_all_directory_symlinks = False manifest = InstallManifest() for path in paths: @@ -64,9 +64,9 @@ def process_manifest(destdir, paths, track, copier, defines_override=defines, link_policy=link_policy ) result = copier.copy(destdir, - remove_unaccounted=remove_unaccounted, - remove_all_directory_symlinks=remove_all_directory_symlinks, - remove_empty_directories=remove_empty_directories) + remove_unaccounted=remove_unaccounted, + remove_all_directory_symlinks=remove_all_directory_symlinks, + remove_empty_directories=remove_empty_directories) if track: # We should record files that we actually copied. @@ -83,21 +83,21 @@ def main(argv): parser.add_argument('destdir', help='Destination directory.') parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).') parser.add_argument('--no-symlinks', action='store_true', - help='Do not install symbolic links. Always copy files') + help='Do not install symbolic links. Always copy files') parser.add_argument('--track', metavar="PATH", required=True, - help='Use installed files tracking information from the given path.') + help='Use installed files tracking information from the given path.') parser.add_argument('-D', action=DefinesAction, - dest='defines', metavar="VAR[=VAL]", - help='Define a variable to override what is specified in the manifest') + dest='defines', metavar="VAR[=VAL]", + help='Define a variable to override what is specified in the manifest') args = parser.parse_args(argv) start = time.time() result = process_manifest(args.destdir, args.manifests, - track=args.track, - no_symlinks=args.no_symlinks, - defines=args.defines) + track=args.track, + no_symlinks=args.no_symlinks, + defines=args.defines) elapsed = time.time() - start @@ -109,5 +109,6 @@ def main(argv): rm_files=result.removed_files_count, rm_dirs=result.removed_directories_count)) + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/symbols_archive.py b/python/mozbuild/mozbuild/action/symbols_archive.py index 772ebf183d06..6555292f50c5 100644 --- a/python/mozbuild/mozbuild/action/symbols_archive.py +++ b/python/mozbuild/mozbuild/action/symbols_archive.py @@ -12,6 +12,7 @@ from mozpack.files import FileFinder from mozpack.mozjar import JarWriter import mozpack.path as mozpath + def make_archive(archive_name, base, exclude, include): compress = ['**/*.sym'] finder = FileFinder(base, ignore=exclude) @@ -27,11 +28,13 @@ def make_archive(archive_name, base, exclude, include): writer.add(p.encode('utf-8'), f, mode=f.mode, compress=should_compress, skip_duplicates=True) + def main(argv): parser = argparse.ArgumentParser(description='Produce a symbols archive') parser.add_argument('archive', help='Which archive to generate') parser.add_argument('base', help='Base directory to package') - parser.add_argument('--full-archive', action='store_true', help='Generate a full symbol archive') + parser.add_argument('--full-archive', action='store_true', + help='Generate a full symbol archive') args = parser.parse_args(argv) @@ -47,5 +50,6 @@ def main(argv): make_archive(args.archive, args.base, excludes, includes) + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/test_archive.py b/python/mozbuild/mozbuild/action/test_archive.py index dfc6e5118d1b..542e16398a64 100644 --- a/python/mozbuild/mozbuild/action/test_archive.py +++ b/python/mozbuild/mozbuild/action/test_archive.py @@ -642,7 +642,7 @@ for k, v in ARCHIVE_FILES.items(): continue ignores = set(itertools.chain(*(e.get('ignore', []) - for e in ARCHIVE_FILES['common']))) + for e in ARCHIVE_FILES['common']))) if not any(p.startswith('%s/' % k) for p in ignores): raise Exception('"common" ignore list probably should contain %s' % k) diff --git a/python/mozbuild/mozbuild/action/tooltool.py b/python/mozbuild/mozbuild/action/tooltool.py index 646b31a19019..efcfdc61e58d 100755 --- a/python/mozbuild/mozbuild/action/tooltool.py +++ b/python/mozbuild/mozbuild/action/tooltool.py @@ -1300,5 +1300,6 @@ def main(argv, _skip_logging=False): return 0 if process_command(options, args) else 1 + if __name__ == "__main__": # pragma: no cover sys.exit(main(sys.argv)) diff --git a/python/mozbuild/mozbuild/action/wrap_rustc.py b/python/mozbuild/mozbuild/action/wrap_rustc.py index e80a13661e1c..39893b174c25 100644 --- a/python/mozbuild/mozbuild/action/wrap_rustc.py +++ b/python/mozbuild/mozbuild/action/wrap_rustc.py @@ -9,6 +9,7 @@ import subprocess import sys import os + def parse_outputs(crate_output, dep_outputs, pass_l_flag): env = {} args = [] @@ -59,6 +60,7 @@ def parse_outputs(crate_output, dep_outputs, pass_l_flag): return env, args + def wrap_rustc(args): parser = argparse.ArgumentParser() parser.add_argument('--crate-out', nargs='?') @@ -73,5 +75,6 @@ def wrap_rustc(args): os.environ.update(new_env) return subprocess.Popen(args.cmd + new_args, cwd=args.cwd).wait() + if __name__ == '__main__': sys.exit(wrap_rustc(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/xpccheck.py b/python/mozbuild/mozbuild/action/xpccheck.py index c3170a8da8fd..37757eff820f 100644 --- a/python/mozbuild/mozbuild/action/xpccheck.py +++ b/python/mozbuild/mozbuild/action/xpccheck.py @@ -15,69 +15,76 @@ import os from glob import glob import manifestparser + def getIniTests(testdir): - mp = manifestparser.ManifestParser(strict=False) - mp.read(os.path.join(testdir, 'xpcshell.ini')) - return mp.tests + mp = manifestparser.ManifestParser(strict=False) + mp.read(os.path.join(testdir, 'xpcshell.ini')) + return mp.tests + def verifyDirectory(initests, directory): - files = glob(os.path.join(os.path.abspath(directory), "test_*")) - for f in files: - if (not os.path.isfile(f)): - continue + files = glob(os.path.join(os.path.abspath(directory), "test_*")) + for f in files: + if (not os.path.isfile(f)): + continue - name = os.path.basename(f) - if name.endswith('.in'): - name = name[:-3] + name = os.path.basename(f) + if name.endswith('.in'): + name = name[:-3] - if not name.endswith('.js'): - continue + if not name.endswith('.js'): + continue + + found = False + for test in initests: + if os.path.join(os.path.abspath(directory), name) == test['path']: + found = True + break + + if not found: + print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | test %s is missing from test manifest %s!" % ( + name, os.path.join(directory, 'xpcshell.ini')) + sys.exit(1) - found = False - for test in initests: - if os.path.join(os.path.abspath(directory), name) == test['path']: - found = True - break - - if not found: - print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | test %s is missing from test manifest %s!" % (name, os.path.join(directory, 'xpcshell.ini')) - sys.exit(1) def verifyIniFile(initests, directory): - files = glob(os.path.join(os.path.abspath(directory), "test_*")) - for test in initests: - name = test['path'].split('/')[-1] + files = glob(os.path.join(os.path.abspath(directory), "test_*")) + for test in initests: + name = test['path'].split('/')[-1] - found = False - for f in files: + found = False + for f in files: - fname = f.split('/')[-1] - if fname.endswith('.in'): - fname = '.in'.join(fname.split('.in')[:-1]) + fname = f.split('/')[-1] + if fname.endswith('.in'): + fname = '.in'.join(fname.split('.in')[:-1]) - if os.path.join(os.path.abspath(directory), fname) == test['path']: - found = True - break + if os.path.join(os.path.abspath(directory), fname) == test['path']: + found = True + break + + if not found: + print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | found %s in xpcshell.ini and not in directory '%s'" % ( + name, directory) + sys.exit(1) - if not found: - print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | found %s in xpcshell.ini and not in directory '%s'" % (name, directory) - sys.exit(1) def main(argv): - if len(argv) < 2: - print >>sys.stderr, "Usage: xpccheck.py [ ...]" - sys.exit(1) + if len(argv) < 2: + print >>sys.stderr, "Usage: xpccheck.py [ ...]" + sys.exit(1) - topsrcdir = argv[0] - for d in argv[1:]: - # xpcshell-unpack is a copy of xpcshell sibling directory and in the Makefile - # we copy all files (including xpcshell.ini from the sibling directory. - if d.endswith('toolkit/mozapps/extensions/test/xpcshell-unpack'): - continue + topsrcdir = argv[0] + for d in argv[1:]: + # xpcshell-unpack is a copy of xpcshell sibling directory and in the Makefile + # we copy all files (including xpcshell.ini from the sibling directory. + if d.endswith('toolkit/mozapps/extensions/test/xpcshell-unpack'): + continue + + initests = getIniTests(d) + verifyDirectory(initests, d) + verifyIniFile(initests, d) - initests = getIniTests(d) - verifyDirectory(initests, d) - verifyIniFile(initests, d) if __name__ == '__main__': main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/xpidl-process.py b/python/mozbuild/mozbuild/action/xpidl-process.py index ae9ba873eed4..8d4077a4b67c 100755 --- a/python/mozbuild/mozbuild/action/xpidl-process.py +++ b/python/mozbuild/mozbuild/action/xpidl-process.py @@ -87,32 +87,33 @@ def process(input_dirs, inc_paths, bindings_conf, cache_dir, header_dir, def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--cache-dir', - help='Directory in which to find or write cached lexer data.') + help='Directory in which to find or write cached lexer data.') parser.add_argument('--depsdir', - help='Directory in which to write dependency files.') + help='Directory in which to write dependency files.') parser.add_argument('--bindings-conf', - help='Path to the WebIDL binding configuration file.') + help='Path to the WebIDL binding configuration file.') parser.add_argument('--input-dir', dest='input_dirs', action='append', default=[], help='Directory(ies) in which to find source .idl files.') parser.add_argument('headerdir', - help='Directory in which to write header files.') + help='Directory in which to write header files.') parser.add_argument('xpcrsdir', - help='Directory in which to write rust xpcom binding files.') + help='Directory in which to write rust xpcom binding files.') parser.add_argument('xptdir', - help='Directory in which to write xpt file.') + help='Directory in which to write xpt file.') parser.add_argument('module', - help='Final module name to use for linked output xpt file.') + help='Final module name to use for linked output xpt file.') parser.add_argument('idls', nargs='+', - help='Source .idl file(s).') + help='Source .idl file(s).') parser.add_argument('-I', dest='incpath', action='append', default=[], - help='Extra directories where to look for included .idl files.') + help='Extra directories where to look for included .idl files.') args = parser.parse_args(argv) incpath = [os.path.join(topsrcdir, p) for p in args.incpath] process(args.input_dirs, incpath, args.bindings_conf, args.cache_dir, - args.headerdir, args.xpcrsdir, args.xptdir, args.depsdir, args.module, - args.idls) + args.headerdir, args.xpcrsdir, args.xptdir, args.depsdir, args.module, + args.idls) + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/analyze/graph.py b/python/mozbuild/mozbuild/analyze/graph.py index 5c5acee36888..c66a5170a6e6 100644 --- a/python/mozbuild/mozbuild/analyze/graph.py +++ b/python/mozbuild/mozbuild/analyze/graph.py @@ -5,6 +5,7 @@ import os import sqlite3 as lite + class Node(object): def __init__(self, graph, node_id): @@ -39,11 +40,12 @@ class Node(object): return os.path.join(parent.get_path(graph), self.name) def calculate_mtime(self): - if self.type == 0: # only files have meaningful costs + if self.type == 0: # only files have meaningful costs return sum(x.mtime for x in self.cmds) else: return None + class Graph(object): def __init__(self, path=None, connect=None): @@ -67,7 +69,7 @@ class Graph(object): self.connect.close() def query_arg(self, q, arg): - assert isinstance(arg, tuple) #execute() requires tuple argument + assert isinstance(arg, tuple) # execute() requires tuple argument cursor = self.connect.cursor() cursor.execute(q, arg) return cursor @@ -112,7 +114,7 @@ class Graph(object): m, s = sec / 60, sec % 60 print ("\n------ Summary for %s ------\ \nTotal Build Time (mm:ss) = %d:%d\nNum Downstream Commands = %d" - % (f, m, s, node.num_cmds)) + % (f, m, s, node.num_cmds)) def populate(self): # make nodes for files with downstream commands @@ -127,5 +129,4 @@ class Graph(object): def get_cost_dict(self): if self.results is None: self.populate() - return {k:v for k,v in self.results if v > 0} - + return {k: v for k, v in self.results if v > 0} diff --git a/python/mozbuild/mozbuild/analyze/hg.py b/python/mozbuild/mozbuild/analyze/hg.py index 35a626669a55..e211b4235936 100644 --- a/python/mozbuild/mozbuild/analyze/hg.py +++ b/python/mozbuild/mozbuild/analyze/hg.py @@ -17,31 +17,36 @@ PUSHLOG_CHUNK_SIZE = 500 URL = 'https://hg.mozilla.org/mozilla-central/json-pushes?' + def unix_epoch(date): - return (date - datetime(1970,1,1)).total_seconds() + return (date - datetime(1970, 1, 1)).total_seconds() + def unix_from_date(n, today): return unix_epoch(today - timedelta(days=n)) + def get_lastpid(session): return session.get(URL+'&version=2').json()['lastpushid'] + def get_pushlog_chunk(session, start, end): # returns pushes sorted by date res = session.get(URL+'version=1&startID={0}&\ endID={1}&full=1'.format(start, end)).json() - return sorted(res.items(), key = lambda x: x[1]['date']) + return sorted(res.items(), key=lambda x: x[1]['date']) + def collect_data(session, date): - if date < 1206031764: #first push - raise Exception ("No pushes exist before March 20, 2008.") + if date < 1206031764: # first push + raise Exception("No pushes exist before March 20, 2008.") lastpushid = get_lastpid(session) data = [] start_id = lastpushid - PUSHLOG_CHUNK_SIZE end_id = lastpushid + 1 while True: res = get_pushlog_chunk(session, start_id, end_id) - starting_date = res[0][1]['date'] # date of oldest push in chunk + starting_date = res[0][1]['date'] # date of oldest push in chunk dates = [x[1]['date'] for x in res] if starting_date < date: i = bisect.bisect_left(dates, date) @@ -52,10 +57,12 @@ def collect_data(session, date): end_id = start_id + 1 start_id = start_id - PUSHLOG_CHUNK_SIZE + def get_data(epoch): session = requests.Session() data = collect_data(session, epoch) - return {k:v for sublist in data for (k,v) in sublist} + return {k: v for sublist in data for (k, v) in sublist} + class Pushlog(object): @@ -78,6 +85,7 @@ class Pushlog(object): keys.sort() return keys + class Push(object): def __init__(self, pid, p_dict): @@ -85,6 +93,7 @@ class Push(object): self.date = p_dict['date'] self.files = [f for x in p_dict['changesets'] for f in x['files']] + class Report(object): def __init__(self, days, path=None, cost_dict=None): @@ -112,7 +121,7 @@ class Report(object): cost = costs.get(f) count = counts.get(f) if cost is not None: - res.append((f, cost, count, round(cost*count,3))) + res.append((f, cost, count, round(cost*count, 3))) return res def get_sorted_report(self, format): @@ -143,7 +152,8 @@ class Report(object): res = self.get_sorted_report(format) if limit is not None: res = self.cut(limit, res) - for x in res: data.append(x) + for x in res: + data.append(x) if format == 'pretty': print (data) else: @@ -160,4 +170,3 @@ class Report(object): with open(file_path, 'wb') as f: f.write(content) print ("Created report: %s" % file_path) - diff --git a/python/mozbuild/mozbuild/android_version_code.py b/python/mozbuild/mozbuild/android_version_code.py index ff9dc6c6b155..8eed2ef5d769 100644 --- a/python/mozbuild/mozbuild/android_version_code.py +++ b/python/mozbuild/mozbuild/android_version_code.py @@ -11,7 +11,8 @@ import time # Builds before this build ID use the v0 version scheme. Builds after this # build ID use the v1 version scheme. -V1_CUTOFF = 20150801000000 # YYYYmmddHHMMSS +V1_CUTOFF = 20150801000000 # YYYYmmddHHMMSS + def android_version_code_v0(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): base = int(str(buildid)[:10]) @@ -30,6 +31,7 @@ def android_version_code_v0(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): raise ValueError("Don't know how to compute android:versionCode " "for CPU arch %s" % cpu_arch) + def android_version_code_v1(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): '''Generate a v1 android:versionCode. @@ -134,6 +136,7 @@ def android_version_code_v1(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): return version + def android_version_code(buildid, *args, **kwargs): base = int(str(buildid)) if base < V1_CUTOFF: @@ -166,9 +169,9 @@ def main(argv): args = parser.parse_args(argv) code = android_version_code(args.buildid, - cpu_arch=args.cpu_arch, - min_sdk=args.min_sdk, - max_sdk=args.max_sdk) + cpu_arch=args.cpu_arch, + min_sdk=args.min_sdk, + max_sdk=args.max_sdk) print(code) return 0 diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py index 47e9451ec57f..c54aefe6bd69 100644 --- a/python/mozbuild/mozbuild/artifacts.py +++ b/python/mozbuild/mozbuild/artifacts.py @@ -77,7 +77,8 @@ from mozpack.mozjar import ( from mozpack.packager.unpack import UnpackFinder import mozpack.path as mozpath -NUM_PUSHHEADS_TO_QUERY_PER_PARENT = 50 # Number of candidate pushheads to cache per parent changeset. +# Number of candidate pushheads to cache per parent changeset. +NUM_PUSHHEADS_TO_QUERY_PER_PARENT = 50 # Number of parent changesets to consider as possible pushheads. # There isn't really such a thing as a reasonable default here, because we don't @@ -313,8 +314,8 @@ class AndroidArtifactJob(ArtifactJob): dirname, basename = os.path.split(p) self.log(logging.INFO, 'artifact', - {'basename': basename}, - 'Adding {basename} to processed archive') + {'basename': basename}, + 'Adding {basename} to processed archive') basedir = 'bin' if not basename.endswith('.so'): @@ -323,7 +324,8 @@ class AndroidArtifactJob(ArtifactJob): writer.add(basename.encode('utf-8'), f.open()) def process_symbols_archive(self, filename, processed_filename): - ArtifactJob.process_symbols_archive(self, filename, processed_filename, skip_compressed=True) + ArtifactJob.process_symbols_archive( + self, filename, processed_filename, skip_compressed=True) if self._symbols_archive_suffix != 'crashreporter-symbols-full.zip': return @@ -440,8 +442,8 @@ class MacArtifactJob(ArtifactJob): oldcwd = os.getcwd() try: self.log(logging.INFO, 'artifact', - {'tempdir': tempdir}, - 'Unpacking DMG into {tempdir}') + {'tempdir': tempdir}, + 'Unpacking DMG into {tempdir}') if self._substs['HOST_OS_ARCH'] == 'Linux': # This is a cross build, use hfsplus and dmg tools to extract the dmg. os.chdir(tempdir) @@ -486,8 +488,8 @@ class MacArtifactJob(ArtifactJob): for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', - {'path': p}, - 'Adding {path} to processed archive') + {'path': p}, + 'Adding {path} to processed archive') destpath = mozpath.join('bin', os.path.basename(p)) writer.add(destpath.encode('utf-8'), f, mode=f.mode) @@ -507,8 +509,8 @@ class MacArtifactJob(ArtifactJob): shutil.rmtree(tempdir) except (OSError, IOError): self.log(logging.WARN, 'artifact', - {'tempdir': tempdir}, - 'Unable to delete {tempdir}') + {'tempdir': tempdir}, + 'Unable to delete {tempdir}') pass @@ -561,8 +563,8 @@ class WinArtifactJob(ArtifactJob): basename = mozpath.relpath(p, self.product) basename = mozpath.join('bin', basename) self.log(logging.INFO, 'artifact', - {'basename': basename}, - 'Adding {basename} to processed archive') + {'basename': basename}, + 'Adding {basename} to processed archive') writer.add(basename.encode('utf-8'), f.open(), mode=f.mode) added_entry = True @@ -668,8 +670,8 @@ class CacheManager(object): def load_cache(self): if self._skip_cache: self.log(logging.INFO, 'artifact', - {}, - 'Skipping cache: ignoring load_cache!') + {}, + 'Skipping cache: ignoring load_cache!') return try: @@ -681,25 +683,26 @@ class CacheManager(object): # exceptions, so it's not worth trying to be fine grained here. # We ignore any exception, so the cache is effectively dropped. self.log(logging.INFO, 'artifact', - {'filename': self._cache_filename, 'exception': repr(e)}, - 'Ignoring exception unpickling cache file {filename}: {exception}') + {'filename': self._cache_filename, 'exception': repr(e)}, + 'Ignoring exception unpickling cache file {filename}: {exception}') pass def dump_cache(self): if self._skip_cache: self.log(logging.INFO, 'artifact', - {}, - 'Skipping cache: ignoring dump_cache!') + {}, + 'Skipping cache: ignoring dump_cache!') return ensureParentDir(self._cache_filename) - pickle.dump(list(reversed(list(self._cache.items()))), open(self._cache_filename, 'wb'), -1) + pickle.dump(list(reversed(list(self._cache.items()))), + open(self._cache_filename, 'wb'), -1) def clear_cache(self): if self._skip_cache: self.log(logging.INFO, 'artifact', - {}, - 'Skipping cache: ignoring clear_cache!') + {}, + 'Skipping cache: ignoring clear_cache!') return with self: @@ -712,11 +715,13 @@ class CacheManager(object): def __exit__(self, type, value, traceback): self.dump_cache() + class PushheadCache(CacheManager): '''Helps map tree/revision pairs to parent pushheads according to the pushlog.''' def __init__(self, cache_dir, log=None, skip_cache=False): - CacheManager.__init__(self, cache_dir, 'pushhead_cache', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) + CacheManager.__init__(self, cache_dir, 'pushhead_cache', + MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) @cachedmethod(operator.attrgetter('_cache')) def parent_pushhead_id(self, tree, revision): @@ -743,11 +748,13 @@ class PushheadCache(CacheManager): p['changesets'][-1] for p in result['pushes'].values() ] + class TaskCache(CacheManager): '''Map candidate pushheads to Task Cluster task IDs and artifact URLs.''' def __init__(self, cache_dir, log=None, skip_cache=False): - CacheManager.__init__(self, cache_dir, 'artifact_url', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) + CacheManager.__init__(self, cache_dir, 'artifact_url', + MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) @cachedmethod(operator.attrgetter('_cache')) def artifacts(self, tree, job, artifact_job_class, rev): @@ -782,7 +789,8 @@ class TaskCache(CacheManager): except KeyError: # Not all revisions correspond to pushes that produce the job we # care about; and even those that do may not have completed yet. - raise ValueError('Task for {namespace} does not exist (yet)!'.format(namespace=namespace)) + raise ValueError( + 'Task for {namespace} does not exist (yet)!'.format(namespace=namespace)) return taskId, list_artifacts(taskId) @@ -820,13 +828,15 @@ class Artifacts(object): substs=self._substs) except KeyError: self.log(logging.INFO, 'artifact', - {'job': self._job}, - 'Unknown job {job}') + {'job': self._job}, + 'Unknown job {job}') raise KeyError("Unknown job") self._task_cache = TaskCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache) - self._artifact_cache = ArtifactCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache) - self._pushhead_cache = PushheadCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache) + self._artifact_cache = ArtifactCache( + self._cache_dir, log=self._log, skip_cache=self._skip_cache) + self._pushhead_cache = PushheadCache( + self._cache_dir, log=self._log, skip_cache=self._skip_cache) def log(self, *args, **kwargs): if self._log: @@ -953,8 +963,8 @@ from remote. Please run `hg pull` and build again. see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial/Bundles""") self.log(logging.INFO, 'artifact', - {'len': len(last_revs)}, - 'hg suggested {len} candidate revisions') + {'len': len(last_revs)}, + 'hg suggested {len} candidate revisions') def to_pair(line): rev, node = line.split(':', 1) @@ -1002,7 +1012,8 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code def find_pushhead_artifacts(self, task_cache, job, tree, pushhead): try: - taskId, artifacts = task_cache.artifacts(tree, job, self._artifact_job.__class__, pushhead) + taskId, artifacts = task_cache.artifacts( + tree, job, self._artifact_job.__class__, pushhead) except ValueError: return None @@ -1023,32 +1034,32 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code def install_from_file(self, filename, distdir): self.log(logging.INFO, 'artifact', - {'filename': filename}, - 'Installing from {filename}') + {'filename': filename}, + 'Installing from {filename}') # Do we need to post-process? processed_filename = filename + PROCESSED_SUFFIX if self._skip_cache and os.path.exists(processed_filename): self.log(logging.INFO, 'artifact', - {'path': processed_filename}, - 'Skipping cache: removing cached processed artifact {path}') + {'path': processed_filename}, + 'Skipping cache: removing cached processed artifact {path}') os.remove(processed_filename) if not os.path.exists(processed_filename): self.log(logging.INFO, 'artifact', - {'filename': filename}, - 'Processing contents of {filename}') + {'filename': filename}, + 'Processing contents of {filename}') self.log(logging.INFO, 'artifact', - {'processed_filename': processed_filename}, - 'Writing processed {processed_filename}') + {'processed_filename': processed_filename}, + 'Writing processed {processed_filename}') self._artifact_job.process_artifact(filename, processed_filename) self._artifact_cache._persist_limit.register_file(processed_filename) self.log(logging.INFO, 'artifact', - {'processed_filename': processed_filename}, - 'Installing from processed {processed_filename}') + {'processed_filename': processed_filename}, + 'Installing from processed {processed_filename}') # Copy all .so files, avoiding modification where possible. ensureParentDir(mozpath.join(distdir, '.dummy')) @@ -1062,20 +1073,20 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code shutil.copyfileobj(zf.open(info), fh) file_existed, file_updated = fh.close() self.log(logging.INFO, 'artifact', - {'updating': 'Updating' if file_updated else 'Not updating', 'filename': n}, - '{updating} {filename}') + {'updating': 'Updating' if file_updated else 'Not updating', 'filename': n}, + '{updating} {filename}') if not file_existed or file_updated: # Libraries and binaries may need to be marked executable, # depending on platform. - perms = info.external_attr >> 16 # See http://stackoverflow.com/a/434689. - perms |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # u+w, a+r. + perms = info.external_attr >> 16 # See http://stackoverflow.com/a/434689. + perms |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # u+w, a+r. os.chmod(n, perms) return 0 def install_from_url(self, url, distdir): self.log(logging.INFO, 'artifact', - {'url': url}, - 'Installing from {url}') + {'url': url}, + 'Installing from {url}') filename = self._artifact_cache.fetch(url) return self.install_from_file(filename, distdir) @@ -1117,7 +1128,7 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code try: if self._hg: revision = subprocess.check_output([self._hg, 'log', '--template', '{node}\n', - '-r', revset], cwd=self._topsrcdir).strip() + '-r', revset], cwd=self._topsrcdir).strip() elif self._git: revset = subprocess.check_output([ self._git, 'rev-parse', '%s^{commit}' % revset], @@ -1165,7 +1176,8 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code url = get_artifact_url(taskId, artifact_name) urls.append(url) if not urls: - raise ValueError('Task {taskId} existed, but no artifacts found!'.format(taskId=taskId)) + raise ValueError( + 'Task {taskId} existed, but no artifacts found!'.format(taskId=taskId)) for url in urls: if self.install_from_url(url, distdir): return 1 @@ -1194,11 +1206,10 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code return self.install_from_recent(distdir) - def clear_cache(self): self.log(logging.INFO, 'artifact', - {}, - 'Deleting cached artifacts and caches.') + {}, + 'Deleting cached artifacts and caches.') self._task_cache.clear_cache() self._artifact_cache.clear_cache() self._pushhead_cache.clear_cache() diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py index dcfb6c1cee49..beaae930df84 100644 --- a/python/mozbuild/mozbuild/backend/common.py +++ b/python/mozbuild/mozbuild/backend/common.py @@ -57,6 +57,7 @@ from mozbuild.util import ( mkdir, ) + class XPIDLManager(object): """Helps manage XPCOM IDLs in the context of the build system.""" @@ -102,6 +103,7 @@ class XPIDLManager(object): """ return itertools.chain(*[m.stems() for m in self.modules.itervalues()]) + class BinariesCollection(object): """Tracks state of binaries produced by the build.""" @@ -109,6 +111,7 @@ class BinariesCollection(object): self.shared_libraries = [] self.programs = [] + class CommonBackend(BuildBackend): """Holds logic common to all build backends.""" @@ -182,7 +185,8 @@ class CommonBackend(BuildBackend): return False elif isinstance(obj, Exports): - objdir_files = [f.full_path for path, files in obj.files.walk() for f in files if isinstance(f, ObjDirPath)] + objdir_files = [f.full_path for path, files in obj.files.walk() + for f in files if isinstance(f, ObjDirPath)] if objdir_files: self._handle_generated_sources(objdir_files) return False @@ -204,7 +208,6 @@ class CommonBackend(BuildBackend): self._handle_generated_sources(mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % stem) for stem in self._idl_manager.idl_stems()) - for config in self._configs: self.backend_input_files.add(config.source) @@ -294,7 +297,7 @@ class CommonBackend(BuildBackend): seen_libs.add(lib) os_libs.append(lib) - return (objs, sorted(seen_pgo_gen_only_objs), no_pgo_objs, \ + return (objs, sorted(seen_pgo_gen_only_objs), no_pgo_objs, shared_libs, os_libs, static_libs) def _make_list_file(self, kind, objdir, objs, name): @@ -332,7 +335,8 @@ class CommonBackend(BuildBackend): return ref def _handle_generated_sources(self, files): - self._generated_sources.update(mozpath.relpath(f, self.environment.topobjdir) for f in files) + self._generated_sources.update(mozpath.relpath( + f, self.environment.topobjdir) for f in files) def _handle_webidl_collection(self, webidls): @@ -411,7 +415,7 @@ class CommonBackend(BuildBackend): 'so it cannot be built in unified mode."\n' '#undef INITGUID\n' '#endif') - f.write('\n'.join(includeTemplate % { "cppfile": s } for + f.write('\n'.join(includeTemplate % {"cppfile": s} for s in source_filenames)) def _write_unified_files(self, unified_source_mapping, output_directory, diff --git a/python/mozbuild/mozbuild/backend/configenvironment.py b/python/mozbuild/mozbuild/backend/configenvironment.py index 76eec5889e42..770f36dce2dd 100644 --- a/python/mozbuild/mozbuild/backend/configenvironment.py +++ b/python/mozbuild/mozbuild/backend/configenvironment.py @@ -118,7 +118,7 @@ class ConfigEnvironment(object): """ def __init__(self, topsrcdir, topobjdir, defines=None, - non_global_defines=None, substs=None, source=None, mozconfig=None): + non_global_defines=None, substs=None, source=None, mozconfig=None): if not source: source = mozpath.join(topobjdir, 'config.status') @@ -148,10 +148,11 @@ class ConfigEnvironment(object): self.bin_suffix = self.substs.get('BIN_SUFFIX', '') global_defines = [name for name in self.defines - if not name in self.non_global_defines] + if not name in self.non_global_defines] self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name, - shell_quote(self.defines[name]).replace('$', '$$')) - for name in sorted(global_defines)]) + shell_quote(self.defines[name]).replace('$', '$$')) + for name in sorted(global_defines)]) + def serialize(name, obj): if isinstance(obj, StringTypes): return obj @@ -159,9 +160,9 @@ class ConfigEnvironment(object): return ' '.join(obj) raise Exception('Unhandled type %s for %s', type(obj), str(name)) self.substs['ALLSUBSTS'] = '\n'.join(sorted(['%s = %s' % (name, - serialize(name, self.substs[name])) for name in self.substs if self.substs[name]])) + serialize(name, self.substs[name])) for name in self.substs if self.substs[name]])) self.substs['ALLEMPTYSUBSTS'] = '\n'.join(sorted(['%s =' % name - for name in self.substs if not self.substs[name]])) + for name in self.substs if not self.substs[name]])) self.substs = ReadOnlyDict(self.substs) @@ -214,7 +215,7 @@ class ConfigEnvironment(object): config = BuildConfig.from_config_status(path) return ConfigEnvironment(config.topsrcdir, config.topobjdir, - config.defines, config.non_global_defines, config.substs, path) + config.defines, config.non_global_defines, config.substs, path) class PartialConfigDict(object): @@ -224,6 +225,7 @@ class PartialConfigDict(object): similar for substs), where the value of FOO is delay-loaded until it is needed. """ + def __init__(self, config_statusd, typ, environ_override=False): self._dict = {} self._datadir = mozpath.join(config_statusd, typ) @@ -338,6 +340,7 @@ class PartialConfigEnvironment(object): intended to be used instead of the defines structure from config.status so that scripts can depend directly on its value. """ + def __init__(self, topobjdir): config_statusd = mozpath.join(topobjdir, 'config.statusd') self.substs = PartialConfigDict(config_statusd, 'substs', environ_override=True) @@ -353,8 +356,8 @@ class PartialConfigEnvironment(object): if name not in config['non_global_defines'] ] acdefines = ' '.join(['-D%s=%s' % (name, - shell_quote(config['defines'][name]).replace('$', '$$')) - for name in sorted(global_defines)]) + shell_quote(config['defines'][name]).replace('$', '$$')) + for name in sorted(global_defines)]) substs['ACDEFINES'] = acdefines all_defines = OrderedDict() diff --git a/python/mozbuild/mozbuild/backend/cpp_eclipse.py b/python/mozbuild/mozbuild/backend/cpp_eclipse.py index be8daf5ac9b1..e9f65fc2ef9a 100644 --- a/python/mozbuild/mozbuild/backend/cpp_eclipse.py +++ b/python/mozbuild/mozbuild/backend/cpp_eclipse.py @@ -26,6 +26,7 @@ from mozbuild.base import ExecutionSummary # Open eclipse: # /Users/bgirard/mozilla/eclipse/eclipse/eclipse/eclipse -data $PWD/workspace + class CppEclipseBackend(CommonBackend): """Backend that generates Cpp Eclipse project files. """ @@ -81,7 +82,8 @@ class CppEclipseBackend(CommonBackend): # Note that unlike VS, Eclipse' indexer seem to crawl the headers and # isn't picky about the local includes. if isinstance(obj, ComputedFlags): - args = self._args_for_dirs.setdefault('tree/' + reldir, {'includes': [], 'defines': []}) + args = self._args_for_dirs.setdefault( + 'tree/' + reldir, {'includes': [], 'defines': []}) # use the same args for any objdirs we include: if reldir == 'dom/bindings': self._args_for_dirs.setdefault('generated-webidl', args) @@ -105,7 +107,8 @@ class CppEclipseBackend(CommonBackend): def consume_finished(self): settings_dir = os.path.join(self._project_dir, '.settings') launch_dir = os.path.join(self._project_dir, 'RunConfigurations') - workspace_settings_dir = os.path.join(self._workspace_dir, '.metadata/.plugins/org.eclipse.core.runtime/.settings') + workspace_settings_dir = os.path.join( + self._workspace_dir, '.metadata/.plugins/org.eclipse.core.runtime/.settings') for dir_name in [self._project_dir, settings_dir, launch_dir, workspace_settings_dir, self._workspace_lang_dir]: try: @@ -129,22 +132,25 @@ class CppEclipseBackend(CommonBackend): workspace_language_path = os.path.join(self._workspace_lang_dir, 'language.settings.xml') with open(workspace_language_path, 'wb') as fh: workspace_lang_settings = WORKSPACE_LANGUAGE_SETTINGS_TEMPLATE - workspace_lang_settings = workspace_lang_settings.replace("@COMPILER_FLAGS@", self._cxx + " " + self._cppflags); + workspace_lang_settings = workspace_lang_settings.replace( + "@COMPILER_FLAGS@", self._cxx + " " + self._cppflags) fh.write(workspace_lang_settings) self._write_launch_files(launch_dir) - core_resources_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.core.resources.prefs') + core_resources_prefs_path = os.path.join( + workspace_settings_dir, 'org.eclipse.core.resources.prefs') with open(core_resources_prefs_path, 'wb') as fh: - fh.write(STATIC_CORE_RESOURCES_PREFS); + fh.write(STATIC_CORE_RESOURCES_PREFS) - core_runtime_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.core.runtime.prefs') + core_runtime_prefs_path = os.path.join( + workspace_settings_dir, 'org.eclipse.core.runtime.prefs') with open(core_runtime_prefs_path, 'wb') as fh: - fh.write(STATIC_CORE_RUNTIME_PREFS); + fh.write(STATIC_CORE_RUNTIME_PREFS) ui_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.ui.prefs') with open(ui_prefs_path, 'wb') as fh: - fh.write(STATIC_UI_PREFS); + fh.write(STATIC_UI_PREFS) cdt_ui_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.cdt.ui.prefs') cdt_ui_prefs = STATIC_CDT_UI_PREFS @@ -155,10 +161,11 @@ class CppEclipseBackend(CommonBackend): XML_PREF_TEMPLATE = """\\n""" for line in FORMATTER_SETTINGS.splitlines(): [pref, val] = line.split("=") - cdt_ui_prefs += XML_PREF_TEMPLATE.replace("@PREF_NAME@", pref).replace("@PREF_VAL@", val) + cdt_ui_prefs += XML_PREF_TEMPLATE.replace("@PREF_NAME@", + pref).replace("@PREF_VAL@", val) cdt_ui_prefs += "\\n\\n" with open(cdt_ui_prefs_path, 'wb') as fh: - fh.write(cdt_ui_prefs); + fh.write(cdt_ui_prefs) cdt_core_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.cdt.core.prefs') with open(cdt_core_prefs_path, 'wb') as fh: @@ -168,11 +175,11 @@ class CppEclipseBackend(CommonBackend): # as the active formatter all its prefs are set in this prefs file, # so we need add those now: cdt_core_prefs += FORMATTER_SETTINGS - fh.write(cdt_core_prefs); + fh.write(cdt_core_prefs) - editor_prefs_path = os.path.join(workspace_settings_dir, "org.eclipse.ui.editors.prefs"); + editor_prefs_path = os.path.join(workspace_settings_dir, "org.eclipse.ui.editors.prefs") with open(editor_prefs_path, 'wb') as fh: - fh.write(EDITOR_SETTINGS); + fh.write(EDITOR_SETTINGS) # Now import the project into the workspace self._import_project() @@ -208,7 +215,7 @@ class CppEclipseBackend(CommonBackend): def _write_noindex(self): noindex_path = os.path.join(self._project_dir, '.settings/org.eclipse.cdt.core.prefs') with open(noindex_path, 'wb') as fh: - fh.write(NOINDEX_TEMPLATE); + fh.write(NOINDEX_TEMPLATE) def _remove_noindex(self): # Below we remove the config file that temporarily disabled the indexer @@ -257,7 +264,8 @@ class CppEclipseBackend(CommonBackend): dirsettings_template = LANGUAGE_SETTINGS_TEMPLATE_DIR_HEADER # Add OS_COMPILE_CXXFLAGS args (same as OS_COMPILE_CFLAGS): - dirsettings_template = dirsettings_template.replace('@PREINCLUDE_FILE_PATH@', os.path.join(self.environment.topobjdir, 'dist/include/mozilla-config.h')) + dirsettings_template = dirsettings_template.replace('@PREINCLUDE_FILE_PATH@', os.path.join( + self.environment.topobjdir, 'dist/include/mozilla-config.h')) dirsettings_template += add_define('MOZILLA_CLIENT', '1') # Add EXTRA_INCLUDES args: @@ -304,7 +312,7 @@ class CppEclipseBackend(CommonBackend): # netwerk/sctp/src uses -U__APPLE__ on Mac # XXX We should make this code smart enough to remove existing defines. continue - d = d[2:] # get rid of leading "-D" + d = d[2:] # get rid of leading "-D" name_value = d.split("=", 1) name = name_value[0] value = "" @@ -314,7 +322,8 @@ class CppEclipseBackend(CommonBackend): dirsettings += LANGUAGE_SETTINGS_TEMPLATE_DIR_FOOTER fh.write(dirsettings) - fh.write(LANGUAGE_SETTINGS_TEMPLATE_FOOTER.replace("@COMPILER_FLAGS@", self._cxx + " " + self._cppflags)) + fh.write(LANGUAGE_SETTINGS_TEMPLATE_FOOTER.replace( + "@COMPILER_FLAGS@", self._cxx + " " + self._cppflags)) def _write_launch_files(self, launch_dir): bin_dir = os.path.join(self.environment.topobjdir, 'dist') @@ -334,21 +343,25 @@ class CppEclipseBackend(CommonBackend): launch = launch.replace('@LAUNCH_ARGS@', '-P -no-remote') fh.write(launch) - #TODO Add more launch configs (and delegate calls to mach) + # TODO Add more launch configs (and delegate calls to mach) def _write_project(self, fh): - project = PROJECT_TEMPLATE; + project = PROJECT_TEMPLATE project = project.replace('@PROJECT_NAME@', self._project_name) project = project.replace('@PROJECT_TOPSRCDIR@', self.environment.topsrcdir) - project = project.replace('@GENERATED_IPDL_FILES@', os.path.join(self.environment.topobjdir, "ipc", "ipdl")) - project = project.replace('@GENERATED_WEBIDL_FILES@', os.path.join(self.environment.topobjdir, "dom", "bindings")) + project = project.replace('@GENERATED_IPDL_FILES@', os.path.join( + self.environment.topobjdir, "ipc", "ipdl")) + project = project.replace('@GENERATED_WEBIDL_FILES@', os.path.join( + self.environment.topobjdir, "dom", "bindings")) fh.write(project) def _write_cproject(self, fh): cproject_header = CPROJECT_TEMPLATE_HEADER - cproject_header = cproject_header.replace('@PROJECT_TOPSRCDIR@', self.environment.topobjdir) - cproject_header = cproject_header.replace('@MACH_COMMAND@', os.path.join(self.environment.topsrcdir, 'mach')) + cproject_header = cproject_header.replace( + '@PROJECT_TOPSRCDIR@', self.environment.topobjdir) + cproject_header = cproject_header.replace( + '@MACH_COMMAND@', os.path.join(self.environment.topsrcdir, 'mach')) fh.write(cproject_header) fh.write(CPROJECT_TEMPLATE_FOOTER) @@ -615,21 +628,21 @@ undoHistorySize=200 """ -STATIC_CORE_RESOURCES_PREFS="""eclipse.preferences.version=1 +STATIC_CORE_RESOURCES_PREFS = """eclipse.preferences.version=1 refresh.enabled=true """ -STATIC_CORE_RUNTIME_PREFS="""eclipse.preferences.version=1 +STATIC_CORE_RUNTIME_PREFS = """eclipse.preferences.version=1 content-types/org.eclipse.cdt.core.cxxSource/file-extensions=mm content-types/org.eclipse.core.runtime.xml/file-extensions=xul content-types/org.eclipse.wst.jsdt.core.jsSource/file-extensions=jsm """ -STATIC_UI_PREFS="""eclipse.preferences.version=1 +STATIC_UI_PREFS = """eclipse.preferences.version=1 showIntro=false """ -STATIC_CDT_CORE_PREFS="""eclipse.preferences.version=1 +STATIC_CDT_CORE_PREFS = """eclipse.preferences.version=1 indexer.updatePolicy=0 """ @@ -797,7 +810,7 @@ org.eclipse.cdt.core.formatter.tabulation.size=2 org.eclipse.cdt.core.formatter.use_tabs_only_for_leading_indentations=false """ -STATIC_CDT_UI_PREFS="""eclipse.preferences.version=1 +STATIC_CDT_UI_PREFS = """eclipse.preferences.version=1 buildConsoleLines=10000 Console.limitConsoleOutput=false ensureNewlineAtEOF=false diff --git a/python/mozbuild/mozbuild/backend/fastermake.py b/python/mozbuild/mozbuild/backend/fastermake.py index 61479a46514f..626313edac45 100644 --- a/python/mozbuild/mozbuild/backend/fastermake.py +++ b/python/mozbuild/mozbuild/backend/fastermake.py @@ -134,10 +134,12 @@ class FasterMakeBackend(CommonBackend, PartialBackend): elif isinstance(obj, GeneratedFile): if obj.outputs: - first_output = mozpath.relpath(mozpath.join(obj.objdir, obj.outputs[0]), self.environment.topobjdir) + first_output = mozpath.relpath(mozpath.join( + obj.objdir, obj.outputs[0]), self.environment.topobjdir) for o in obj.outputs[1:]: fullpath = mozpath.join(obj.objdir, o) - self._generated_files_map[mozpath.relpath(fullpath, self.environment.topobjdir)] = first_output + self._generated_files_map[mozpath.relpath( + fullpath, self.environment.topobjdir)] = first_output # We don't actually handle GeneratedFiles, we just need to know if # we can build multiple of them from a single make invocation in the # faster backend. @@ -194,7 +196,6 @@ class FasterMakeBackend(CommonBackend, PartialBackend): mk.create_rule([target]).add_dependencies( '$(TOPOBJDIR)/%s' % d for d in deps) - # This is not great, but it's better to have some dependencies on these Python files. python_deps = [ '$(TOPSRCDIR)/python/mozbuild/mozbuild/action/l10n_merge.py', @@ -208,7 +209,8 @@ class FasterMakeBackend(CommonBackend, PartialBackend): for (merge, ref_file, l10n_file) in deps: rule = mk.create_rule([merge]).add_dependencies( [ref_file, l10n_file] + python_deps) - rule.add_commands(['$(PYTHON) -m mozbuild.action.l10n_merge --output {} --ref-file {} --l10n-file {}'.format(merge, ref_file, l10n_file)]) + rule.add_commands( + ['$(PYTHON) -m mozbuild.action.l10n_merge --output {} --ref-file {} --l10n-file {}'.format(merge, ref_file, l10n_file)]) # Add a dummy rule for the l10n file since it might not exist. mk.create_rule([l10n_file]) diff --git a/python/mozbuild/mozbuild/backend/mach_commands.py b/python/mozbuild/mozbuild/backend/mach_commands.py index 028fefcd0999..28e7b1aec805 100644 --- a/python/mozbuild/mozbuild/backend/mach_commands.py +++ b/python/mozbuild/mozbuild/backend/mach_commands.py @@ -20,10 +20,11 @@ from mach.decorators import ( Command, ) + @CommandProvider class MachCommands(MachCommandBase): @Command('ide', category='devenv', - description='Generate a project and launch an IDE.') + description='Generate a project and launch an IDE.') @CommandArgument('ide', choices=['eclipse', 'visualstudio']) @CommandArgument('args', nargs=argparse.REMAINDER) def eclipse(self, ide, args): diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index d550cd876f53..a4cff5b46989 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -126,7 +126,7 @@ MOZBUILD_VARIABLES = [ b'TEST_DIRS', b'TOOL_DIRS', # XXX config/Makefile.in specifies this in a make invocation - #'USE_EXTENSION_MANIFEST', + # 'USE_EXTENSION_MANIFEST', b'XPCSHELL_TESTS', b'XPIDL_MODULE', ] @@ -225,9 +225,9 @@ class BackendMakeFile(object): self.fh.write('NONRECURSIVE_TARGETS += export\n') self.fh.write('NONRECURSIVE_TARGETS_export += xpidl\n') self.fh.write('NONRECURSIVE_TARGETS_export_xpidl_DIRECTORY = ' - '$(DEPTH)/xpcom/xpidl\n') + '$(DEPTH)/xpcom/xpidl\n') self.fh.write('NONRECURSIVE_TARGETS_export_xpidl_TARGETS += ' - 'export\n') + 'export\n') return self.fh.close() @@ -248,6 +248,7 @@ class RecursiveMakeTraversal(object): """ SubDirectoryCategories = ['dirs', 'tests'] SubDirectoriesTuple = namedtuple('SubDirectories', SubDirectoryCategories) + class SubDirectories(SubDirectoriesTuple): def __new__(self): return RecursiveMakeTraversal.SubDirectoriesTuple.__new__(self, [], []) @@ -390,7 +391,7 @@ class RecursiveMakeBackend(CommonBackend): self._test_manifests = {} self.backend_input_files.add(mozpath.join(self.environment.topobjdir, - 'config', 'autoconf.mk')) + 'config', 'autoconf.mk')) self._install_manifests = defaultdict(InstallManifest) # The build system relies on some install manifests always existing @@ -424,7 +425,7 @@ class RecursiveMakeBackend(CommonBackend): if obj.objdir not in self._backend_files: self._backend_files[obj.objdir] = \ BackendMakeFile(obj.srcdir, obj.objdir, obj.config, - obj.topsrcdir, self.environment.topobjdir, self.dry_run) + obj.topsrcdir, self.environment.topobjdir, self.dry_run) return self._backend_files[obj.objdir] def consume_object(self, obj): @@ -609,19 +610,19 @@ class RecursiveMakeBackend(CommonBackend): \t@$(TOUCH) $@ """.format(stub=stub_file, - output=first_output, - dep_file=dep_file, - inputs=' ' + ' '.join(inputs) if inputs else '', - flags=' ' + ' '.join(shell_quote(f) for f in obj.flags) if obj.flags else '', - backend=' backend.mk' if obj.flags else '', - # Locale repacks repack multiple locales from a single configured objdir, - # so standard mtime dependencies won't work properly when the build is re-run - # with a different locale as input. IS_LANGUAGE_REPACK will reliably be set - # in this situation, so simply force the generation to run in that case. - force=force, - locale='--locale=$(AB_CD) ' if obj.localized else '', - script=obj.script, - method=obj.method)) + output=first_output, + dep_file=dep_file, + inputs=' ' + ' '.join(inputs) if inputs else '', + flags=' ' + ' '.join(shell_quote(f) for f in obj.flags) if obj.flags else '', + backend=' backend.mk' if obj.flags else '', + # Locale repacks repack multiple locales from a single configured objdir, + # so standard mtime dependencies won't work properly when the build is re-run + # with a different locale as input. IS_LANGUAGE_REPACK will reliably be set + # in this situation, so simply force the generation to run in that case. + force=force, + locale='--locale=$(AB_CD) ' if obj.localized else '', + script=obj.script, + method=obj.method)) elif isinstance(obj, JARManifest): self._no_skip['libs'].add(backend_file.relobjdir) @@ -784,7 +785,7 @@ class RecursiveMakeBackend(CommonBackend): main, all_deps = \ self._traversal.compute_dependencies(filter) for dir, deps in all_deps.items(): - if deps is not None or (dir in self._idl_dirs \ + if deps is not None or (dir in self._idl_dirs and tier == 'export'): rule = root_deps_mk.create_rule(['%s/%s' % (dir, tier)]) if deps: @@ -795,8 +796,8 @@ class RecursiveMakeBackend(CommonBackend): if main: rule.add_dependencies('%s/%s' % (d, tier) for d in main) - all_compile_deps = reduce(lambda x,y: x|y, - self._compile_graph.values()) if self._compile_graph else set() + all_compile_deps = reduce(lambda x, y: x | y, + self._compile_graph.values()) if self._compile_graph else set() # Include the following as dependencies of the top recursion target for # compilation: # - nodes that are not dependended upon by anything. Typically, this @@ -902,8 +903,8 @@ class RecursiveMakeBackend(CommonBackend): if include_curdir_build_rules: makefile.add_statement('\n' - '# Make sometimes gets confused between "foo" and "$(CURDIR)/foo".\n' - '# Help it out by explicitly specifiying dependencies.') + '# Make sometimes gets confused between "foo" and "$(CURDIR)/foo".\n' + '# Help it out by explicitly specifiying dependencies.') makefile.add_statement('all_absolute_unified_files := \\\n' ' $(addprefix $(CURDIR)/,$(%s))' % unified_files_makefile_variable) @@ -949,7 +950,7 @@ class RecursiveMakeBackend(CommonBackend): stub = not os.path.exists(makefile_in) if not stub: self.log(logging.DEBUG, 'substitute_makefile', - {'path': makefile}, 'Substituting makefile: {path}') + {'path': makefile}, 'Substituting makefile: {path}') self._makefile_in_count += 1 # In the export and libs tiers, we don't skip directories @@ -961,7 +962,7 @@ class RecursiveMakeBackend(CommonBackend): self._no_skip[tier].add(bf.relobjdir) else: self.log(logging.DEBUG, 'stub_makefile', - {'path': makefile}, 'Creating stub Makefile: {path}') + {'path': makefile}, 'Creating stub Makefile: {path}') obj = self.Substitution() obj.output_path = makefile @@ -976,7 +977,7 @@ class RecursiveMakeBackend(CommonBackend): # XPI_PKGNAME or INSTALL_EXTENSION_ID can't be skipped and # must run during the 'tools' tier. for t in (b'XPI_PKGNAME', b'INSTALL_EXTENSION_ID', - b'tools'): + b'tools'): if t not in content: continue if t == b'tools' and not re.search('(?:^|\s)tools.*::', content, re.M): @@ -984,13 +985,13 @@ class RecursiveMakeBackend(CommonBackend): if objdir == self.environment.topobjdir: continue self._no_skip['tools'].add(mozpath.relpath(objdir, - self.environment.topobjdir)) + self.environment.topobjdir)) # Directories with a Makefile containing a check target # can't be skipped and must run during the 'check' tier. if re.search('(?:^|\s)check.*::', content, re.M): self._no_skip['check'].add(mozpath.relpath(objdir, - self.environment.topobjdir)) + self.environment.topobjdir)) # Detect any Makefile.ins that contain variables on the # moz.build-only list @@ -1070,7 +1071,7 @@ class RecursiveMakeBackend(CommonBackend): fh.write('DIRS := %s\n' % ' '.join( relativize(backend_file.objdir, obj.dirs))) self._traversal.add(backend_file.relobjdir, - dirs=relativize(self.environment.topobjdir, obj.dirs)) + dirs=relativize(self.environment.topobjdir, obj.dirs)) # The directory needs to be registered whether subdirectories have been # registered or not. @@ -1094,7 +1095,8 @@ class RecursiveMakeBackend(CommonBackend): if obj.target and not obj.is_custom(): backend_file.write('FINAL_TARGET = $(DEPTH)/%s\n' % (obj.target)) else: - backend_file.write('FINAL_TARGET = $(if $(XPI_NAME),$(DIST)/xpi-stage/$(XPI_NAME),$(DIST)/bin)$(DIST_SUBDIR:%=/%)\n') + backend_file.write( + 'FINAL_TARGET = $(if $(XPI_NAME),$(DIST)/xpi-stage/$(XPI_NAME),$(DIST)/bin)$(DIST_SUBDIR:%=/%)\n') if not obj.enabled: backend_file.write('NO_DIST_INSTALL := 1\n') @@ -1110,7 +1112,7 @@ class RecursiveMakeBackend(CommonBackend): for module in manager.modules: build_files.add_optional_exists(mozpath.join('.deps', - '%s.pp' % module)) + '%s.pp' % module)) modules = manager.modules xpt_modules = sorted(modules.keys()) @@ -1150,9 +1152,9 @@ class RecursiveMakeBackend(CommonBackend): obj = self.Substitution() obj.output_path = mozpath.join(self.environment.topobjdir, 'config', - 'makefiles', 'xpidl', 'Makefile') + 'makefiles', 'xpidl', 'Makefile') obj.input_path = mozpath.join(self.environment.topsrcdir, 'config', - 'makefiles', 'xpidl', 'Makefile.in') + 'makefiles', 'xpidl', 'Makefile.in') obj.topsrcdir = self.environment.topsrcdir obj.topobjdir = self.environment.topobjdir obj.config = self.environment @@ -1236,7 +1238,7 @@ class RecursiveMakeBackend(CommonBackend): # Much of the logic in this function could be moved to CommonBackend. for source in obj.source_relpaths: self.backend_input_files.add(mozpath.join(obj.topsrcdir, - source)) + source)) # Don't allow files to be defined multiple times unless it is allowed. # We currently allow duplicates for non-test files or test files if @@ -1251,7 +1253,7 @@ class RecursiveMakeBackend(CommonBackend): for base, pattern, dest in obj.pattern_installs: try: self._install_manifests['_test_files'].add_pattern_link(base, - pattern, dest) + pattern, dest) except ValueError: if not obj.dupe_manifest: raise @@ -1264,7 +1266,7 @@ class RecursiveMakeBackend(CommonBackend): raise m = self._test_manifests.setdefault(obj.flavor, - (obj.install_prefix, set())) + (obj.install_prefix, set())) m[1].add(obj.manifest_obj_relpath) try: @@ -1295,7 +1297,8 @@ class RecursiveMakeBackend(CommonBackend): def _process_per_source_flag(self, per_source_flag, backend_file): for flag in per_source_flag.flags: - backend_file.write('%s_FLAGS += %s\n' % (mozpath.basename(per_source_flag.file_name), flag)) + backend_file.write('%s_FLAGS += %s\n' % + (mozpath.basename(per_source_flag.file_name), flag)) def _process_computed_flags(self, computed_flags, backend_file): for var, flags in computed_flags.get_flags(): @@ -1357,7 +1360,7 @@ class RecursiveMakeBackend(CommonBackend): else: target_name = obj.KIND return '%s/%s' % (mozpath.relpath(obj.objdir, - self.environment.topobjdir), target_name) + self.environment.topobjdir), target_name) def _process_linked_libraries(self, obj, backend_file): def pretty_relpath(lib, name): @@ -1414,7 +1417,7 @@ class RecursiveMakeBackend(CommonBackend): # incorrect list file format to the host compiler as well as when # creating an archive with AR, which doesn't understand list files. if (objs == obj.objs and not isinstance(obj, (HostLibrary, StaticLibrary)) or - isinstance(obj, StaticLibrary) and obj.no_expand_lib): + isinstance(obj, StaticLibrary) and obj.no_expand_lib): backend_file.write_once('%s_OBJS := %s\n' % (obj.name, objs_ref)) if profile_gen_objs: @@ -1651,7 +1654,7 @@ class RecursiveMakeBackend(CommonBackend): def _write_manifests(self, dest, manifests): man_dir = mozpath.join(self.environment.topobjdir, '_build_manifests', - dest) + dest) for k, manifest in manifests.items(): with self._write_file(mozpath.join(man_dir, k)) as fh: @@ -1688,7 +1691,7 @@ class RecursiveMakeBackend(CommonBackend): pp.context.update(extra) if not pp.context.get('autoconfmk', ''): pp.context['autoconfmk'] = 'autoconf.mk' - pp.handleLine(b'# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n'); + pp.handleLine(b'# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n') pp.handleLine(b'DEPTH := @DEPTH@\n') pp.handleLine(b'topobjdir := @topobjdir@\n') pp.handleLine(b'topsrcdir := @top_srcdir@\n') @@ -1732,18 +1735,18 @@ class RecursiveMakeBackend(CommonBackend): rule.add_commands([ '$(RM) $@', '$(call py_action,preprocessor,$(DEFINES) $(ACDEFINES) ' - '$< -o $@)' + '$< -o $@)' ]) mk.add_statement('ALL_IPDLSRCS := %s %s' % (' '.join(sorted_nonstatic_ipdl_basenames), - ' '.join(sorted_static_ipdl_sources))) + ' '.join(sorted_static_ipdl_sources))) self._add_unified_build_rules(mk, unified_ipdl_cppsrcs_mapping, unified_files_makefile_variable='CPPSRCS') # Preprocessed ipdl files are generated in ipdl_dir. mk.add_statement('IPDLDIRS := %s %s' % (ipdl_dir, ' '.join(sorted(set(mozpath.dirname(p) - for p in sorted_static_ipdl_sources))))) + for p in sorted_static_ipdl_sources))))) with self._write_file(mozpath.join(ipdl_dir, 'ipdlsrcs.mk')) as ipdls: mk.dump(ipdls, removal_guard=False) @@ -1752,7 +1755,7 @@ class RecursiveMakeBackend(CommonBackend): webidls, expected_build_output_files, global_define_files): include_dir = mozpath.join(self.environment.topobjdir, 'dist', - 'include') + 'include') for f in expected_build_output_files: if f.startswith(include_dir): self._install_manifests['dist_include'].add_optional_exists( @@ -1783,12 +1786,12 @@ class RecursiveMakeBackend(CommonBackend): # which would modify content in the source directory. '$(RM) $@', '$(call py_action,preprocessor,$(DEFINES) $(ACDEFINES) ' - '$< -o $@)' + '$< -o $@)' ]) self._add_unified_build_rules(mk, - unified_source_mapping, - unified_files_makefile_variable='unified_binding_cpp_files') + unified_source_mapping, + unified_files_makefile_variable='unified_binding_cpp_files') webidls_mk = mozpath.join(bindings_dir, 'webidlsrcs.mk') with self._write_file(webidls_mk) as fh: diff --git a/python/mozbuild/mozbuild/backend/tup.py b/python/mozbuild/mozbuild/backend/tup.py index 046586ddc325..3e174e4f17be 100644 --- a/python/mozbuild/mozbuild/backend/tup.py +++ b/python/mozbuild/mozbuild/backend/tup.py @@ -332,9 +332,9 @@ class TupBackend(CommonBackend): tiers.set_tiers(('tup',)) tiers.begin_tier('tup') status = config.run_process(args=args, - line_handler=output.on_line, - ensure_exit_code=False, - append_env=self._get_mozconfig_env(config)) + line_handler=output.on_line, + ensure_exit_code=False, + append_env=self._get_mozconfig_env(config)) tiers.finish_tier('tup') if not status and self.environment.substs.get('MOZ_AUTOMATION'): config.log_manager.enable_unstructured() @@ -475,7 +475,6 @@ class TupBackend(CommonBackend): # accurate once we start building libraries in their final locations. inputs = objs + static_libs + shared_libs + [self._shlibs] - rust_linked = [l for l in prog.linked_libraries if isinstance(l, RustLibrary)] @@ -513,12 +512,10 @@ class TupBackend(CommonBackend): display='LINK %o' ) - def _gen_host_programs(self, backend_file): for p in backend_file.host_programs: self._gen_host_program(backend_file, p) - def _gen_host_program(self, backend_file, prog): _, _, _, _, extra_libs, _ = self._expand_libs(prog) objs = prog.objs @@ -559,7 +556,6 @@ class TupBackend(CommonBackend): display='LINK %o' ) - def _gen_static_library(self, backend_file): ar = [ backend_file.environment.substs['AR'], @@ -584,7 +580,6 @@ class TupBackend(CommonBackend): display='AR %o' ) - def consume_object(self, obj): """Write out build files necessary to build with tup.""" @@ -695,7 +690,7 @@ class TupBackend(CommonBackend): with self._write_file(mozpath.join(self.environment.topobjdir, 'Tuprules.tup')) as fh: acdefines_flags = ' '.join(['-D%s=%s' % (name, shell_quote(value)) - for (name, value) in sorted(self.environment.acdefines.iteritems())]) + for (name, value) in sorted(self.environment.acdefines.iteritems())]) # TODO: AB_CD only exists in Makefiles at the moment. acdefines_flags += ' -DAB_CD=en-US' @@ -729,12 +724,13 @@ class TupBackend(CommonBackend): # Ask the user to figure out where to run 'tup init' before # continuing. raise Exception("Please run `tup init --no-sync` in a common " - "ancestor directory of your objdir and srcdir, possibly " - "%s. To reduce file scanning overhead, this directory " - "should contain the fewest files possible that are not " - "necessary for this build." % tup_base_dir) + "ancestor directory of your objdir and srcdir, possibly " + "%s. To reduce file scanning overhead, this directory " + "should contain the fewest files possible that are not " + "necessary for this build." % tup_base_dir) tup = self.environment.substs.get('TUP', 'tup') - self._cmd.run_process(cwd=tup_base_dir, log_name='tup', args=[tup, 'init', '--no-sync']) + self._cmd.run_process(cwd=tup_base_dir, log_name='tup', + args=[tup, 'init', '--no-sync']) def _get_cargo_flags(self, obj): @@ -984,16 +980,15 @@ class TupBackend(CommonBackend): obj.name), output_group) - for val in enumerate(invocations): _process(*val) - def _gen_rust_rules(self, obj, backend_file): cargo_flags = self._get_cargo_flags(obj) cargo_env = self._get_cargo_env(obj, backend_file) output_lines = [] + def accumulate_output(line): output_lines.append(line) @@ -1014,7 +1009,6 @@ class TupBackend(CommonBackend): self._gen_cargo_rules(obj, cargo_plan, cargo_env, output_group) self.backend_input_files |= set(cargo_plan['inputs']) - def _process_generated_file(self, backend_file, obj): if obj.script and obj.method: backend_file.export_shell() @@ -1025,8 +1019,8 @@ class TupBackend(CommonBackend): obj.script, obj.method, obj.outputs[0], - '%s.pp' % obj.outputs[0], # deps file required - 'unused', # deps target is required + '%s.pp' % obj.outputs[0], # deps file required + 'unused', # deps target is required ]) full_inputs = [f.full_path for f in obj.inputs] cmd.extend(full_inputs) @@ -1163,7 +1157,8 @@ class TupBackend(CommonBackend): output=mozpath.join(output_dir, output), output_group=output_group) else: - backend_file.symlink_rule(f.full_path, output=f.target_basename, output_group=output_group) + backend_file.symlink_rule( + f.full_path, output=f.target_basename, output_group=output_group) else: if (self.environment.is_artifact_build and any(mozpath.match(f.target_basename, p) for p in self._compile_env_gen_files)): @@ -1175,18 +1170,19 @@ class TupBackend(CommonBackend): f.target_basename) gen_backend_file = self._get_backend_file(f.context.relobjdir) if gen_backend_file.requires_delay([f]): - gen_backend_file.delayed_installed_files.append((f.full_path, output, output_group)) + gen_backend_file.delayed_installed_files.append( + (f.full_path, output, output_group)) else: gen_backend_file.symlink_rule(f.full_path, output=output, output_group=output_group) - def _process_final_target_pp_files(self, obj, backend_file): for i, (path, files) in enumerate(obj.files.walk()): self._add_features(obj.install_target, path) for f in files: self._preprocess(backend_file, f.full_path, - destdir=mozpath.join(self.environment.topobjdir, obj.install_target, path), + destdir=mozpath.join(self.environment.topobjdir, + obj.install_target, path), target=f.target_basename) def _process_computed_flags(self, obj, backend_file): @@ -1315,7 +1311,8 @@ class TupBackend(CommonBackend): cmd.extend(['-I%s' % d for d in ipdldirs]) cmd.extend(sorted_ipdl_sources) - outputs = ['IPCMessageTypeName.cpp', mozpath.join(outheaderdir, 'IPCMessageStart.h'), 'ipdl_lextab.py', 'ipdl_yacctab.py'] + outputs = ['IPCMessageTypeName.cpp', mozpath.join( + outheaderdir, 'IPCMessageStart.h'), 'ipdl_lextab.py', 'ipdl_yacctab.py'] for filename in sorted_ipdl_sources: filepath, ext = os.path.splitext(filename) @@ -1379,4 +1376,5 @@ class TupBackend(CommonBackend): backend_file.sources['.cpp'].extend(sorted(global_define_files)) test_backend_file = self._get_backend_file('dom/bindings/test') - test_backend_file.sources['.cpp'].extend(sorted('../%sBinding.cpp' % s for s in webidls.all_test_stems())) + test_backend_file.sources['.cpp'].extend( + sorted('../%sBinding.cpp' % s for s in webidls.all_test_stems())) diff --git a/python/mozbuild/mozbuild/backend/visualstudio.py b/python/mozbuild/mozbuild/backend/visualstudio.py index 410b47351a2d..6e60e2b46172 100644 --- a/python/mozbuild/mozbuild/backend/visualstudio.py +++ b/python/mozbuild/mozbuild/backend/visualstudio.py @@ -34,21 +34,25 @@ from mozbuild.base import ExecutionSummary MSBUILD_NAMESPACE = 'http://schemas.microsoft.com/developer/msbuild/2003' + def get_id(name): return str(uuid.uuid5(uuid.NAMESPACE_URL, name)).upper() + def visual_studio_product_to_solution_version(version): if version == '2017': return '12.00', '15' else: raise Exception('Unknown version seen: %s' % version) + def visual_studio_product_to_platform_toolset_version(version): if version == '2017': return 'v141' else: raise Exception('Unknown version seen: %s' % version) + class VisualStudioBackend(CommonBackend): """Generate Visual Studio project files. @@ -100,7 +104,7 @@ class VisualStudioBackend(CommonBackend): elif isinstance(obj, UnifiedSources): # XXX we should be letting CommonBackend.consume_object call this # for us instead. - self._process_unified_sources(obj); + self._process_unified_sources(obj) elif isinstance(obj, Library): self._libs_to_paths[obj.basename] = reldir @@ -133,9 +137,9 @@ class VisualStudioBackend(CommonBackend): out_proj_dir = os.path.join(self._out_dir, self._projsubdir) projects = self._write_projects_for_sources(self._libs_to_paths, - "library", out_proj_dir) + "library", out_proj_dir) projects.update(self._write_projects_for_sources(self._progs_to_paths, - "binary", out_proj_dir)) + "binary", out_proj_dir)) # Generate projects that can be used to build common targets. for target in ('export', 'binaries', 'tools', 'full'): @@ -145,15 +149,15 @@ class VisualStudioBackend(CommonBackend): command += ' %s' % target project_id = self._write_vs_project(out_proj_dir, basename, target, - build_command=command, - clean_command='$(SolutionDir)\\mach.bat build clean') + build_command=command, + clean_command='$(SolutionDir)\\mach.bat build clean') projects[basename] = (project_id, basename, target) # A project that can be used to regenerate the visual studio projects. basename = 'target_vs' project_id = self._write_vs_project(out_proj_dir, basename, 'visual-studio', - build_command='$(SolutionDir)\\mach.bat build-backend -b VisualStudio') + build_command='$(SolutionDir)\\mach.bat build-backend -b VisualStudio') projects[basename] = (project_id, basename, 'visual-studio') # Write out a shared property file with common variables. @@ -190,7 +194,7 @@ class VisualStudioBackend(CommonBackend): headers = [t[0] for t in finder.find('*.h')] headers = [os.path.normpath(os.path.join('$(TopSrcDir)', - path, f)) for f in headers] + path, f)) for f in headers] includes = [ os.path.join('$(TopSrcDir)', path), @@ -201,7 +205,7 @@ class VisualStudioBackend(CommonBackend): includes.append('$(TopObjDir)\\dist\\include') for v in ('NSPR_CFLAGS', 'NSS_CFLAGS', 'MOZ_JPEG_CFLAGS', - 'MOZ_PNG_CFLAGS', 'MOZ_ZLIB_CFLAGS', 'MOZ_PIXMAN_CFLAGS'): + 'MOZ_PNG_CFLAGS', 'MOZ_ZLIB_CFLAGS', 'MOZ_PIXMAN_CFLAGS'): if not config: break @@ -223,7 +227,7 @@ class VisualStudioBackend(CommonBackend): else: defines.append('%s=%s' % (k, v)) - debugger=None + debugger = None if prefix == 'binary': if item.startswith(self.environment.substs['MOZ_APP_NAME']): app_args = '-no-remote -profile $(TopObjDir)\\tmp\\profile-default' @@ -236,12 +240,13 @@ class VisualStudioBackend(CommonBackend): basename = '%s_%s' % (prefix, item) project_id = self._write_vs_project(out_dir, basename, item, - includes=includes, - forced_includes=['$(TopObjDir)\\dist\\include\\mozilla-config.h'], - defines=defines, - headers=headers, - sources=sources, - debugger=debugger) + includes=includes, + forced_includes=[ + '$(TopObjDir)\\dist\\include\\mozilla-config.h'], + defines=defines, + headers=headers, + sources=sources, + debugger=debugger) projects[basename] = (project_id, basename, item) @@ -416,7 +421,7 @@ class VisualStudioBackend(CommonBackend): fh.write(b'$env:%s = "%s"\r\n' % (k, v)) relpath = os.path.relpath(self.environment.topsrcdir, - self.environment.topobjdir).replace('\\', '/') + self.environment.topobjdir).replace('\\', '/') fh.write(b'$bashargs = "%s/mach", "--log-no-times"\r\n' % relpath) fh.write(b'$bashargs = $bashargs + $args\r\n') @@ -425,9 +430,9 @@ class VisualStudioBackend(CommonBackend): fh.write(b'$procargs = "-c", $expanded\r\n') fh.write(b'Start-Process -WorkingDirectory $env:TOPOBJDIR ' - b'-FilePath $env:MOZILLABUILD\\msys\\bin\\bash ' - b'-ArgumentList $procargs ' - b'-Wait -NoNewWindow\r\n') + b'-FilePath $env:MOZILLABUILD\\msys\\bin\\bash ' + b'-ArgumentList $procargs ' + b'-Wait -NoNewWindow\r\n') def _write_mach_batch(self, fh): """Write out a batch script that builds the tree. @@ -445,12 +450,12 @@ class VisualStudioBackend(CommonBackend): # relative paths, since munging c:\ to /c/ is slightly more # complicated. relpath = os.path.relpath(self.environment.topsrcdir, - self.environment.topobjdir).replace('\\', '/') + self.environment.topobjdir).replace('\\', '/') # We go through mach because it has the logic for choosing the most # appropriate build tool. fh.write(b'"%%MOZILLABUILD%%\\msys\\bin\\bash" ' - b'-c "%s/mach --log-no-times %%1 %%2 %%3 %%4 %%5 %%6 %%7"' % relpath) + b'-c "%s/mach --log-no-times %%1 %%2 %%3 %%4 %%5 %%6 %%7"' % relpath) def _write_vs_project(self, out_dir, basename, name, **kwargs): root = '%s.vcxproj' % basename @@ -458,21 +463,21 @@ class VisualStudioBackend(CommonBackend): with self._write_file(os.path.join(out_dir, root), mode='rb') as fh: project_id, name = VisualStudioBackend.write_vs_project(fh, - self._version, project_id, name, **kwargs) + self._version, project_id, name, **kwargs) with self._write_file(os.path.join(out_dir, '%s.user' % root), mode='rb') as fh: fh.write('\r\n') fh.write('\r\n' % - MSBUILD_NAMESPACE) + MSBUILD_NAMESPACE) fh.write('\r\n') return project_id @staticmethod def write_vs_project(fh, version, project_id, name, includes=[], - forced_includes=[], defines=[], - build_command=None, clean_command=None, - debugger=None, headers=[], sources=[]): + forced_includes=[], defines=[], + build_command=None, clean_command=None, + debugger=None, headers=[], sources=[]): impl = getDOMImplementation() doc = impl.createDocument(MSBUILD_NAMESPACE, 'Project', None) @@ -510,7 +515,8 @@ class VisualStudioBackend(CommonBackend): rn.appendChild(doc.createTextNode('mozilla')) pts = pg.appendChild(doc.createElement('PlatformToolset')) - pts.appendChild(doc.createTextNode(visual_studio_product_to_platform_toolset_version(version))) + pts.appendChild(doc.createTextNode( + visual_studio_product_to_platform_toolset_version(version))) i = project.appendChild(doc.createElement('Import')) i.setAttribute('Project', '$(VCTargetsPath)\\Microsoft.Cpp.Default.props') diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py index b0d7ac8432cd..a2f18a36693f 100644 --- a/python/mozbuild/mozbuild/base.py +++ b/python/mozbuild/mozbuild/base.py @@ -53,12 +53,14 @@ def ancestors(path): break path = newpath + def samepath(path1, path2): if hasattr(os.path, 'samefile'): return os.path.samefile(path1, path2) return os.path.normcase(os.path.realpath(path1)) == \ os.path.normcase(os.path.realpath(path2)) + class BadEnvironmentException(Exception): """Base class for errors raised when the build environment is not sane.""" @@ -69,6 +71,7 @@ class BuildEnvironmentNotFoundException(BadEnvironmentException): class ObjdirMismatchException(BadEnvironmentException): """Raised when the current dir is an objdir and doesn't match the mozconfig.""" + def __init__(self, objdir1, objdir2): self.objdir1 = objdir1 self.objdir2 = objdir2 @@ -85,6 +88,7 @@ class MozbuildObject(ProcessExecutionMixin): running processes, etc. This classes provides that functionality. Other modules can inherit from this class to obtain this functionality easily. """ + def __init__(self, topsrcdir, settings, log_manager, topobjdir=None, mozconfig=MozconfigLoader.AUTODETECT): """Create a new Mozbuild object instance. @@ -176,8 +180,8 @@ class MozbuildObject(ProcessExecutionMixin): if topsrcdir == topobjdir: raise BadEnvironmentException('The object directory appears ' - 'to be the same as your source directory (%s). This build ' - 'configuration is not supported.' % topsrcdir) + 'to be the same as your source directory (%s). This build ' + 'configuration is not supported.' % topsrcdir) # If we can't resolve topobjdir, oh well. We'll figure out when we need # one. @@ -191,7 +195,7 @@ class MozbuildObject(ProcessExecutionMixin): if '@CONFIG_GUESS@' in topobjdir: topobjdir = topobjdir.replace('@CONFIG_GUESS@', - self.resolve_config_guess()) + self.resolve_config_guess()) if not os.path.isabs(topobjdir): topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir)) @@ -253,9 +257,10 @@ class MozbuildObject(ProcessExecutionMixin): def virtualenv_manager(self): if self._virtualenv_manager is None: self._virtualenv_manager = VirtualenvManager(self.topsrcdir, - self.topobjdir, os.path.join(self.topobjdir, '_virtualenvs', 'init'), - sys.stdout, os.path.join(self.topsrcdir, 'build', - 'virtualenv_packages.txt')) + self.topobjdir, os.path.join( + self.topobjdir, '_virtualenvs', 'init'), + sys.stdout, os.path.join(self.topsrcdir, 'build', + 'virtualenv_packages.txt')) return self._virtualenv_manager @@ -490,7 +495,6 @@ class MozbuildObject(ProcessExecutionMixin): return BuildReader(config, finder=finder) - @memoized_property def python3(self): """Obtain info about a Python 3 executable. @@ -542,10 +546,10 @@ class MozbuildObject(ProcessExecutionMixin): if substs['OS_ARCH'] == 'Darwin': if substs['MOZ_BUILD_APP'] == 'xulrunner': - stem = os.path.join(stem, 'XUL.framework'); + stem = os.path.join(stem, 'XUL.framework') else: stem = os.path.join(stem, substs['MOZ_MACBUNDLE_NAME'], 'Contents', - 'MacOS') + 'MacOS') elif where == 'default': stem = os.path.join(stem, 'bin') @@ -578,13 +582,14 @@ class MozbuildObject(ProcessExecutionMixin): notifier = which.which('terminal-notifier') except which.WhichError: raise Exception('Install terminal-notifier to get ' - 'a notification when the build finishes.') + 'a notification when the build finishes.') self.run_process([notifier, '-title', - 'Mozilla Build System', '-group', 'mozbuild', - '-message', msg], ensure_exit_code=False) + 'Mozilla Build System', '-group', 'mozbuild', + '-message', msg], ensure_exit_code=False) elif sys.platform.startswith('win'): from ctypes import Structure, windll, POINTER, sizeof from ctypes.wintypes import DWORD, HANDLE, WINFUNCTYPE, BOOL, UINT + class FLASHWINDOW(Structure): _fields_ = [("cbSize", UINT), ("hwnd", HANDLE), @@ -604,21 +609,21 @@ class MozbuildObject(ProcessExecutionMixin): return params = FLASHWINDOW(sizeof(FLASHWINDOW), - console, - FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0) + console, + FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0) FlashWindowEx(params) else: try: notifier = which.which('notify-send') except which.WhichError: raise Exception('Install notify-send (usually part of ' - 'the libnotify package) to get a notification when ' - 'the build finishes.') + 'the libnotify package) to get a notification when ' + 'the build finishes.') self.run_process([notifier, '--app-name=Mozilla Build System', - 'Mozilla Build System', msg], ensure_exit_code=False) + 'Mozilla Build System', msg], ensure_exit_code=False) except Exception as e: self.log(logging.WARNING, 'notifier-failed', {'error': - e.message}, 'Notification center failed: {error}') + e.message}, 'Notification center failed: {error}') def _ensure_objdir_exists(self): if os.path.isdir(self.statedir): @@ -646,10 +651,10 @@ class MozbuildObject(ProcessExecutionMixin): return PathArgument(arg, self.topsrcdir, self.topobjdir) def _run_make(self, directory=None, filename=None, target=None, log=True, - srcdir=False, allow_parallel=True, line_handler=None, - append_env=None, explicit_env=None, ignore_errors=False, - ensure_exit_code=0, silent=True, print_directory=True, - pass_thru=False, num_jobs=0, keep_going=False): + srcdir=False, allow_parallel=True, line_handler=None, + append_env=None, explicit_env=None, ignore_errors=False, + ensure_exit_code=0, silent=True, print_directory=True, + pass_thru=False, num_jobs=0, keep_going=False): """Invoke make. directory -- Relative directory to look for Makefile in. @@ -791,11 +796,11 @@ class MozbuildObject(ProcessExecutionMixin): if xcode_lisense_error: raise Exception('Xcode requires accepting to the license agreement.\n' - 'Please run Xcode and accept the license agreement.') + 'Please run Xcode and accept the license agreement.') if self._is_windows(): raise Exception('Could not find a suitable make implementation.\n' - 'Please use MozillaBuild 1.9 or newer') + 'Please use MozillaBuild 1.9 or newer') else: raise Exception('Could not find a suitable make implementation.') @@ -820,13 +825,12 @@ class MozbuildObject(ProcessExecutionMixin): """ return cls(self.topsrcdir, self.settings, self.log_manager, - topobjdir=self.topobjdir) + topobjdir=self.topobjdir) def _activate_virtualenv(self): self.virtualenv_manager.ensure() self.virtualenv_manager.activate() - def _set_log_level(self, verbose): self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG) @@ -835,7 +839,8 @@ class MozbuildObject(ProcessExecutionMixin): pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv') if not os.path.exists(pipenv): for package in ['certifi', 'pipenv', 'six', 'virtualenv', 'virtualenv-clone']: - path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party/python', package)) + path = os.path.normpath(os.path.join( + self.topsrcdir, 'third_party/python', package)) self.virtualenv_manager.install_pip_package(path, vendored=True) return pipenv @@ -861,10 +866,10 @@ class MachCommandBase(MozbuildObject): detect_virtualenv_mozinfo = True if hasattr(context, 'detect_virtualenv_mozinfo'): detect_virtualenv_mozinfo = getattr(context, - 'detect_virtualenv_mozinfo') + 'detect_virtualenv_mozinfo') try: dummy = MozbuildObject.from_environment(cwd=context.cwd, - detect_virtualenv_mozinfo=detect_virtualenv_mozinfo) + detect_virtualenv_mozinfo=detect_virtualenv_mozinfo) topsrcdir = dummy.topsrcdir topobjdir = dummy._topobjdir if topobjdir: @@ -881,12 +886,12 @@ class MachCommandBase(MozbuildObject): pass except ObjdirMismatchException as e: print('Ambiguous object directory detected. We detected that ' - 'both %s and %s could be object directories. This is ' - 'typically caused by having a mozconfig pointing to a ' - 'different object directory from the current working ' - 'directory. To solve this problem, ensure you do not have a ' - 'default mozconfig in searched paths.' % (e.objdir1, - e.objdir2)) + 'both %s and %s could be object directories. This is ' + 'typically caused by having a mozconfig pointing to a ' + 'different object directory from the current working ' + 'directory. To solve this problem, ensure you do not have a ' + 'default mozconfig in searched paths.' % (e.objdir1, + e.objdir2)) sys.exit(1) except MozconfigLoadException as e: @@ -903,7 +908,7 @@ class MachCommandBase(MozbuildObject): sys.exit(1) MozbuildObject.__init__(self, topsrcdir, context.settings, - context.log_manager, topobjdir=topobjdir) + context.log_manager, topobjdir=topobjdir) self._mach_context = context diff --git a/python/mozbuild/mozbuild/chunkify.py b/python/mozbuild/mozbuild/chunkify.py index da4b0791dae5..76f713493a84 100644 --- a/python/mozbuild/mozbuild/chunkify.py +++ b/python/mozbuild/mozbuild/chunkify.py @@ -54,4 +54,3 @@ def chunkify(things, this_chunk, chunks): return things[start:end] except TypeError: return islice(things, start, end) - diff --git a/python/mozbuild/mozbuild/codecoverage/chrome_map.py b/python/mozbuild/mozbuild/codecoverage/chrome_map.py index eb5fbab9fbe6..44ac25205d0b 100644 --- a/python/mozbuild/mozbuild/codecoverage/chrome_map.py +++ b/python/mozbuild/mozbuild/codecoverage/chrome_map.py @@ -26,6 +26,8 @@ from manifest_handler import ChromeManifestHandler _line_comment_re = re.compile('^//@line (\d+) "(.+)"$') + + def generate_pp_info(path, topsrcdir): with open(path) as fh: # (start, end) -> (included_source, start) @@ -57,6 +59,8 @@ def generate_pp_info(path, topsrcdir): # This build backend is assuming the build to have happened already, as it is parsing # built preprocessed files to generate data to map them to the original sources. + + class ChromeMapBackend(CommonBackend): def _init(self): CommonBackend._init(self) diff --git a/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py b/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py index ef83388f2141..b21a2b2ba16a 100644 --- a/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py +++ b/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py @@ -17,6 +17,7 @@ from mozpack.chrome.manifest import parse_manifest import mozpack.path as mozpath from manifest_handler import ChromeManifestHandler + class LcovRecord(object): __slots__ = ("test_name", "source_file", @@ -30,6 +31,7 @@ class LcovRecord(object): "lines", "line_count", "covered_line_count") + def __init__(self): self.functions = {} self.function_exec_counts = {} @@ -72,6 +74,7 @@ class LcovRecord(object): self.branch_count = len(self.branches) self.covered_branch_count = len([c for c in self.branches.values() if c]) + class RecordRewriter(object): # Helper class for rewriting/spliting individual lcov records according # to what the preprocessor did. @@ -164,7 +167,8 @@ class RecordRewriter(object): def rewrite_record(self, record, pp_info): # Rewrite the lines in the given record according to preprocessor info # and split to additional records when pp_info has included file info. - self._current_pp_info = dict([(tuple([int(l) for l in k.split(',')]), v) for k, v in pp_info.items()]) + self._current_pp_info = dict( + [(tuple([int(l) for l in k.split(',')]), v) for k, v in pp_info.items()]) self._ranges = sorted(self._current_pp_info.keys()) self._additions = {} self._rewrite_lines(record) @@ -178,6 +182,7 @@ class RecordRewriter(object): r.resummarize() return generated_records + class LcovFile(object): # Simple parser/pretty-printer for lcov format. # lcov parsing based on http://ltp.sourceforge.net/coverage/lcov/geninfo.1.php @@ -404,6 +409,7 @@ class LcovFile(object): class UrlFinderError(Exception): pass + class UrlFinder(object): # Given a "chrome://" or "resource://" url, uses data from the UrlMapBackend # and install manifests to find a path to the source file and the corresponding @@ -580,7 +586,8 @@ class UrlFinder(object): return url_obj.path, None dir_parts = parts[0].rsplit(app_name + '/', 1) - url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', 'bin', dir_parts[1].lstrip('/'), parts[1].lstrip('/'))) + url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', + 'bin', dir_parts[1].lstrip('/'), parts[1].lstrip('/'))) elif '.xpi!' in url: # This matching mechanism is quite brittle and based on examples seen in the wild. # There's no rule to match the XPI name to the path in dist/xpi-stage. @@ -590,7 +597,8 @@ class UrlFinder(object): addon_name = addon_name[:-len('-test@mozilla.org')] elif addon_name.endswith('@mozilla.org'): addon_name = addon_name[:-len('@mozilla.org')] - url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', 'xpi-stage', addon_name, parts[1].lstrip('/'))) + url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', + 'xpi-stage', addon_name, parts[1].lstrip('/'))) elif url_obj.scheme == 'file' and os.path.isabs(url_obj.path): path = url_obj.path if not os.path.isfile(path): @@ -607,6 +615,7 @@ class UrlFinder(object): self._final_mapping[url] = result return result + class LcovFileRewriter(object): # Class for partial parses of LCOV format and rewriting to resolve urls # and preprocessed file lines. @@ -694,5 +703,6 @@ def main(): rewriter.rewrite_files(files, args.output_file, args.output_suffix) + if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/codecoverage/packager.py b/python/mozbuild/mozbuild/codecoverage/packager.py index 6e5c975c67ed..6c41f815f73e 100644 --- a/python/mozbuild/mozbuild/codecoverage/packager.py +++ b/python/mozbuild/mozbuild/codecoverage/packager.py @@ -17,6 +17,7 @@ from mozpack.manifests import ( ) import mozpack.path as mozpath + def describe_install_manifest(manifest, dest_dir): try: manifest = InstallManifest(manifest) @@ -75,5 +76,6 @@ def cli(args=sys.argv[1:]): return package_coverage_data(args.root, args.output_file) + if __name__ == '__main__': sys.exit(cli()) diff --git a/python/mozbuild/mozbuild/compilation/codecomplete.py b/python/mozbuild/mozbuild/compilation/codecomplete.py index 05583961a552..affef00f65d1 100644 --- a/python/mozbuild/mozbuild/compilation/codecomplete.py +++ b/python/mozbuild/mozbuild/compilation/codecomplete.py @@ -26,9 +26,9 @@ class Introspection(MachCommandBase): """Instropection commands.""" @Command('compileflags', category='devenv', - description='Display the compilation flags for a given source file') + description='Display the compilation flags for a given source file') @CommandArgument('what', default=None, - help='Source file to display compilation flags for') + help='Source file to display compilation flags for') def compileflags(self, what): from mozbuild.util import resolve_target_to_make from mozbuild.compilation import util @@ -39,7 +39,7 @@ class Introspection(MachCommandBase): path_arg = self._wrap_path_argument(what) make_dir, make_target = resolve_target_to_make(self.topobjdir, - path_arg.relpath()) + path_arg.relpath()) if make_dir is None and make_target is None: return 1 diff --git a/python/mozbuild/mozbuild/compilation/database.py b/python/mozbuild/mozbuild/compilation/database.py index bbb634f712aa..ad1b628bfc8b 100644 --- a/python/mozbuild/mozbuild/compilation/database.py +++ b/python/mozbuild/mozbuild/compilation/database.py @@ -176,8 +176,8 @@ class CompileDBBackend(CommonBackend): if canonical_suffix not in self.COMPILERS: return db = self._db.setdefault((objdir, filename, unified), - cenv.substs[self.COMPILERS[canonical_suffix]].split() + - ['-o', '/dev/null', '-c']) + cenv.substs[self.COMPILERS[canonical_suffix]].split() + + ['-o', '/dev/null', '-c']) reldir = reldir or mozpath.relpath(objdir, cenv.topobjdir) def append_var(name): diff --git a/python/mozbuild/mozbuild/compilation/util.py b/python/mozbuild/mozbuild/compilation/util.py index 32ff2f8767ac..74b638045725 100644 --- a/python/mozbuild/mozbuild/compilation/util.py +++ b/python/mozbuild/mozbuild/compilation/util.py @@ -5,14 +5,16 @@ import os from mozbuild import shellutil + def check_top_objdir(topobjdir): top_make = os.path.join(topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been built yet. Please run ' - '|mach build| with no arguments.') + '|mach build| with no arguments.') return False return True + def get_build_vars(directory, cmd): build_vars = {} @@ -27,13 +29,14 @@ def get_build_vars(directory, cmd): try: old_logger = cmd.log_manager.replace_terminal_handler(None) cmd._run_make(directory=directory, target='showbuild', log=False, - print_directory=False, allow_parallel=False, silent=True, - line_handler=on_line) + print_directory=False, allow_parallel=False, silent=True, + line_handler=on_line) finally: cmd.log_manager.replace_terminal_handler(old_logger) return build_vars + def sanitize_cflags(flags): # We filter out -Xclang arguments as clang based tools typically choke on # passing these flags down to the clang driver. -Xclang tells the clang diff --git a/python/mozbuild/mozbuild/compilation/warnings.py b/python/mozbuild/mozbuild/compilation/warnings.py index 999189ea6ee5..386cbcf89871 100644 --- a/python/mozbuild/mozbuild/compilation/warnings.py +++ b/python/mozbuild/mozbuild/compilation/warnings.py @@ -87,22 +87,22 @@ class CompilerWarning(dict): return func(self._cmpkey(), other._cmpkey()) def __eq__(self, other): - return self._compare(other, lambda s,o: s == o) + return self._compare(other, lambda s, o: s == o) def __neq__(self, other): - return self._compare(other, lambda s,o: s != o) + return self._compare(other, lambda s, o: s != o) def __lt__(self, other): - return self._compare(other, lambda s,o: s < o) + return self._compare(other, lambda s, o: s < o) def __le__(self, other): - return self._compare(other, lambda s,o: s <= o) + return self._compare(other, lambda s, o: s <= o) def __gt__(self, other): - return self._compare(other, lambda s,o: s > o) + return self._compare(other, lambda s, o: s > o) def __ge__(self, other): - return self._compare(other, lambda s,o: s >= o) + return self._compare(other, lambda s, o: s >= o) def __hash__(self): """Define so this can exist inside a set, etc.""" @@ -132,6 +132,7 @@ class WarningsDatabase(object): Callers should periodically prune old, invalid warnings from the database by calling prune(). A good time to do this is at the end of a build. """ + def __init__(self): """Create an empty database.""" self._files = {} @@ -304,6 +305,7 @@ class WarningsCollector(object): output from the compiler. Therefore, it can maintain state to parse multi-line warning messages. """ + def __init__(self, cb, objdir=None): """Initialize a new collector. diff --git a/python/mozbuild/mozbuild/config_status.py b/python/mozbuild/mozbuild/config_status.py index c5bd24728e27..0a40bb87b53a 100644 --- a/python/mozbuild/mozbuild/config_status.py +++ b/python/mozbuild/mozbuild/config_status.py @@ -80,14 +80,14 @@ def config_status(topobjdir='.', topsrcdir='.', defines=None, if 'CONFIG_FILES' in os.environ: raise Exception('Using the CONFIG_FILES environment variable is not ' - 'supported.') + 'supported.') if 'CONFIG_HEADERS' in os.environ: raise Exception('Using the CONFIG_HEADERS environment variable is not ' - 'supported.') + 'supported.') if not os.path.isabs(topsrcdir): raise Exception('topsrcdir must be defined as an absolute directory: ' - '%s' % topsrcdir) + '%s' % topsrcdir) default_backends = ['RecursiveMake'] default_backends = (substs or {}).get('BUILD_BACKENDS', ['RecursiveMake']) @@ -112,8 +112,8 @@ def config_status(topobjdir='.', topsrcdir='.', defines=None, topobjdir = os.path.abspath('.') env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines, - non_global_defines=non_global_defines, substs=substs, - source=source, mozconfig=mozconfig) + non_global_defines=non_global_defines, substs=substs, + source=source, mozconfig=mozconfig) with FileAvoidWrite(os.path.join(topobjdir, 'mozinfo.json')) as f: write_mozinfo(f, env, os.environ) diff --git a/python/mozbuild/mozbuild/configure/__init__.py b/python/mozbuild/mozbuild/configure/__init__.py index 6b411bdc28a7..8e95b04d94b8 100644 --- a/python/mozbuild/mozbuild/configure/__init__.py +++ b/python/mozbuild/mozbuild/configure/__init__.py @@ -49,6 +49,7 @@ class ConfigureError(Exception): class SandboxDependsFunction(object): '''Sandbox-visible representation of @depends functions.''' + def __init__(self, unsandboxed): self._or = unsandboxed.__or__ self._and = unsandboxed.__and__ @@ -233,6 +234,7 @@ class CombinedDependsFunction(DependsFunction): def __ne__(self, other): return not self == other + class SandboxedGlobal(dict): '''Identifiable dict type for use as function global''' @@ -357,10 +359,12 @@ class ConfigureSandbox(dict): # that can't be converted to ascii. Make our log methods robust to this # by detecting the encoding that a producer is likely to have used. encoding = getpreferredencoding() + def wrapped_log_method(logger, key): method = getattr(logger, key) if not encoding: return method + def wrapped(*args, **kwargs): out_args = [ arg.decode(encoding) if isinstance(arg, str) else arg @@ -661,7 +665,7 @@ class ConfigureSandbox(dict): when = self._normalize_when(kwargs.get('when'), 'option') args = [self._resolve(arg) for arg in args] kwargs = {k: self._resolve(v) for k, v in kwargs.iteritems() - if k != 'when'} + if k != 'when'} option = Option(*args, **kwargs) if when: self._conditions[option] = when diff --git a/python/mozbuild/mozbuild/configure/check_debug_ranges.py b/python/mozbuild/mozbuild/configure/check_debug_ranges.py index c0caa9cc52f9..6fc8f2764318 100644 --- a/python/mozbuild/mozbuild/configure/check_debug_ranges.py +++ b/python/mozbuild/mozbuild/configure/check_debug_ranges.py @@ -12,6 +12,7 @@ import subprocess import sys import re + def get_range_for(compilation_unit, debug_info): '''Returns the range offset for a given compilation unit in a given debug_info.''' @@ -32,6 +33,7 @@ def get_range_for(compilation_unit, debug_info): ranges = nfo.rsplit(None, 1)[1] return None + def get_range_length(range, debug_ranges): '''Returns the number of items in the range starting at the given offset.''' @@ -42,8 +44,9 @@ def get_range_length(range, debug_ranges): length += 1 return length + def main(bin, compilation_unit): - p = subprocess.Popen(['objdump', '-W', bin], stdout = subprocess.PIPE, stderr = subprocess.PIPE) + p = subprocess.Popen(['objdump', '-W', bin], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = p.communicate() sections = re.split('\n(Contents of the|The section) ', out) debug_info = [s for s in sections if s.startswith('.debug_info')] diff --git a/python/mozbuild/mozbuild/configure/libstdcxx.py b/python/mozbuild/mozbuild/configure/libstdcxx.py index b78d0cb44c5d..58afaa2bda1b 100755 --- a/python/mozbuild/mozbuild/configure/libstdcxx.py +++ b/python/mozbuild/mozbuild/configure/libstdcxx.py @@ -21,12 +21,14 @@ import re re_for_ld = re.compile('.*\((.*)\).*') + def parse_readelf_line(x): """Return the version from a readelf line that looks like: 0x00ec: Rev: 1 Flags: none Index: 8 Cnt: 2 Name: GLIBCXX_3.4.6 """ return x.split(':')[-1].split('_')[-1].strip() + def parse_ld_line(x): """Parse a line from the output of ld -t. The output of gold is just the full path, gnu ld prints "-lstdc++ (path)". @@ -36,11 +38,13 @@ def parse_ld_line(x): return t.groups()[0].strip() return x.strip() + def split_ver(v): """Covert the string '1.2.3' into the list [1,2,3] """ return [int(x) for x in v.split('.')] + def cmp_ver(a, b): """Compare versions in the form 'a.b.c' """ @@ -49,17 +53,19 @@ def cmp_ver(a, b): return i - j return 0 + def encode_ver(v): """Encode the version as a single number. """ t = split_ver(v) return t[0] << 16 | t[1] << 8 | t[2] + def find_version(args): """Given a base command line for a compiler, find the version of the libstdc++ it uses. """ - args += ['-shared', '-Wl,-t'] + args += ['-shared', '-Wl,-t'] p = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) candidates = [x for x in p.stdout if 'libstdc++.so' in x] candidates = [x for x in candidates if 'skipping incompatible' not in x] @@ -77,9 +83,10 @@ candidates: p = subprocess.Popen(['readelf', '-V', libstdcxx], stdout=subprocess.PIPE) versions = [parse_readelf_line(x) for x in p.stdout.readlines() if 'Name: GLIBCXX' in x] - last_version = sorted(versions, cmp = cmp_ver)[-1] + last_version = sorted(versions, cmp=cmp_ver)[-1] return (last_version, encode_ver(last_version)) + if __name__ == '__main__': """Given the value of environment variable CXX or HOST_CXX, find the version of the libstdc++ it uses. diff --git a/python/mozbuild/mozbuild/configure/lint.py b/python/mozbuild/mozbuild/configure/lint.py index 1a6f5527ff1e..9afe7a78fc6f 100644 --- a/python/mozbuild/mozbuild/configure/lint.py +++ b/python/mozbuild/mozbuild/configure/lint.py @@ -237,7 +237,6 @@ class LintSandbox(ConfigureSandbox): name, default)) self._raise_from(e, frame.f_back if frame else None) - def _check_help_for_option_with_func_default(self, option, *args, **kwargs): default = kwargs['default'] @@ -285,6 +284,7 @@ class LintSandbox(ConfigureSandbox): def imports_impl(self, _import, _from=None, _as=None): wrapper = super(LintSandbox, self).imports_impl(_import, _from=_from, _as=_as) + def decorator(func): self._has_imports.add(func) return wrapper(func) diff --git a/python/mozbuild/mozbuild/configure/lint_util.py b/python/mozbuild/mozbuild/configure/lint_util.py index 4d6fea3f63b1..52e33920ee53 100644 --- a/python/mozbuild/mozbuild/configure/lint_util.py +++ b/python/mozbuild/mozbuild/configure/lint_util.py @@ -46,7 +46,7 @@ def disassemble_as_iter(co): c = code[i] op = ord(c) opname = dis.opname[op] - i += 1; + i += 1 if op >= dis.HAVE_ARGUMENT: arg = ord(code[i]) + ord(code[i + 1]) * 256 + extended_arg extended_arg = 0 diff --git a/python/mozbuild/mozbuild/configure/options.py b/python/mozbuild/mozbuild/configure/options.py index 8022c304a2b6..8a403272c31c 100644 --- a/python/mozbuild/mozbuild/configure/options.py +++ b/python/mozbuild/mozbuild/configure/options.py @@ -106,6 +106,7 @@ class PositiveOptionValue(OptionValue): in the form of a tuple for when values are given to the option (in the form --option=value[,value2...]. ''' + def __nonzero__(self): return True @@ -424,6 +425,7 @@ class CommandLineHelper(object): Extra options can be added afterwards through API calls. For those, conflicting values will raise an exception. ''' + def __init__(self, environ=os.environ, argv=sys.argv): self._environ = dict(environ) self._args = OrderedDict() diff --git a/python/mozbuild/mozbuild/configure/util.py b/python/mozbuild/mozbuild/configure/util.py index 9d8b2eb0ec66..cd8cd445c8d0 100644 --- a/python/mozbuild/mozbuild/configure/util.py +++ b/python/mozbuild/mozbuild/configure/util.py @@ -14,6 +14,7 @@ from collections import deque from contextlib import contextmanager from distutils.version import LooseVersion + def getpreferredencoding(): # locale._parse_localename makes locale.getpreferredencoding # return None when LC_ALL is C, instead of e.g. 'US-ASCII' or @@ -29,6 +30,7 @@ def getpreferredencoding(): encoding = 'utf-8' return encoding + class Version(LooseVersion): '''A simple subclass of distutils.version.LooseVersion. Adds attributes for `major`, `minor`, `patch` for the first three @@ -40,13 +42,14 @@ class Version(LooseVersion): v.minor == 2 v.patch == 0 ''' + def __init__(self, version): # Can't use super, LooseVersion's base class is not a new-style class. LooseVersion.__init__(self, version) # Take the first three integer components, stopping at the first # non-integer and padding the rest with zeroes. (self.major, self.minor, self.patch) = list(itertools.chain( - itertools.takewhile(lambda x:isinstance(x, int), self.version), + itertools.takewhile(lambda x: isinstance(x, int), self.version), (0, 0, 0)))[:3] def __cmp__(self, other): @@ -71,6 +74,7 @@ class ConfigureOutputHandler(logging.Handler): printed out. This feature is only enabled under the `queue_debug` context manager. ''' + def __init__(self, stdout=sys.stdout, stderr=sys.stderr, maxlen=20): super(ConfigureOutputHandler, self).__init__() @@ -193,6 +197,7 @@ class LineIO(object): '''File-like class that sends each line of the written data to a callback (without carriage returns). ''' + def __init__(self, callback, errors='strict'): self._callback = callback self._buf = '' diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py index 12033358f993..0df7d6f3c6ea 100644 --- a/python/mozbuild/mozbuild/controller/building.py +++ b/python/mozbuild/mozbuild/controller/building.py @@ -79,7 +79,7 @@ Preferences. INSTALL_TESTS_CLOBBER = ''.join([TextWrapper().fill(line) + '\n' for line in -''' + ''' The build system was unable to install tests because the CLOBBER file has \ been updated. This means if you edited any test files, your changes may not \ be picked up until a full/clobber build is performed. @@ -107,7 +107,7 @@ this is a clobber bug and not due to local changes. BuildOutputResult = namedtuple('BuildOutputResult', - ('warning', 'state_changed', 'message')) + ('warning', 'state_changed', 'message')) class TierStatus(object): @@ -173,9 +173,9 @@ class TierStatus(object): def add_resources_to_dict(self, entry, start=None, end=None, phase=None): """Helper function to append resource information to a dict.""" cpu_percent = self.resources.aggregate_cpu_percent(start=start, - end=end, phase=phase, per_cpu=False) + end=end, phase=phase, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(start=start, end=end, - phase=phase, per_cpu=False) + phase=phase, per_cpu=False) io = self.resources.aggregate_io(start=start, end=end, phase=phase) if cpu_percent is None: @@ -339,8 +339,8 @@ class BuildMonitor(MozbuildObject): json.dump(self.resources.as_dict(), fh, indent=2) except Exception as e: self.log(logging.WARNING, 'build_resources_error', - {'msg': str(e)}, - 'Exception when writing resource usage file: {msg}') + {'msg': str(e)}, + 'Exception when writing resource usage file: {msg}') def _get_finder_cpu_usage(self): """Obtain the CPU usage of the Finder app on OS X. @@ -438,9 +438,9 @@ class BuildMonitor(MozbuildObject): return None cpu_percent = self.resources.aggregate_cpu_percent(phase=None, - per_cpu=False) + per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(phase=None, - per_cpu=False) + per_cpu=False) io = self.resources.aggregate_io(phase=None) o = dict( @@ -462,9 +462,9 @@ class BuildMonitor(MozbuildObject): for usage in self.resources.range_usage(): cpu_percent = self.resources.aggregate_cpu_percent(usage.start, - usage.end, per_cpu=False) + usage.end, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(usage.start, - usage.end, per_cpu=False) + usage.end, per_cpu=False) entry = dict( start=usage.start, @@ -474,11 +474,10 @@ class BuildMonitor(MozbuildObject): ) self.tiers.add_resources_to_dict(entry, start=usage.start, - end=usage.end) + end=usage.end) o['resources'].append(entry) - # If the imports for this file ran before the in-tree virtualenv # was bootstrapped (for instance, for a clobber build in automation), # psutil might not be available. @@ -524,8 +523,8 @@ class BuildMonitor(MozbuildObject): sin /= 1048576 sout /= 1048576 self.log(logging.WARNING, 'swap_activity', - {'sin': sin, 'sout': sout}, - 'Swap in/out (MB): {sin}/{sout}') + {'sin': sin, 'sout': sout}, + 'Swap in/out (MB): {sin}/{sout}') def ccache_stats(self): ccache_stats = None @@ -548,6 +547,7 @@ class TerminalLoggingHandler(logging.Handler): This class should probably live elsewhere, like the mach core. Consider this a proving ground for its usefulness. """ + def __init__(self): logging.Handler.__init__(self) @@ -683,7 +683,6 @@ class BuildOutputManager(OutputManager): # collection child process hasn't been told to stop. self.monitor.stop_resource_recording() - def on_line(self, line): warning, state_changed, message = self.monitor.on_line(line) @@ -744,7 +743,7 @@ class StaticAnalysisOutputManager(OutputManager): if warning: self.log(logging.INFO, 'compiler_warning', warning, - 'Warning: {flag} in {filename}: {message}') + 'Warning: {flag} in {filename}: {message}') if relevant: self.log(logging.INFO, 'build_output', {'line': line}, '{line}') @@ -786,7 +785,7 @@ class CCacheStats(object): STATS_KEYS = [ # (key, description) # Refer to stats.c in ccache project for all the descriptions. - ('stats_zeroed', 'stats zero time'), # Old name prior to ccache 3.4 + ('stats_zeroed', 'stats zero time'), # Old name prior to ccache 3.4 ('stats_zeroed', 'stats zeroed'), ('stats_updated', 'stats updated'), ('cache_hit_direct', 'cache hit (direct)'), @@ -1005,11 +1004,11 @@ class BuildDriver(MozbuildObject): if directory is not None and not what: print('Can only use -C/--directory with an explicit target ' - 'name.') + 'name.') return 1 if directory is not None: - disable_extra_make_dependencies=True + disable_extra_make_dependencies = True directory = mozpath.normsep(directory) if directory.startswith('/'): directory = directory[1:] @@ -1104,7 +1103,7 @@ class BuildDriver(MozbuildObject): else: make_dir, make_target = \ resolve_target_to_make(self.topobjdir, - path_arg.relpath()) + path_arg.relpath()) if make_dir is None and make_target is None: return 1 @@ -1146,10 +1145,11 @@ class BuildDriver(MozbuildObject): # intelligent about encountering undefined state. no_build_status = b'1' if make_dir is not None else b'' status = self._run_make(directory=make_dir, target=make_target, - line_handler=output.on_line, log=False, print_directory=False, - ensure_exit_code=False, num_jobs=jobs, silent=not verbose, - append_env={b'NO_BUILDSTATUS_MESSAGES': no_build_status}, - keep_going=keep_going) + line_handler=output.on_line, log=False, print_directory=False, + ensure_exit_code=False, num_jobs=jobs, silent=not verbose, + append_env={ + b'NO_BUILDSTATUS_MESSAGES': no_build_status}, + keep_going=keep_going) if status != 0: break @@ -1204,8 +1204,8 @@ class BuildDriver(MozbuildObject): # in these directories. pathToThirdparty = os.path.join(self.topsrcdir, "tools", - "rewriting", - "ThirdPartyPaths.txt") + "rewriting", + "ThirdPartyPaths.txt") if os.path.exists(pathToThirdparty): with open(pathToThirdparty) as f: @@ -1284,7 +1284,8 @@ class BuildDriver(MozbuildObject): long_build = monitor.elapsed > 600 if long_build: - output.on_line('We know it took a while, but your build finally finished successfully!') + output.on_line( + 'We know it took a while, but your build finally finished successfully!') else: output.on_line('Your build was successful!') @@ -1299,7 +1300,7 @@ class BuildDriver(MozbuildObject): app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print('For more information on what to do now, see ' - 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') + 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') except Exception: # Ignore Exceptions in case we can't find config.status (such # as when doing OSX Universal builds) @@ -1325,14 +1326,14 @@ class BuildDriver(MozbuildObject): # Only print build status messages when we have an active # monitor. if not buildstatus_messages: - append_env[b'NO_BUILDSTATUS_MESSAGES'] = b'1' + append_env[b'NO_BUILDSTATUS_MESSAGES'] = b'1' status = self._run_client_mk(target='configure', line_handler=line_handler, append_env=append_env) if not status: print('Configure complete!') - print('Be sure to run |mach build| to pick up any changes'); + print('Be sure to run |mach build| to pick up any changes') return status diff --git a/python/mozbuild/mozbuild/controller/clobber.py b/python/mozbuild/mozbuild/controller/clobber.py index f5fd08e2e129..a597a390c4f2 100644 --- a/python/mozbuild/mozbuild/controller/clobber.py +++ b/python/mozbuild/mozbuild/controller/clobber.py @@ -16,7 +16,7 @@ from textwrap import TextWrapper CLOBBER_MESSAGE = ''.join([TextWrapper().fill(line) + '\n' for line in -''' + ''' The CLOBBER file has been updated, indicating that an incremental build since \ your last build will probably not work. A full/clobber build is required. @@ -39,6 +39,7 @@ Well, are ya? -- you can ignore this clobber requirement by running: $ touch {clobber_file} '''.splitlines()]) + class Clobberer(object): def __init__(self, topsrcdir, topobjdir): """Create a new object to manage clobbering the tree. @@ -69,7 +70,7 @@ class Clobberer(object): # Object directory clobber older than current is fine. if os.path.getmtime(self.src_clobber) <= \ - os.path.getmtime(self.obj_clobber): + os.path.getmtime(self.obj_clobber): return False @@ -182,8 +183,8 @@ class Clobberer(object): if not allow_auto: return True, False, \ self._message('Automatic clobbering is not enabled\n' - ' (add "mk_add_options AUTOCLOBBER=1" to your ' - 'mozconfig).') + ' (add "mk_add_options AUTOCLOBBER=1" to your ' + 'mozconfig).') if cwd.startswith(self.topobjdir) and cwd != self.topobjdir: return True, False, self._message( @@ -204,4 +205,4 @@ class Clobberer(object): lines = [' ' + line for line in self.clobber_cause()] return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines), - no_reason=' ' + reason, clobber_file=self.obj_clobber) + no_reason=' ' + reason, clobber_file=self.obj_clobber) diff --git a/python/mozbuild/mozbuild/doctor.py b/python/mozbuild/mozbuild/doctor.py index 8b8b6678c84e..be121cfa6de5 100644 --- a/python/mozbuild/mozbuild/doctor.py +++ b/python/mozbuild/mozbuild/doctor.py @@ -34,6 +34,7 @@ hour. Backup programs that rely on this feature may be affected. https://technet.microsoft.com/en-us/library/cc785435.aspx ''' + class Doctor(object): def __init__(self, srcdir, objdir, fix): self.srcdir = mozpath.normpath(srcdir) @@ -69,7 +70,7 @@ class Doctor(object): denied = True if denied: print('run "mach doctor --fix" AS ADMIN to re-attempt fixing your system') - elif False: # elif fixable: + elif False: # elif fixable: print('run "mach doctor --fix" as admin to attempt fixing your system') return int(not good) @@ -202,7 +203,7 @@ class Doctor(object): status = 'GOOD' desc = 'lastaccess disabled systemwide' elif disablelastaccess == 0: - if False: # if self.fix: + if False: # if self.fix: choice = self.prompt_bool(DISABLE_LASTACCESS_WIN) if not choice: return {'status': 'BAD, NOT FIXED', diff --git a/python/mozbuild/mozbuild/dotproperties.py b/python/mozbuild/mozbuild/dotproperties.py index 972ff23299d0..40fd5bbc6153 100644 --- a/python/mozbuild/mozbuild/dotproperties.py +++ b/python/mozbuild/mozbuild/dotproperties.py @@ -16,6 +16,7 @@ if sys.version_info[0] == 3: else: str_type = basestring + class DotProperties: r'''A thin representation of a key=value .properties file.''' diff --git a/python/mozbuild/mozbuild/faster_daemon.py b/python/mozbuild/mozbuild/faster_daemon.py index ea9a32340220..c9d3a4346292 100644 --- a/python/mozbuild/mozbuild/faster_daemon.py +++ b/python/mozbuild/mozbuild/faster_daemon.py @@ -108,8 +108,8 @@ class Daemon(object): ['name', '.hg', 'wholename'], ['dirname', '.git'], ['name', '.git', 'wholename'], + ], ], - ], ], 'fields': ['name'], } diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py index 2e3242937616..0f8e3c7eb689 100644 --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -89,6 +89,7 @@ class Context(KeyedDefaultDict): config is the ConfigEnvironment for this context. """ + def __init__(self, allowed_variables={}, config=None, finder=None): self._allowed_variables = allowed_variables self.main_path = None @@ -163,7 +164,7 @@ class Context(KeyedDefaultDict): @memoize def _srcdir(self, path): return mozpath.join(self.config.topsrcdir, - self._relsrcdir(path)).rstrip('/') + self._relsrcdir(path)).rstrip('/') @property def srcdir(self): @@ -182,7 +183,7 @@ class Context(KeyedDefaultDict): def relobjdir(self): assert self.main_path return mozpath.relpath(mozpath.dirname(self.main_path), - self.config.topsrcdir) + self.config.topsrcdir) def _factory(self, key): """Function called when requesting a missing key.""" @@ -269,6 +270,7 @@ class SubContext(Context, ContextDerivedValue): Sub-contexts inherit paths and other relevant state from the parent context. """ + def __init__(self, parent): assert isinstance(parent, Context) @@ -570,6 +572,7 @@ class PathMeta(type): cls = SourcePath return super(PathMeta, cls).__call__(context, value) + class Path(ContextDerivedValue, unicode): """Stores and resolves a source path relative to a given context @@ -636,6 +639,7 @@ class Path(ContextDerivedValue, unicode): class SourcePath(Path): """Like Path, but limited to paths in the source directory.""" + def __init__(self, context, value): if value.startswith('!'): raise ValueError('Object directory paths are not allowed') @@ -676,6 +680,7 @@ class RenamedSourcePath(SourcePath): This class is not meant to be exposed to moz.build sandboxes as of now, and is not supported by the RecursiveMake backend. """ + def __init__(self, context, value): assert isinstance(value, tuple) source, self._target_basename = value @@ -688,13 +693,14 @@ class RenamedSourcePath(SourcePath): class ObjDirPath(Path): """Like Path, but limited to paths in the object directory.""" + def __init__(self, context, value=None): if not value.startswith('!'): raise ValueError('Object directory paths must start with ! prefix') super(ObjDirPath, self).__init__(context, value) if value.startswith('!/'): - path = mozpath.join(context.config.topobjdir,value[2:]) + path = mozpath.join(context.config.topobjdir, value[2:]) else: path = mozpath.join(context.objdir, value[1:]) self.full_path = mozpath.normpath(path) @@ -702,6 +708,7 @@ class ObjDirPath(Path): class AbsolutePath(Path): """Like Path, but allows arbitrary paths outside the source and object directories.""" + def __init__(self, context, value=None): if not value.startswith('%'): raise ValueError('Absolute paths must start with % prefix') @@ -717,6 +724,7 @@ def ContextDerivedTypedList(klass, base_class=List): """Specialized TypedList for use with ContextDerivedValue types. """ assert issubclass(klass, ContextDerivedValue) + class _TypedList(ContextDerivedValue, TypedList(klass, base_class)): def __init__(self, context, iterable=[], **kwargs): self.context = context @@ -729,6 +737,7 @@ def ContextDerivedTypedList(klass, base_class=List): return _TypedList + @memoize def ContextDerivedTypedListWithItems(type, base_class=List): """Specialized TypedList for use with ContextDerivedValue types. @@ -862,6 +871,7 @@ def ContextDerivedTypedHierarchicalStringList(type): return _TypedListWithItems + def OrderedPathListWithAction(action): """Returns a class which behaves as a StrictOrderingOnAppendList, but invokes the given callable with each input and a context as it is @@ -871,7 +881,7 @@ def OrderedPathListWithAction(action): filesystem-reading mode. """ class _OrderedListWithAction(ContextDerivedTypedList(SourcePath, - StrictOrderingOnAppendListWithAction)): + StrictOrderingOnAppendListWithAction)): def __init__(self, context, *args): def _action(item): return item, action(context, item) @@ -879,6 +889,7 @@ def OrderedPathListWithAction(action): return _OrderedListWithAction + def TypedListWithAction(typ, action): """Returns a class which behaves as a TypedList with the provided type, but invokes the given given callable with each input and a context as it is @@ -894,6 +905,7 @@ def TypedListWithAction(typ, action): super(_TypedListWithAction, self).__init__(action=_action, *args) return _TypedListWithAction + ManifestparserManifestList = OrderedPathListWithAction(read_manifestparser_manifest) ReftestManifestList = OrderedPathListWithAction(read_reftest_manifest) @@ -905,7 +917,7 @@ DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList), ('tags', OrderedStringList), ('flavors', OrderedTestFlavorList)) BugzillaComponent = TypedNamedTuple('BugzillaComponent', - [('product', unicode), ('component', unicode)]) + [('product', unicode), ('component', unicode)]) SchedulingComponents = ContextDerivedTypedRecord( ('inclusive', TypedList(unicode, StrictOrderingOnAppendList)), ('exclusive', TypedList(unicode, StrictOrderingOnAppendList))) @@ -965,14 +977,14 @@ class Files(SubContext): VARIABLES = { 'BUG_COMPONENT': (BugzillaComponent, tuple, - """The bug component that tracks changes to these files. + """The bug component that tracks changes to these files. Values are a 2-tuple of unicode describing the Bugzilla product and component. e.g. ``('Firefox Build System', 'General')``. """), 'FINAL': (bool, bool, - """Mark variable assignments as finalized. + """Mark variable assignments as finalized. During normal processing, values from newer Files contexts overwrite previously set values. Last write wins. This behavior is @@ -986,7 +998,7 @@ class Files(SubContext): See :ref:`mozbuild_files_metadata_finalizing` for more info. """), 'IMPACTED_TESTS': (DependentTestsEntry, list, - """File patterns, tags, and flavors for tests relevant to these files. + """File patterns, tags, and flavors for tests relevant to these files. Maps source files to the tests potentially impacted by those files. Tests can be specified by file pattern, tag, or flavor. @@ -1034,7 +1046,7 @@ class Files(SubContext): any plain mochitest. """), 'SCHEDULES': (Schedules, list, - """Maps source files to the CI tasks that should be scheduled when + """Maps source files to the CI tasks that should be scheduled when they change. The tasks are grouped by named components, and those names appear again in the taskgraph configuration `($topsrcdir/taskgraph/). @@ -1190,33 +1202,33 @@ SUBCONTEXTS = {cls.__name__: cls for cls in SUBCONTEXTS} VARIABLES = { 'SOURCES': (ContextDerivedTypedListWithItems(Path, StrictOrderingOnAppendListWithFlagsFactory({'no_pgo': bool, 'flags': List, 'pgo_generate_only': bool})), list, - """Source code files. + """Source code files. This variable contains a list of source code files to compile. Accepts assembler, C, C++, Objective C/C++. """), 'FILES_PER_UNIFIED_FILE': (int, int, - """The number of source files to compile into each unified source file. + """The number of source files to compile into each unified source file. """), 'IS_RUST_LIBRARY': (bool, bool, - """Whether the current library defined by this moz.build is built by Rust. + """Whether the current library defined by this moz.build is built by Rust. The library defined by this moz.build should have a build definition in a Cargo.toml file that exists in this moz.build's directory. """), 'RUST_LIBRARY_FEATURES': (List, list, - """Cargo features to activate for this library. + """Cargo features to activate for this library. This variable should not be used directly; you should be using the RustLibrary template instead. """), 'RUST_LIBRARY_TARGET_DIR': (unicode, unicode, - """Where CARGO_TARGET_DIR should point when compiling this library. If + """Where CARGO_TARGET_DIR should point when compiling this library. If not set, it defaults to the current objdir. It should be a relative path to the current objdir; absolute paths should not be used. @@ -1225,22 +1237,22 @@ VARIABLES = { """), 'HOST_RUST_LIBRARY_FEATURES': (List, list, - """Cargo features to activate for this host library. + """Cargo features to activate for this host library. This variable should not be used directly; you should be using the HostRustLibrary template instead. """), 'RUST_TESTS': (TypedList(unicode), list, - """Names of Rust tests to build and run via `cargo test`. + """Names of Rust tests to build and run via `cargo test`. """), 'RUST_TEST_FEATURES': (TypedList(unicode), list, - """Cargo features to activate for RUST_TESTS. + """Cargo features to activate for RUST_TESTS. """), 'UNIFIED_SOURCES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, - """Source code files that can be compiled together. + """Source code files that can be compiled together. This variable contains a list of source code files to compile, that can be concatenated all together and built as a single source @@ -1249,7 +1261,7 @@ VARIABLES = { """), 'GENERATED_FILES': (GeneratedFilesList, list, - """Generic generated files. + """Generic generated files. This variable contains a list of files for the build system to generate at export time. The generation method may be declared @@ -1300,7 +1312,7 @@ VARIABLES = { """), 'DEFINES': (InitializedDefines, dict, - """Dictionary of compiler defines to declare. + """Dictionary of compiler defines to declare. These are passed in to the compiler as ``-Dkey='value'`` for string values, ``-Dkey=value`` for numeric values, or ``-Dkey`` if the @@ -1328,14 +1340,14 @@ VARIABLES = { """), 'DELAYLOAD_DLLS': (List, list, - """Delay-loaded DLLs. + """Delay-loaded DLLs. This variable contains a list of DLL files which the module being linked should load lazily. This only has an effect when building with MSVC. """), 'DIRS': (ContextDerivedTypedList(SourcePath), list, - """Child directories to descend into looking for build frontend files. + """Child directories to descend into looking for build frontend files. This works similarly to the ``DIRS`` variable in make files. Each str value in the list is the name of a child directory. When this file is @@ -1349,7 +1361,7 @@ VARIABLES = { """), 'HAS_MISC_RULE': (bool, bool, - """Whether this directory should be traversed in the ``misc`` tier. + """Whether this directory should be traversed in the ``misc`` tier. Many ``libs`` rules still exist in Makefile.in files. We highly prefer that these rules exist in the ``misc`` tier/target so that they can be @@ -1365,7 +1377,7 @@ VARIABLES = { """), 'FINAL_TARGET_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of files to be installed into the application directory. + """List of files to be installed into the application directory. ``FINAL_TARGET_FILES`` will copy (or symlink, if the platform supports it) the contents of its files to the directory specified by @@ -1380,11 +1392,11 @@ VARIABLES = { """), 'FINAL_TARGET_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """Like ``FINAL_TARGET_FILES``, with preprocessing. + """Like ``FINAL_TARGET_FILES``, with preprocessing. """), 'LOCALIZED_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of locale-dependent files to be installed into the application + """List of locale-dependent files to be installed into the application directory. This functions similarly to ``FINAL_TARGET_FILES``, but the files are @@ -1426,14 +1438,14 @@ VARIABLES = { """), 'LOCALIZED_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """Like ``LOCALIZED_FILES``, with preprocessing. + """Like ``LOCALIZED_FILES``, with preprocessing. Note that the ``AB_CD`` define is available and expands to the current locale being packaged, as with preprocessed entries in jar manifests. """), 'LOCALIZED_GENERATED_FILES': (GeneratedFilesList, list, - """Like ``GENERATED_FILES``, but for files whose content varies based on the locale in use. + """Like ``GENERATED_FILES``, but for files whose content varies based on the locale in use. For simple cases of text substitution, prefer ``LOCALIZED_PP_FILES``. @@ -1457,7 +1469,7 @@ VARIABLES = { """), 'OBJDIR_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of files to be installed anywhere in the objdir. Use sparingly. + """List of files to be installed anywhere in the objdir. Use sparingly. ``OBJDIR_FILES`` is similar to FINAL_TARGET_FILES, but it allows copying anywhere in the object directory. This is intended for various one-off @@ -1466,11 +1478,11 @@ VARIABLES = { """), 'OBJDIR_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """Like ``OBJDIR_FILES``, with preprocessing. Use sparingly. + """Like ``OBJDIR_FILES``, with preprocessing. Use sparingly. """), 'FINAL_LIBRARY': (unicode, unicode, - """Library in which the objects of the current directory will be linked. + """Library in which the objects of the current directory will be linked. This variable contains the name of a library, defined elsewhere with ``LIBRARY_NAME``, in which the objects of the current directory will be @@ -1478,7 +1490,7 @@ VARIABLES = { """), 'CPP_UNIT_TESTS': (StrictOrderingOnAppendList, list, - """Compile a list of C++ unit test names. + """Compile a list of C++ unit test names. Each name in this variable corresponds to an executable built from the corresponding source file with the same base name. @@ -1489,40 +1501,40 @@ VARIABLES = { """), 'FORCE_SHARED_LIB': (bool, bool, - """Whether the library in this directory is a shared library. + """Whether the library in this directory is a shared library. """), 'FORCE_STATIC_LIB': (bool, bool, - """Whether the library in this directory is a static library. + """Whether the library in this directory is a static library. """), 'USE_STATIC_LIBS': (bool, bool, - """Whether the code in this directory is a built against the static + """Whether the code in this directory is a built against the static runtime library. This variable only has an effect when building with MSVC. """), 'HOST_SOURCES': (ContextDerivedTypedList(Path, StrictOrderingOnAppendList), list, - """Source code files to compile with the host compiler. + """Source code files to compile with the host compiler. This variable contains a list of source code files to compile. with the host compiler. """), 'HOST_LIBRARY_NAME': (unicode, unicode, - """Name of target library generated when cross compiling. + """Name of target library generated when cross compiling. """), 'LIBRARY_DEFINES': (OrderedDict, dict, - """Dictionary of compiler defines to declare for the entire library. + """Dictionary of compiler defines to declare for the entire library. This variable works like DEFINES, except that declarations apply to all libraries that link into this library via FINAL_LIBRARY. """), 'LIBRARY_NAME': (unicode, unicode, - """The code name of the library generated for a directory. + """The code name of the library generated for a directory. By default STATIC_LIBRARY_NAME and SHARED_LIBRARY_NAME take this name. In ``example/components/moz.build``,:: @@ -1534,89 +1546,89 @@ VARIABLES = { """), 'SHARED_LIBRARY_NAME': (unicode, unicode, - """The name of the static library generated for a directory, if it needs to + """The name of the static library generated for a directory, if it needs to differ from the library code name. Implies FORCE_SHARED_LIB. """), 'SHARED_LIBRARY_OUTPUT_CATEGORY': (unicode, unicode, - """The output category for this context's shared library. If set this will + """The output category for this context's shared library. If set this will correspond to the build command that will build this shared library, and the library will not be built as part of the default build. """), 'RUST_LIBRARY_OUTPUT_CATEGORY': (unicode, unicode, - """The output category for this context's rust library. If set this will + """The output category for this context's rust library. If set this will correspond to the build command that will build this rust library, and the library will not be built as part of the default build. """), 'IS_FRAMEWORK': (bool, bool, - """Whether the library to build should be built as a framework on OSX. + """Whether the library to build should be built as a framework on OSX. This implies the name of the library won't be prefixed nor suffixed. Implies FORCE_SHARED_LIB. """), 'STATIC_LIBRARY_NAME': (unicode, unicode, - """The name of the static library generated for a directory, if it needs to + """The name of the static library generated for a directory, if it needs to differ from the library code name. Implies FORCE_STATIC_LIB. """), 'USE_LIBS': (StrictOrderingOnAppendList, list, - """List of libraries to link to programs and libraries. + """List of libraries to link to programs and libraries. """), 'HOST_USE_LIBS': (StrictOrderingOnAppendList, list, - """List of libraries to link to host programs and libraries. + """List of libraries to link to host programs and libraries. """), 'HOST_OS_LIBS': (List, list, - """List of system libraries for host programs and libraries. + """List of system libraries for host programs and libraries. """), 'LOCAL_INCLUDES': (ContextDerivedTypedList(Path, StrictOrderingOnAppendList), list, - """Additional directories to be searched for include files by the compiler. + """Additional directories to be searched for include files by the compiler. """), 'NO_PGO': (bool, bool, - """Whether profile-guided optimization is disable in this directory. + """Whether profile-guided optimization is disable in this directory. """), 'OS_LIBS': (List, list, - """System link libraries. + """System link libraries. This variable contains a list of system libaries to link against. """), 'RCFILE': (unicode, unicode, - """The program .rc file. + """The program .rc file. This variable can only be used on Windows. """), 'RESFILE': (unicode, unicode, - """The program .res file. + """The program .res file. This variable can only be used on Windows. """), 'RCINCLUDE': (unicode, unicode, - """The resource script file to be included in the default .res file. + """The resource script file to be included in the default .res file. This variable can only be used on Windows. """), 'DEFFILE': (Path, unicode, - """The program .def (module definition) file. + """The program .def (module definition) file. This variable can only be used on Windows. """), 'SYMBOLS_FILE': (Path, unicode, - """A file containing a list of symbols to export from a shared library. + """A file containing a list of symbols to export from a shared library. The given file contains a list of symbols to be exported, and is preprocessed. @@ -1626,7 +1638,7 @@ VARIABLES = { """), 'SIMPLE_PROGRAMS': (StrictOrderingOnAppendList, list, - """Compile a list of executable names. + """Compile a list of executable names. Each name in this variable corresponds to an executable built from the corresponding source file with the same base name. @@ -1637,7 +1649,7 @@ VARIABLES = { """), 'SONAME': (unicode, unicode, - """The soname of the shared object currently being linked + """The soname of the shared object currently being linked soname is the "logical name" of a shared object, often used to provide version backwards compatibility. This variable makes sense only for @@ -1645,7 +1657,7 @@ VARIABLES = { """), 'HOST_SIMPLE_PROGRAMS': (StrictOrderingOnAppendList, list, - """Compile a list of host executable names. + """Compile a list of host executable names. Each name in this variable corresponds to a hosst executable built from the corresponding source file with the same base name. @@ -1656,21 +1668,21 @@ VARIABLES = { """), 'RUST_PROGRAMS': (StrictOrderingOnAppendList, list, - """Compile a list of Rust host executable names. + """Compile a list of Rust host executable names. Each name in this variable corresponds to an executable built from the Cargo.toml in the same directory. """), 'HOST_RUST_PROGRAMS': (StrictOrderingOnAppendList, list, - """Compile a list of Rust executable names. + """Compile a list of Rust executable names. Each name in this variable corresponds to an executable built from the Cargo.toml in the same directory. """), 'CONFIGURE_SUBST_FILES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, - """Output files that will be generated using configure-like substitution. + """Output files that will be generated using configure-like substitution. This is a substitute for ``AC_OUTPUT`` in autoconf. For each path in this list, we will search for a file in the srcdir having the name @@ -1680,7 +1692,7 @@ VARIABLES = { """), 'CONFIGURE_DEFINE_FILES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, - """Output files generated from configure/config.status. + """Output files generated from configure/config.status. This is a substitute for ``AC_CONFIG_HEADER`` in autoconf. This is very similar to ``CONFIGURE_SUBST_FILES`` except the generation logic takes @@ -1688,7 +1700,7 @@ VARIABLES = { """), 'EXPORTS': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of files to be exported, and in which subdirectories. + """List of files to be exported, and in which subdirectories. ``EXPORTS`` is generally used to list the include files to be exported to ``dist/include``, but it can be used for other files as well. This variable @@ -1706,16 +1718,16 @@ VARIABLES = { ``GENERATED_FILES``. """), - 'PROGRAM' : (unicode, unicode, - """Compiled executable name. + 'PROGRAM': (unicode, unicode, + """Compiled executable name. If the configuration token ``BIN_SUFFIX`` is set, its value will be automatically appended to ``PROGRAM``. If ``PROGRAM`` already ends with ``BIN_SUFFIX``, ``PROGRAM`` will remain unchanged. """), - 'HOST_PROGRAM' : (unicode, unicode, - """Compiled host executable name. + 'HOST_PROGRAM': (unicode, unicode, + """Compiled host executable name. If the configuration token ``HOST_BIN_SUFFIX`` is set, its value will be automatically appended to ``HOST_PROGRAM``. If ``HOST_PROGRAM`` already @@ -1723,7 +1735,7 @@ VARIABLES = { """), 'DIST_INSTALL': (Enum(None, False, True), bool, - """Whether to install certain files into the dist directory. + """Whether to install certain files into the dist directory. By default, some files types are installed in the dist directory, and some aren't. Set this variable to True to force the installation of @@ -1736,7 +1748,7 @@ VARIABLES = { """), 'JAR_MANIFESTS': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, - """JAR manifest files that should be processed as part of the build. + """JAR manifest files that should be processed as part of the build. JAR manifests are files in the tree that define how to package files into JARs and how chrome registration is performed. For more info, @@ -1745,7 +1757,7 @@ VARIABLES = { # IDL Generation. 'XPIDL_SOURCES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, - """XPCOM Interface Definition Files (xpidl). + """XPCOM Interface Definition Files (xpidl). This is a list of files that define XPCOM interface definitions. Entries must be files that exist. Entries are almost certainly ``.idl`` @@ -1753,7 +1765,7 @@ VARIABLES = { """), 'XPIDL_MODULE': (unicode, unicode, - """XPCOM Interface Definition Module Name. + """XPCOM Interface Definition Module Name. This is the name of the ``.xpt`` file that is created by linking ``XPIDL_SOURCES`` together. If unspecified, it defaults to be the same @@ -1761,66 +1773,66 @@ VARIABLES = { """), 'XPCOM_MANIFESTS': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, - """XPCOM Component Manifest Files. + """XPCOM Component Manifest Files. This is a list of files that define XPCOM components to be added to the component registry. """), 'PREPROCESSED_IPDL_SOURCES': (StrictOrderingOnAppendList, list, - """Preprocessed IPDL source files. + """Preprocessed IPDL source files. These files will be preprocessed, then parsed and converted to ``.cpp`` files. """), 'IPDL_SOURCES': (StrictOrderingOnAppendList, list, - """IPDL source files. + """IPDL source files. These are ``.ipdl`` files that will be parsed and converted to ``.cpp`` files. """), 'WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """WebIDL source files. + """WebIDL source files. These will be parsed and converted to ``.cpp`` and ``.h`` files. """), 'GENERATED_EVENTS_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """WebIDL source files for generated events. + """WebIDL source files for generated events. These will be parsed and converted to ``.cpp`` and ``.h`` files. """), 'TEST_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """Test WebIDL source files. + """Test WebIDL source files. These will be parsed and converted to ``.cpp`` and ``.h`` files if tests are enabled. """), 'GENERATED_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """Generated WebIDL source files. + """Generated WebIDL source files. These will be generated from some other files. """), 'PREPROCESSED_TEST_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """Preprocessed test WebIDL source files. + """Preprocessed test WebIDL source files. These will be preprocessed, then parsed and converted to .cpp and ``.h`` files if tests are enabled. """), 'PREPROCESSED_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """Preprocessed WebIDL source files. + """Preprocessed WebIDL source files. These will be preprocessed before being parsed and converted. """), 'WEBIDL_EXAMPLE_INTERFACES': (StrictOrderingOnAppendList, list, - """Names of example WebIDL interfaces to build as part of the build. + """Names of example WebIDL interfaces to build as part of the build. Names in this list correspond to WebIDL interface names defined in WebIDL files included in the build from one of the \*WEBIDL_FILES @@ -1829,96 +1841,96 @@ VARIABLES = { # Test declaration. 'A11Y_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining a11y tests. + """List of manifest files defining a11y tests. """), 'BROWSER_CHROME_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining browser chrome tests. + """List of manifest files defining browser chrome tests. """), 'ANDROID_INSTRUMENTATION_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining Android instrumentation tests. + """List of manifest files defining Android instrumentation tests. """), 'FIREFOX_UI_FUNCTIONAL_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining firefox-ui-functional tests. + """List of manifest files defining firefox-ui-functional tests. """), 'FIREFOX_UI_UPDATE_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining firefox-ui-update tests. + """List of manifest files defining firefox-ui-update tests. """), 'PUPPETEER_FIREFOX_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining puppeteer unit tests for Firefox. + """List of manifest files defining puppeteer unit tests for Firefox. """), 'MARIONETTE_LAYOUT_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining marionette-layout tests. + """List of manifest files defining marionette-layout tests. """), 'MARIONETTE_GPU_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining marionette-gpu tests. + """List of manifest files defining marionette-gpu tests. """), 'MARIONETTE_UNIT_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining marionette-unit tests. + """List of manifest files defining marionette-unit tests. """), 'METRO_CHROME_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining metro browser chrome tests. + """List of manifest files defining metro browser chrome tests. """), 'MOCHITEST_CHROME_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining mochitest chrome tests. + """List of manifest files defining mochitest chrome tests. """), 'MARIONETTE_DOM_MEDIA_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining marionette-media tests. + """List of manifest files defining marionette-media tests. """), 'MOCHITEST_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining mochitest tests. + """List of manifest files defining mochitest tests. """), 'REFTEST_MANIFESTS': (ReftestManifestList, list, - """List of manifest files defining reftests. + """List of manifest files defining reftests. These are commonly named reftest.list. """), 'CRASHTEST_MANIFESTS': (ReftestManifestList, list, - """List of manifest files defining crashtests. + """List of manifest files defining crashtests. These are commonly named crashtests.list. """), 'WEBRTC_SIGNALLING_TEST_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining WebRTC signalling tests. + """List of manifest files defining WebRTC signalling tests. """), 'XPCSHELL_TESTS_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining xpcshell tests. + """List of manifest files defining xpcshell tests. """), 'PYTHON_UNITTEST_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining python unit tests. + """List of manifest files defining python unit tests. """), 'CRAMTEST_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining cram unit tests. + """List of manifest files defining cram unit tests. """), # The following variables are used to control the target of installed files. 'XPI_NAME': (unicode, unicode, - """The name of an extension XPI to generate. + """The name of an extension XPI to generate. When this variable is present, the results of this directory will end up being packaged into an extension instead of the main dist/bin results. """), 'DIST_SUBDIR': (unicode, unicode, - """The name of an alternate directory to install files to. + """The name of an alternate directory to install files to. When this variable is present, the results of this directory will end up being placed in the $(DIST_SUBDIR) subdirectory of where it would @@ -1926,7 +1938,7 @@ VARIABLES = { """), 'FINAL_TARGET': (FinalTargetValue, unicode, - """The name of the directory to install targets to. + """The name of the directory to install targets to. The directory is relative to the top of the object directory. The default value is dependent on the values of XPI_NAME and DIST_SUBDIR. If @@ -1936,7 +1948,7 @@ VARIABLES = { """), 'USE_EXTENSION_MANIFEST': (bool, bool, - """Controls the name of the manifest for JAR files. + """Controls the name of the manifest for JAR files. By default, the name of the manifest is ${JAR_MANIFEST}.manifest. Setting this variable to ``True`` changes the name of the manifest to @@ -2006,7 +2018,7 @@ VARIABLES = { """), 'SPHINX_TREES': (dict, dict, - """Describes what the Sphinx documentation tree will look like. + """Describes what the Sphinx documentation tree will look like. Keys are relative directories inside the final Sphinx documentation tree to install files into. Values are directories (relative to this @@ -2014,26 +2026,26 @@ VARIABLES = { """), 'SPHINX_PYTHON_PACKAGE_DIRS': (StrictOrderingOnAppendList, list, - """Directories containing Python packages that Sphinx documents. + """Directories containing Python packages that Sphinx documents. """), 'COMPILE_FLAGS': (CompileFlags, dict, - """Recipe for compile flags for this context. Not to be manipulated + """Recipe for compile flags for this context. Not to be manipulated directly. """), 'LINK_FLAGS': (LinkFlags, dict, - """Recipe for linker flags for this context. Not to be manipulated + """Recipe for linker flags for this context. Not to be manipulated directly. """), 'ASM_FLAGS': (AsmFlags, dict, - """Recipe for linker flags for this context. Not to be manipulated + """Recipe for linker flags for this context. Not to be manipulated directly. """), 'CFLAGS': (List, list, - """Flags passed to the C compiler for all of the C source files + """Flags passed to the C compiler for all of the C source files declared in this directory. Note that the ordering of flags matters here, these flags will be @@ -2042,7 +2054,7 @@ VARIABLES = { """), 'CXXFLAGS': (List, list, - """Flags passed to the C++ compiler for all of the C++ source files + """Flags passed to the C++ compiler for all of the C++ source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2051,17 +2063,17 @@ VARIABLES = { """), 'HOST_COMPILE_FLAGS': (HostCompileFlags, dict, - """Recipe for host compile flags for this context. Not to be manipulated + """Recipe for host compile flags for this context. Not to be manipulated directly. """), 'HOST_DEFINES': (InitializedDefines, dict, - """Dictionary of compiler defines to declare for host compilation. + """Dictionary of compiler defines to declare for host compilation. See ``DEFINES`` for specifics. """), 'CMFLAGS': (List, list, - """Flags passed to the Objective-C compiler for all of the Objective-C + """Flags passed to the Objective-C compiler for all of the Objective-C source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2070,7 +2082,7 @@ VARIABLES = { """), 'CMMFLAGS': (List, list, - """Flags passed to the Objective-C++ compiler for all of the + """Flags passed to the Objective-C++ compiler for all of the Objective-C++ source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2079,7 +2091,7 @@ VARIABLES = { """), 'ASFLAGS': (List, list, - """Flags passed to the assembler for all of the assembly source files + """Flags passed to the assembler for all of the assembly source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2088,7 +2100,7 @@ VARIABLES = { """), 'HOST_CFLAGS': (List, list, - """Flags passed to the host C compiler for all of the C source files + """Flags passed to the host C compiler for all of the C source files declared in this directory. Note that the ordering of flags matters here, these flags will be @@ -2097,7 +2109,7 @@ VARIABLES = { """), 'HOST_CXXFLAGS': (List, list, - """Flags passed to the host C++ compiler for all of the C++ source files + """Flags passed to the host C++ compiler for all of the C++ source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2106,7 +2118,7 @@ VARIABLES = { """), 'LDFLAGS': (List, list, - """Flags passed to the linker when linking all of the libraries and + """Flags passed to the linker when linking all of the libraries and executables declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2115,7 +2127,7 @@ VARIABLES = { """), 'EXTRA_DSO_LDOPTS': (List, list, - """Flags passed to the linker when linking a shared library. + """Flags passed to the linker when linking a shared library. Note that the ordering of flags matter here, these flags will be added to the linker's command line in the same order as they @@ -2123,7 +2135,7 @@ VARIABLES = { """), 'WIN32_EXE_LDFLAGS': (List, list, - """Flags passed to the linker when linking a Windows .exe executable + """Flags passed to the linker when linking a Windows .exe executable declared in this directory. Note that the ordering of flags matter here, these flags will be @@ -2134,7 +2146,7 @@ VARIABLES = { """), 'TEST_HARNESS_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of files to be installed for test harnesses. + """List of files to be installed for test harnesses. ``TEST_HARNESS_FILES`` can be used to install files to any directory under $objdir/_tests. Files can be appended to a field to indicate @@ -2149,17 +2161,17 @@ VARIABLES = { """), 'NO_EXPAND_LIBS': (bool, bool, - """Forces to build a real static library, and no corresponding fake + """Forces to build a real static library, and no corresponding fake library. """), 'NO_COMPONENTS_MANIFEST': (bool, bool, - """Do not create a binary-component manifest entry for the + """Do not create a binary-component manifest entry for the corresponding XPCOMBinaryComponent. """), 'USE_NASM': (bool, bool, - """Use the nasm assembler to assemble assembly files from SOURCES. + """Use the nasm assembler to assemble assembly files from SOURCES. By default, the build will use the toolchain assembler, $(AS), to assemble source files in assembly language (.s or .asm files). Setting @@ -2170,7 +2182,7 @@ VARIABLES = { """), 'USE_YASM': (bool, bool, - """Use the yasm assembler to assemble assembly files from SOURCES. + """Use the yasm assembler to assemble assembly files from SOURCES. By default, the build will use the toolchain assembler, $(AS), to assemble source files in assembly language (.s or .asm files). Setting @@ -2181,7 +2193,7 @@ VARIABLES = { """), 'USE_INTEGRATED_CLANGCL_AS': (bool, bool, - """Use the integrated clang-cl assembler to assemble assembly files from SOURCES. + """Use the integrated clang-cl assembler to assemble assembly files from SOURCES. This allows using clang-cl to assemble assembly files which is useful on platforms like aarch64 where the alternative is to have to run a @@ -2193,7 +2205,7 @@ VARIABLES = { for name, (storage_type, input_types, docs) in VARIABLES.items(): if storage_type == list: raise RuntimeError('%s has a "list" storage type. Use "List" instead.' - % name) + % name) # Set of variables that are only allowed in templates: TEMPLATE_VARIABLES = { @@ -2212,7 +2224,7 @@ TEMPLATE_VARIABLES = { for name in TEMPLATE_VARIABLES: if name not in VARIABLES: raise RuntimeError('%s is in TEMPLATE_VARIABLES but not in VARIABLES.' - % name) + % name) storage_type, input_types, docs = VARIABLES[name] docs += 'This variable is only available in templates.\n' VARIABLES[name] = (storage_type, input_types, docs) @@ -2229,7 +2241,7 @@ for name in TEMPLATE_VARIABLES: # FUNCTIONS = { 'include': (lambda self: self._include, (SourcePath,), - """Include another mozbuild file in the context of this one. + """Include another mozbuild file in the context of this one. This is similar to a ``#include`` in C languages. The filename passed to the function will be read and its contents will be evaluated within the @@ -2256,7 +2268,7 @@ FUNCTIONS = { """), 'export': (lambda self: self._export, (str,), - """Make the specified variable available to all child directories. + """Make the specified variable available to all child directories. The variable specified by the argument string is added to the environment of all directories specified in the DIRS and TEST_DIRS @@ -2283,7 +2295,7 @@ FUNCTIONS = { """), 'warning': (lambda self: self._warning, (str,), - """Issue a warning. + """Issue a warning. Warnings are string messages that are printed during execution. @@ -2291,13 +2303,13 @@ FUNCTIONS = { """), 'error': (lambda self: self._error, (str,), - """Issue a fatal error. + """Issue a fatal error. If this function is called, processing is aborted immediately. """), 'template': (lambda self: self._template_decorator, (FunctionType,), - """Decorator for template declarations. + """Decorator for template declarations. Templates are a special kind of functions that can be declared in mozbuild files. Uppercase variables assigned in the function scope @@ -2356,7 +2368,7 @@ TestDirsPlaceHolder = List() # SPECIAL_VARIABLES = { 'TOPSRCDIR': (lambda context: context.config.topsrcdir, str, - """Constant defining the top source directory. + """Constant defining the top source directory. The top source directory is the parent directory containing the source code and all build files. It is typically the root directory of a @@ -2364,7 +2376,7 @@ SPECIAL_VARIABLES = { """), 'TOPOBJDIR': (lambda context: context.config.topobjdir, str, - """Constant defining the top object directory. + """Constant defining the top object directory. The top object directory is the parent directory which will contain the output of the build. This is commonly referred to as "the object @@ -2372,7 +2384,7 @@ SPECIAL_VARIABLES = { """), 'RELATIVEDIR': (lambda context: context.relsrcdir, str, - """Constant defining the relative path of this file. + """Constant defining the relative path of this file. The relative path is from ``TOPSRCDIR``. This is defined as relative to the main file being executed, regardless of whether additional @@ -2380,14 +2392,14 @@ SPECIAL_VARIABLES = { """), 'SRCDIR': (lambda context: context.srcdir, str, - """Constant defining the source directory of this file. + """Constant defining the source directory of this file. This is the path inside ``TOPSRCDIR`` where this file is located. It is the same as ``TOPSRCDIR + RELATIVEDIR``. """), 'OBJDIR': (lambda context: context.objdir, str, - """The path to the object directory for this file. + """The path to the object directory for this file. Is is the same as ``TOPOBJDIR + RELATIVEDIR``. """), @@ -2406,32 +2418,32 @@ SPECIAL_VARIABLES = { """), 'EXTRA_COMPONENTS': (lambda context: context['FINAL_TARGET_FILES'].components._strings, list, - """Additional component files to distribute. + """Additional component files to distribute. This variable contains a list of files to copy into ``$(FINAL_TARGET)/components/``. """), 'EXTRA_PP_COMPONENTS': (lambda context: context['FINAL_TARGET_PP_FILES'].components._strings, list, - """Javascript XPCOM files. + """Javascript XPCOM files. This variable contains a list of files to preprocess. Generated files will be installed in the ``/components`` directory of the distribution. """), 'JS_PREFERENCE_FILES': (lambda context: context['FINAL_TARGET_FILES'].defaults.pref._strings, list, - """Exported JavaScript files. + """Exported JavaScript files. A list of files copied into the dist directory for packaging and installation. Path will be defined for gre or application prefs dir based on what is building. """), 'JS_PREFERENCE_PP_FILES': (lambda context: context['FINAL_TARGET_PP_FILES'].defaults.pref._strings, list, - """Like JS_PREFERENCE_FILES, preprocessed.. + """Like JS_PREFERENCE_FILES, preprocessed.. """), 'RESOURCE_FILES': (lambda context: context['FINAL_TARGET_FILES'].res, list, - """List of resources to be exported, and in which subdirectories. + """List of resources to be exported, and in which subdirectories. ``RESOURCE_FILES`` is used to list the resource files to be exported to ``dist/bin/res``, but it can be used for other files as well. This variable @@ -2446,7 +2458,7 @@ SPECIAL_VARIABLES = { """), 'CONTENT_ACCESSIBLE_FILES': (lambda context: context['FINAL_TARGET_FILES'].contentaccessible, list, - """List of files which can be accessed by web content through resource:// URIs. + """List of files which can be accessed by web content through resource:// URIs. ``CONTENT_ACCESSIBLE_FILES`` is used to list the files to be exported to ``dist/bin/contentaccessible``. Files can also be appended to a @@ -2454,21 +2466,21 @@ SPECIAL_VARIABLES = { """), 'EXTRA_JS_MODULES': (lambda context: context['FINAL_TARGET_FILES'].modules, list, - """Additional JavaScript files to distribute. + """Additional JavaScript files to distribute. This variable contains a list of files to copy into ``$(FINAL_TARGET)/modules. """), 'EXTRA_PP_JS_MODULES': (lambda context: context['FINAL_TARGET_PP_FILES'].modules, list, - """Additional JavaScript files to distribute. + """Additional JavaScript files to distribute. This variable contains a list of files to copy into ``$(FINAL_TARGET)/modules``, after preprocessing. """), 'TESTING_JS_MODULES': (lambda context: context['TEST_HARNESS_FILES'].modules, list, - """JavaScript modules to install in the test-only destination. + """JavaScript modules to install in the test-only destination. Some JavaScript modules (JSMs) are test-only and not distributed with Firefox. This variable defines them. @@ -2480,8 +2492,8 @@ SPECIAL_VARIABLES = { """), 'TEST_DIRS': (lambda context: context['DIRS'] if context.config.substs.get('ENABLE_TESTS') - else TestDirsPlaceHolder, list, - """Like DIRS but only for directories that contain test-only code. + else TestDirsPlaceHolder, list, + """Like DIRS but only for directories that contain test-only code. If tests are not enabled, this variable will be ignored. diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index ff317b02b978..1f2db7f29fae 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -192,6 +192,7 @@ class ComputedFlags(ContextDerived): flags[dest_var].extend(value) return flags.items() + class XPIDLModule(ContextDerived): """Describes an XPIDL module to be compiled.""" @@ -207,6 +208,7 @@ class XPIDLModule(ContextDerived): self.name = name self.idl_files = idl_files + class BaseDefines(ContextDerived): """Context derived container object for DEFINES/HOST_DEFINES, which are OrderedDicts. @@ -232,12 +234,15 @@ class BaseDefines(ContextDerived): else: self.defines.update(more_defines) + class Defines(BaseDefines): pass + class HostDefines(BaseDefines): pass + class WebIDLCollection(ContextDerived): """Collects WebIDL info referenced during the build.""" @@ -676,7 +681,7 @@ class StaticLibrary(Library): ) def __init__(self, context, basename, real_name=None, - link_into=None, no_expand_lib=False): + link_into=None, no_expand_lib=False): Library.__init__(self, context, basename, real_name) self.link_into = link_into self.no_expand_lib = no_expand_lib @@ -708,8 +713,8 @@ class RustLibrary(StaticLibrary): # many other things in the build system depend on that. assert self.crate_type == 'staticlib' self.lib_name = '%s%s%s' % (context.config.rust_lib_prefix, - basename.replace('-', '_'), - context.config.rust_lib_suffix) + basename.replace('-', '_'), + context.config.rust_lib_suffix) self.dependencies = dependencies self.features = features self.target_dir = target_dir @@ -892,8 +897,8 @@ class TestManifest(ContextDerived): ) def __init__(self, context, path, manifest, flavor=None, - install_prefix=None, relpath=None, sources=(), - dupe_manifest=False): + install_prefix=None, relpath=None, sources=(), + dupe_manifest=False): ContextDerived.__init__(self, context) assert flavor in all_test_flavors() @@ -1036,7 +1041,7 @@ class UnifiedSources(BaseSources): unified_prefix = unified_prefix.replace('/', '_') suffix = self.canonical_suffix[1:] - unified_prefix='Unified_%s_%s' % (suffix, unified_prefix) + unified_prefix = 'Unified_%s_%s' % (suffix, unified_prefix) self.unified_source_mapping = list(group_unified_files(source_files, unified_prefix=unified_prefix, unified_suffix=suffix, @@ -1100,6 +1105,7 @@ class FinalTargetPreprocessedFiles(ContextDerived): ContextDerived.__init__(self, sandbox) self.files = files + class LocalizedFiles(FinalTargetFiles): """Sandbox container object for LOCALIZED_FILES, which is a HierarchicalStringList. @@ -1187,12 +1193,14 @@ class GeneratedFile(ContextDerived): '.inc', '.py', '.rs', - 'node.stub', # To avoid VPATH issues with installing node files: https://bugzilla.mozilla.org/show_bug.cgi?id=1461714#c55 - 'android_apks', # We need to compile Java to generate JNI wrappers for native code compilation to consume. + 'node.stub', # To avoid VPATH issues with installing node files: https://bugzilla.mozilla.org/show_bug.cgi?id=1461714#c55 + # We need to compile Java to generate JNI wrappers for native code compilation to consume. + 'android_apks', '.profdata', '.webidl' ) - self.required_for_compile = [f for f in self.outputs if f.endswith(suffixes) or 'stl_wrappers/' in f] + self.required_for_compile = [ + f for f in self.outputs if f.endswith(suffixes) or 'stl_wrappers/' in f] class ChromeManifestEntry(ContextDerived): diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index bafe3c76f113..183656d75c01 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -185,7 +185,8 @@ class TreeMetadataEmitter(LoggingMixin): objs = list(emitfn(out)) self._emitter_time += time.time() - start - for o in emit_objs(objs): yield o + for o in emit_objs(objs): + yield o else: raise Exception('Unhandled output type: %s' % type(out)) @@ -196,7 +197,8 @@ class TreeMetadataEmitter(LoggingMixin): objs = list(self._emit_libs_derived(contexts)) self._emitter_time += time.time() - start - for o in emit_objs(objs): yield o + for o in emit_objs(objs): + yield o def _emit_libs_derived(self, contexts): @@ -234,11 +236,11 @@ class TreeMetadataEmitter(LoggingMixin): if isinstance(collection, WebIDLCollection): # Test webidl sources are added here as a somewhat special # case. - idl_sources[mozpath.join(root, 'test')] = [s for s in collection.all_test_cpp_basenames()] + idl_sources[mozpath.join(root, 'test')] = [ + s for s in collection.all_test_cpp_basenames()] yield collection - # Next do FINAL_LIBRARY linkage. for lib in (l for libs in self._libs.values() for l in libs): if not isinstance(lib, (StaticLibrary, RustLibrary)) or not lib.link_into: @@ -255,14 +257,14 @@ class TreeMetadataEmitter(LoggingMixin): # both a static and a shared library in a directory, and having # that as a FINAL_LIBRARY. if len(set(type(l) for l in candidates)) == len(candidates) and \ - len(set(l.objdir for l in candidates)) == 1: + len(set(l.objdir for l in candidates)) == 1: for c in candidates: c.link_library(lib) else: raise SandboxValidationError( 'FINAL_LIBRARY ("%s") matches a LIBRARY_NAME defined in ' 'multiple places:\n %s' % (lib.link_into, - '\n '.join(l.objdir for l in candidates)), + '\n '.join(l.objdir for l in candidates)), contexts[lib.objdir]) # ...and USE_LIBS linkage. @@ -281,13 +283,13 @@ class TreeMetadataEmitter(LoggingMixin): for lib in self._static_linking_shared: if all(isinstance(o, StaticLibrary) for o in recurse_refs(lib)): shared_libs = sorted(l.basename for l in lib.linked_libraries - if isinstance(l, SharedLibrary)) + if isinstance(l, SharedLibrary)) raise SandboxValidationError( 'The static "%s" library is not used in a shared library ' 'or a program, but USE_LIBS contains the following shared ' 'library names:\n %s\n\nMaybe you can remove the ' 'static "%s" library?' % (lib.basename, - '\n '.join(shared_libs), lib.basename), + '\n '.join(shared_libs), lib.basename), contexts[lib.objdir]) # Propagate LIBRARY_DEFINES to all child libraries recursively. @@ -305,7 +307,6 @@ class TreeMetadataEmitter(LoggingMixin): propagate_defines(lib, lib.lib_defines) yield lib - for lib in (l for libs in self._libs.values() for l in libs): lib_defines = list(lib.lib_defines.get_defines()) if lib_defines: @@ -325,7 +326,6 @@ class TreeMetadataEmitter(LoggingMixin): for obj in self._binaries.values(): yield obj - LIBRARY_NAME_VAR = { 'host': 'HOST_LIBRARY_NAME', 'target': 'LIBRARY_NAME', @@ -393,14 +393,14 @@ class TreeMetadataEmitter(LoggingMixin): for d in self._external_paths: if dir.startswith('%s/' % d): candidates = [self._get_external_library(dir, name, - force_static)] + force_static)] break if not candidates: raise SandboxValidationError( '%s contains "%s", but there is no "%s" %s in %s.' % (variable, path, name, - self.LIBRARY_NAME_VAR[obj.KIND], dir), context) + self.LIBRARY_NAME_VAR[obj.KIND], dir), context) if len(candidates) > 1: # If there's more than one remaining candidate, it could be @@ -423,11 +423,11 @@ class TreeMetadataEmitter(LoggingMixin): raise SandboxValidationError( '%s contains "static:%s", but there is no static ' '"%s" %s in %s.' % (variable, path, name, - self.LIBRARY_NAME_VAR[obj.KIND], dir), context) + self.LIBRARY_NAME_VAR[obj.KIND], dir), context) raise SandboxValidationError( '%s contains "static:%s", but there is no static "%s" ' '%s in the tree' % (variable, name, name, - self.LIBRARY_NAME_VAR[obj.KIND]), context) + self.LIBRARY_NAME_VAR[obj.KIND]), context) if not candidates: raise SandboxValidationError( @@ -437,23 +437,23 @@ class TreeMetadataEmitter(LoggingMixin): elif len(candidates) > 1: paths = (mozpath.join(l.relsrcdir, 'moz.build') - for l in candidates) + for l in candidates) raise SandboxValidationError( '%s contains "%s", which matches a %s defined in multiple ' 'places:\n %s' % (variable, path, - self.LIBRARY_NAME_VAR[obj.KIND], - '\n '.join(paths)), context) + self.LIBRARY_NAME_VAR[obj.KIND], + '\n '.join(paths)), context) elif force_static and not isinstance(candidates[0], StaticLibrary): raise SandboxValidationError( '%s contains "static:%s", but there is only a shared "%s" ' 'in %s. You may want to add FORCE_STATIC_LIB=True in ' '%s/moz.build, or remove "static:".' % (variable, path, - name, candidates[0].relobjdir, candidates[0].relobjdir), + name, candidates[0].relobjdir, candidates[0].relobjdir), context) elif isinstance(obj, StaticLibrary) and isinstance(candidates[0], - SharedLibrary): + SharedLibrary): self._static_linking_shared.add(obj) obj.link_library(candidates[0]) @@ -485,25 +485,29 @@ class TreeMetadataEmitter(LoggingMixin): # A simple version number. if isinstance(values, (str, unicode)): raise SandboxValidationError( - '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), + '%s %s of crate %s does not list a path' % ( + description, dep_crate_name, crate_name), context) dep_path = values.get('path', None) if not dep_path: raise SandboxValidationError( - '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), + '%s %s of crate %s does not list a path' % ( + description, dep_crate_name, crate_name), context) # Try to catch the case where somebody listed a # local path for development. if os.path.isabs(dep_path): raise SandboxValidationError( - '%s %s of crate %s has a non-relative path' % (description, dep_crate_name, crate_name), + '%s %s of crate %s has a non-relative path' % ( + description, dep_crate_name, crate_name), context) if not os.path.exists(mozpath.join(context.config.topsrcdir, crate_dir, dep_path)): raise SandboxValidationError( - '%s %s of crate %s refers to a non-existent path' % (description, dep_crate_name, crate_name), + '%s %s of crate %s refers to a non-existent path' % ( + description, dep_crate_name, crate_name), context) def _rust_library(self, context, libname, static_args, cls=RustLibrary): @@ -549,7 +553,6 @@ class TreeMetadataEmitter(LoggingMixin): return cls(context, libname, cargo_file, crate_type, dependencies, features, cargo_target_dir, **static_args) - def _handle_gn_dirs(self, context): for target_dir in context.get('GN_DIRS', []): context['DIRS'] += [target_dir] @@ -569,10 +572,10 @@ class TreeMetadataEmitter(LoggingMixin): yield GnProjectData(context, target_dir, gn_dir, non_unified_sources) - def _handle_linkables(self, context, passthru, generated_files): linkables = [] host_linkables = [] + def add_program(prog, var): if var.startswith('HOST_'): host_linkables.append(prog) @@ -584,14 +587,14 @@ class TreeMetadataEmitter(LoggingMixin): raise SandboxValidationError( 'Cannot use "%s" as %s name, ' 'because it is already used in %s' % (program, kind, - self._binaries[program].relsrcdir), context) + self._binaries[program].relsrcdir), context) for kind, cls in [('PROGRAM', Program), ('HOST_PROGRAM', HostProgram)]: program = context.get(kind) if program: check_unique_binary(program, kind) self._binaries[program] = cls(context, program) self._linkage.append((context, self._binaries[program], - kind.replace('PROGRAM', 'USE_LIBS'))) + kind.replace('PROGRAM', 'USE_LIBS'))) add_program(self._binaries[program], kind) all_rust_programs = [] @@ -605,7 +608,7 @@ class TreeMetadataEmitter(LoggingMixin): # Verify Rust program definitions. if all_rust_programs: - config, cargo_file = self._parse_cargo_file(context); + config, cargo_file = self._parse_cargo_file(context) bin_section = config.get('bin', None) if not bin_section: raise SandboxValidationError( @@ -634,12 +637,12 @@ class TreeMetadataEmitter(LoggingMixin): raise SandboxValidationError( 'Cannot use "%s" in %s, ' 'because it is already used in %s' % (program, kind, - self._binaries[program].relsrcdir), context) + self._binaries[program].relsrcdir), context) self._binaries[program] = cls(context, program, - is_unit_test=kind == 'CPP_UNIT_TESTS') + is_unit_test=kind == 'CPP_UNIT_TESTS') self._linkage.append((context, self._binaries[program], - 'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS' - else 'USE_LIBS')) + 'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS' + else 'USE_LIBS')) add_program(self._binaries[program], kind) host_libname = context.get('HOST_LIBRARY_NAME') @@ -648,7 +651,7 @@ class TreeMetadataEmitter(LoggingMixin): if host_libname: if host_libname == libname: raise SandboxValidationError('LIBRARY_NAME and ' - 'HOST_LIBRARY_NAME must have a different value', context) + 'HOST_LIBRARY_NAME must have a different value', context) is_rust_library = context.get('IS_RUST_LIBRARY') if is_rust_library: @@ -779,7 +782,7 @@ class TreeMetadataEmitter(LoggingMixin): raise SandboxValidationError( 'Path specified in SYMBOLS_FILE does not exist: %s ' '(resolved to %s)' % (symbols_file, - symbols_file.full_path), context) + symbols_file.full_path), context) shared_args['symbols_file'] = True else: if symbols_file.target_basename not in generated_files: @@ -803,8 +806,8 @@ class TreeMetadataEmitter(LoggingMixin): if lib.defines: defines = lib.defines.get_defines() yield GeneratedFile(context, script, - 'generate_symbols_file', lib.symbols_file, - [symbols_file], defines) + 'generate_symbols_file', lib.symbols_file, + [symbols_file], defines) if static_lib: is_rust_library = context.get('IS_RUST_LIBRARY') if is_rust_library: @@ -818,7 +821,7 @@ class TreeMetadataEmitter(LoggingMixin): if lib_defines: if not libname: raise SandboxValidationError('LIBRARY_DEFINES needs a ' - 'LIBRARY_NAME to take effect', context) + 'LIBRARY_NAME to take effect', context) lib.lib_defines.update(lib_defines) # Only emit sources if we have linkables defined in the same context. @@ -849,7 +852,7 @@ class TreeMetadataEmitter(LoggingMixin): for f in context_srcs: if f in seen_sources: raise SandboxValidationError('Source file should only ' - 'be added to %s once: %s' % (symbol, f), context) + 'be added to %s once: %s' % (symbol, f), context) seen_sources.add(f) full_path = f.full_path if isinstance(f, SourcePath): @@ -866,18 +869,18 @@ class TreeMetadataEmitter(LoggingMixin): if flags.pgo_generate_only: if not isinstance(f, Path): raise SandboxValidationError('pgo_generate_only file' - 'must not be a generated file: %s' % f, context) + 'must not be a generated file: %s' % f, context) if mozpath.splitext(f)[1] != '.cpp': raise SandboxValidationError('pgo_generate_only file' - 'must be a .cpp file: %s' % f, context) + 'must be a .cpp file: %s' % f, context) if flags.no_pgo: raise SandboxValidationError('pgo_generate_only files' - 'cannot be marked no_pgo: %s' % f, context) + 'cannot be marked no_pgo: %s' % f, context) pgo_generate_only.add(f) if isinstance(f, SourcePath) and not os.path.exists(full_path): raise SandboxValidationError('File listed in %s does not ' - 'exist: \'%s\'' % (symbol, full_path), context) + 'exist: \'%s\'' % (symbol, full_path), context) # UNIFIED_SOURCES only take SourcePaths, so there should be no # generated source in here @@ -891,7 +894,7 @@ class TreeMetadataEmitter(LoggingMixin): if no_pgo: if no_pgo_sources: raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo ' - 'cannot be set at the same time', context) + 'cannot be set at the same time', context) passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo if no_pgo_sources: passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources @@ -1003,7 +1006,6 @@ class TreeMetadataEmitter(LoggingMixin): l.cxx_link = True break - def emit_from_context(self, context): """Convert a Context to tree metadata objects. @@ -1022,7 +1024,8 @@ class TreeMetadataEmitter(LoggingMixin): # We always emit a directory traversal descriptor. This is needed by # the recursive make backend. - for o in self._emit_directory_traversal_from_context(context): yield o + for o in self._emit_directory_traversal_from_context(context): + yield o for obj in self._process_xpidl(context): yield obj @@ -1052,7 +1055,7 @@ class TreeMetadataEmitter(LoggingMixin): if context.config.substs.get('OS_TARGET') == 'WINNT' and \ context['DELAYLOAD_DLLS']: context['LDFLAGS'].extend([('-DELAYLOAD:%s' % dll) - for dll in context['DELAYLOAD_DLLS']]) + for dll in context['DELAYLOAD_DLLS']]) context['OS_LIBS'].append('delayimp') for v in ['CMFLAGS', 'CMMFLAGS']: @@ -1077,7 +1080,7 @@ class TreeMetadataEmitter(LoggingMixin): raise SandboxValidationError( 'Path specified in DEFFILE does not exist: %s ' '(resolved to %s)' % (deffile, - deffile.full_path), context) + deffile.full_path), context) path = mozpath.relpath(deffile.full_path, context.objdir) else: path = deffile.target_basename @@ -1138,7 +1141,7 @@ class TreeMetadataEmitter(LoggingMixin): for path in context['CONFIGURE_SUBST_FILES']: sub = self._create_substitution(ConfigFileSubstitution, context, - path) + path) generated_files.add(str(sub.relpath)) yield sub @@ -1185,17 +1188,17 @@ class TreeMetadataEmitter(LoggingMixin): if not isinstance(local_include, ObjDirPath): if not os.path.exists(full_path): raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' - 'does not exist: %s (resolved to %s)' % (local_include, - full_path), context) + 'does not exist: %s (resolved to %s)' % (local_include, + full_path), context) if not os.path.isdir(full_path): raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' - 'is a filename, but a directory is required: %s ' - '(resolved to %s)' % (local_include, full_path), context) + 'is a filename, but a directory is required: %s ' + '(resolved to %s)' % (local_include, full_path), context) if (full_path == context.config.topsrcdir or full_path == context.config.topobjdir): raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' - '(%s) resolves to the topsrcdir or topobjdir (%s), which is ' - 'not allowed' % (local_include, full_path), context) + '(%s) resolves to the topsrcdir or topobjdir (%s), which is ' + 'not allowed' % (local_include, full_path), context) include_obj = LocalInclude(context, local_include) local_includes.append(include_obj.path.full_path) yield include_obj @@ -1207,7 +1210,8 @@ class TreeMetadataEmitter(LoggingMixin): for obj in self._handle_linkables(context, passthru, generated_files): yield obj - generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()]) + generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) + for k in self._binaries.keys()]) components = [] for var, cls in ( @@ -1376,7 +1380,6 @@ class TreeMetadataEmitter(LoggingMixin): if context.objdir in self._host_compile_dirs: yield computed_host_flags - def _create_substitution(self, cls, context, path): sub = cls(context) sub.input_path = '%s.in' % path.full_path @@ -1395,12 +1398,12 @@ class TreeMetadataEmitter(LoggingMixin): if not xpidl_module: if context['XPIDL_SOURCES']: raise SandboxValidationError('XPIDL_MODULE must be defined if ' - 'XPIDL_SOURCES is defined.', context) + 'XPIDL_SOURCES is defined.', context) return if not context['XPIDL_SOURCES']: raise SandboxValidationError('XPIDL_MODULE cannot be defined ' - 'unless there are XPIDL_SOURCES', context) + 'unless there are XPIDL_SOURCES', context) if context['DIST_INSTALL'] is False: self.log(logging.WARN, 'mozbuild_warning', dict( @@ -1410,7 +1413,7 @@ class TreeMetadataEmitter(LoggingMixin): for idl in context['XPIDL_SOURCES']: if not os.path.exists(idl.full_path): raise SandboxValidationError('File %s from XPIDL_SOURCES ' - 'does not exist' % idl.full_path, context) + 'does not exist' % idl.full_path, context) yield XPIDLModule(context, xpidl_module, context['XPIDL_SOURCES']) @@ -1482,7 +1485,7 @@ class TreeMetadataEmitter(LoggingMixin): path = manifest_path.full_path manifest_dir = mozpath.dirname(path) manifest_reldir = mozpath.dirname(mozpath.relpath(path, - context.config.topsrcdir)) + context.config.topsrcdir)) manifest_sources = [mozpath.relpath(pth, context.config.topsrcdir) for pth in mpmanifest.source_files] install_prefix = mozpath.join(install_root, install_subdir) @@ -1490,22 +1493,22 @@ class TreeMetadataEmitter(LoggingMixin): try: if not mpmanifest.tests: raise SandboxValidationError('Empty test manifest: %s' - % path, context) + % path, context) defaults = mpmanifest.manifest_defaults[os.path.normpath(path)] obj = TestManifest(context, path, mpmanifest, flavor=flavor, - install_prefix=install_prefix, - relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), - sources=manifest_sources, - dupe_manifest='dupe-manifest' in defaults) + install_prefix=install_prefix, + relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), + sources=manifest_sources, + dupe_manifest='dupe-manifest' in defaults) filtered = mpmanifest.tests missing = [t['name'] for t in filtered if not os.path.exists(t['path'])] if missing: raise SandboxValidationError('Test manifest (%s) lists ' - 'test that does not exist: %s' % ( - path, ', '.join(missing)), context) + 'test that does not exist: %s' % ( + path, ', '.join(missing)), context) out_dir = mozpath.join(install_prefix, manifest_reldir) if 'install-to-subdir' in defaults: @@ -1529,8 +1532,8 @@ class TreeMetadataEmitter(LoggingMixin): install_path[2:])), install_path not in install_info.external_installs]): raise SandboxValidationError('Error processing test ' - 'manifest %s: entry in support-files not present ' - 'in the srcdir: %s' % (path, install_path), context) + 'manifest %s: entry in support-files not present ' + 'in the srcdir: %s' % (path, install_path), context) obj.deferred_installs |= install_info.deferred_installs @@ -1541,7 +1544,7 @@ class TreeMetadataEmitter(LoggingMixin): # test package. They function as identifiers rather than files. if package_tests: manifest_relpath = mozpath.relpath(test['path'], - mozpath.dirname(test['manifest'])) + mozpath.dirname(test['manifest'])) obj.installs[mozpath.normpath(test['path'])] = \ ((mozpath.join(out_dir, manifest_relpath)), True) @@ -1568,28 +1571,28 @@ class TreeMetadataEmitter(LoggingMixin): del obj.installs[mozpath.join(manifest_dir, f)] except KeyError: raise SandboxValidationError('Error processing test ' - 'manifest %s: entry in generated-files not present ' - 'elsewhere in manifest: %s' % (path, f), context) + 'manifest %s: entry in generated-files not present ' + 'elsewhere in manifest: %s' % (path, f), context) yield obj except (AssertionError, Exception): raise SandboxValidationError('Error processing test ' - 'manifest file %s: %s' % (path, - '\n'.join(traceback.format_exception(*sys.exc_info()))), - context) + 'manifest file %s: %s' % (path, + '\n'.join(traceback.format_exception(*sys.exc_info()))), + context) def _process_reftest_manifest(self, context, flavor, manifest_path, manifest): manifest_full_path = manifest_path.full_path manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, - context.config.topsrcdir)) + context.config.topsrcdir)) # reftest manifests don't come from manifest parser. But they are # similar enough that we can use the same emitted objects. Note # that we don't perform any installs for reftests. obj = TestManifest(context, manifest_full_path, manifest, - flavor=flavor, install_prefix='%s/' % flavor, - relpath=mozpath.join(manifest_reldir, - mozpath.basename(manifest_path))) + flavor=flavor, install_prefix='%s/' % flavor, + relpath=mozpath.join(manifest_reldir, + mozpath.basename(manifest_path))) for test, source_manifest in sorted(manifest.tests): obj.tests.append({ @@ -1608,7 +1611,7 @@ class TreeMetadataEmitter(LoggingMixin): jar_manifests = context.get('JAR_MANIFESTS', []) if len(jar_manifests) > 1: raise SandboxValidationError('While JAR_MANIFESTS is a list, ' - 'it is currently limited to one value.', context) + 'it is currently limited to one value.', context) for path in jar_manifests: yield JARManifest(context, path) @@ -1620,8 +1623,8 @@ class TreeMetadataEmitter(LoggingMixin): if os.path.exists(os.path.join(context.srcdir, 'jar.mn')): if 'jar.mn' not in jar_manifests: raise SandboxValidationError('A jar.mn exists but it ' - 'is not referenced in the moz.build file. ' - 'Please define JAR_MANIFESTS.', context) + 'is not referenced in the moz.build file. ' + 'Please define JAR_MANIFESTS.', context) def _emit_directory_traversal_from_context(self, context): o = DirectoryTraversal(context) @@ -1630,6 +1633,6 @@ class TreeMetadataEmitter(LoggingMixin): # Some paths have a subconfigure, yet also have a moz.build. Those # shouldn't end up in self._external_paths. if o.objdir: - self._external_paths -= { o.relobjdir } + self._external_paths -= {o.relobjdir} yield o diff --git a/python/mozbuild/mozbuild/frontend/gyp_reader.py b/python/mozbuild/mozbuild/frontend/gyp_reader.py index 04efc5181108..f06669fb21e7 100644 --- a/python/mozbuild/mozbuild/frontend/gyp_reader.py +++ b/python/mozbuild/mozbuild/frontend/gyp_reader.py @@ -40,7 +40,7 @@ sys.modules['gyp.generator.mozbuild'] = sys.modules[__name__] # chrome_src for the default includes, so go backwards from the pylib # directory, which is the parent directory of gyp module. chrome_src = mozpath.abspath(mozpath.join(mozpath.dirname(gyp.__file__), - '../../../../..')) + '../../../../..')) script_dir = mozpath.join(chrome_src, 'build') @@ -74,10 +74,11 @@ class GypContext(TemplateContext): relobjdir is the object directory that will be used for this context, relative to the topobjdir defined in the ConfigEnvironment. """ + def __init__(self, config, relobjdir): self._relobjdir = relobjdir TemplateContext.__init__(self, template='Gyp', - allowed_variables=VARIABLES, config=config) + allowed_variables=VARIABLES, config=config) def handle_actions(actions, context, action_overrides): @@ -88,10 +89,12 @@ def handle_actions(actions, context, action_overrides): raise RuntimeError('GYP action %s not listed in action_overrides' % name) outputs = action['outputs'] if len(outputs) > 1: - raise NotImplementedError('GYP actions with more than one output not supported: %s' % name) + raise NotImplementedError( + 'GYP actions with more than one output not supported: %s' % name) output = outputs[0] if not output.startswith(idir): - raise NotImplementedError('GYP actions outputting to somewhere other than <(INTERMEDIATE_DIR) not supported: %s' % output) + raise NotImplementedError( + 'GYP actions outputting to somewhere other than <(INTERMEDIATE_DIR) not supported: %s' % output) output = output[len(idir):] context['GENERATED_FILES'] += [output] g = context['GENERATED_FILES'][output] @@ -104,7 +107,8 @@ def handle_copies(copies, context): for copy in copies: dest = copy['destination'] if not dest.startswith(dist): - raise NotImplementedError('GYP copies to somewhere other than <(PRODUCT_DIR)/dist not supported: %s' % dest) + raise NotImplementedError( + 'GYP copies to somewhere other than <(PRODUCT_DIR)/dist not supported: %s' % dest) dest_paths = dest[len(dist):].split('/') exports = context['EXPORTS'] while dest_paths: @@ -130,8 +134,8 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, # directory. Since several targets can be in a given build_file, # separate them in subdirectories using the build_file basename # and the target_name. - reldir = mozpath.relpath(mozpath.dirname(build_file), - mozpath.dirname(path)) + reldir = mozpath.relpath(mozpath.dirname(build_file), + mozpath.dirname(path)) subdir = '%s_%s' % ( mozpath.splitext(mozpath.basename(build_file))[0], target_name, @@ -155,12 +159,13 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, target_conf = spec['configurations'][c] if 'actions' in spec: - handle_actions(spec['actions'], context, action_overrides) + handle_actions(spec['actions'], context, action_overrides) if 'copies' in spec: - handle_copies(spec['copies'], context) + handle_copies(spec['copies'], context) use_libs = [] libs = [] + def add_deps(s): for t in s.get('dependencies', []) + s.get('dependencies_original', []): ty = targets[t]['type'] @@ -171,7 +176,7 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, if ty in ('static_library', 'none'): add_deps(targets[t]) libs.extend(spec.get('libraries', [])) - #XXX: this sucks, but webrtc breaks with this right now because + # XXX: this sucks, but webrtc breaks with this right now because # it builds a library called 'gtest' and we just get lucky # that it isn't in USE_LIBS by that name anywhere. if no_chromium: @@ -179,20 +184,20 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, os_libs = [] for l in libs: - if l.startswith('-'): - os_libs.append(l) - elif l.endswith('.lib'): - os_libs.append(l[:-4]) - elif l: - # For library names passed in from moz.build. - use_libs.append(os.path.basename(l)) + if l.startswith('-'): + os_libs.append(l) + elif l.endswith('.lib'): + os_libs.append(l[:-4]) + elif l: + # For library names passed in from moz.build. + use_libs.append(os.path.basename(l)) if spec['type'] == 'none': - if not ('actions' in spec or 'copies' in spec): - continue + if not ('actions' in spec or 'copies' in spec): + continue elif spec['type'] in ('static_library', 'shared_library', 'executable'): - # Remove leading 'lib' from the target_name if any, and use as - # library name. + # Remove leading 'lib' from the target_name if any, and use as + # library name. name = spec['target_name'] if spec['type'] in ('static_library', 'shared_library'): if name.startswith('lib'): @@ -222,9 +227,9 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, ext = mozpath.splitext(f)[-1] extensions.add(ext) if f.startswith('$INTERMEDIATE_DIR/'): - s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/', '!')) + s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/', '!')) else: - s = SourcePath(context, f) + s = SourcePath(context, f) if ext == '.h': continue if ext == '.def': @@ -282,7 +287,8 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, if include.startswith('/'): resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:])) elif not include.startswith(('!', '%')): - resolved = mozpath.abspath(mozpath.join(mozpath.dirname(build_file), include)) + resolved = mozpath.abspath(mozpath.join( + mozpath.dirname(build_file), include)) if not include.startswith(('!', '%')) and not os.path.exists(resolved): continue context['LOCAL_INCLUDES'] += [include] @@ -325,17 +331,17 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, raise NotImplementedError('Unsupported gyp target type: %s' % spec['type']) if not no_chromium: - # Add some features to all contexts. Put here in case LOCAL_INCLUDES - # order matters. - context['LOCAL_INCLUDES'] += [ - '!/ipc/ipdl/_ipdlheaders', - '/ipc/chromium/src', - '/ipc/glue', - ] - # These get set via VC project file settings for normal GYP builds. - if config.substs['OS_TARGET'] == 'WINNT': - context['DEFINES']['UNICODE'] = True - context['DEFINES']['_UNICODE'] = True + # Add some features to all contexts. Put here in case LOCAL_INCLUDES + # order matters. + context['LOCAL_INCLUDES'] += [ + '!/ipc/ipdl/_ipdlheaders', + '/ipc/chromium/src', + '/ipc/glue', + ] + # These get set via VC project file settings for normal GYP builds. + if config.substs['OS_TARGET'] == 'WINNT': + context['DEFINES']['UNICODE'] = True + context['DEFINES']['_UNICODE'] = True context['COMPILE_FLAGS']['OS_INCLUDES'] = [] for key, value in gyp_dir_attrs.sandbox_vars.items(): @@ -368,6 +374,7 @@ class GypProcessor(object): gyp dependencies will be. gyp_dir_attrs are attributes set for the dir from moz.build. """ + def __init__(self, config, gyp_dir_attrs, path, output, executor, action_overrides, non_unified_sources): self._path = path diff --git a/python/mozbuild/mozbuild/frontend/mach_commands.py b/python/mozbuild/mozbuild/frontend/mach_commands.py index 0db4e7cba108..e27c7aeb3ed3 100644 --- a/python/mozbuild/mozbuild/frontend/mach_commands.py +++ b/python/mozbuild/mozbuild/frontend/mach_commands.py @@ -21,6 +21,7 @@ import mozpack.path as mozpath TOPSRCDIR = os.path.abspath(os.path.join(__file__, '../../../../../')) + class InvalidPathException(Exception): """Represents an error due to an invalid path.""" @@ -28,11 +29,11 @@ class InvalidPathException(Exception): @CommandProvider class MozbuildFileCommands(MachCommandBase): @Command('mozbuild-reference', category='build-dev', - description='View reference documentation on mozbuild files.') + description='View reference documentation on mozbuild files.') @CommandArgument('symbol', default=None, nargs='*', - help='Symbol to view help on. If not specified, all will be shown.') + help='Symbol to view help on. If not specified, all will be shown.') @CommandArgument('--name-only', '-n', default=False, action='store_true', - help='Print symbol names only.') + help='Print symbol names only.') def reference(self, symbol, name_only=False): # mozbuild.sphinx imports some Sphinx modules, so we need to be sure # the optional Sphinx package is installed. @@ -284,7 +285,6 @@ class MozbuildFileCommands(MachCommandBase): print(e.message) return 1 - def _get_files_info(self, paths, rev=None): reader = self.mozbuild_reader(config_mode='empty', vcs_revision=rev) @@ -328,7 +328,6 @@ class MozbuildFileCommands(MachCommandBase): return reader.files_info(allpaths) - @SubCommand('file-info', 'schedules', 'Show the combined SCHEDULES for the files listed.') @CommandArgument('paths', nargs='+', diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbuild/frontend/reader.py index 7eca98a0ad0d..30972618aca5 100644 --- a/python/mozbuild/mozbuild/frontend/reader.py +++ b/python/mozbuild/mozbuild/frontend/reader.py @@ -80,7 +80,6 @@ from mozbuild.base import ExecutionSummary from concurrent.futures.process import ProcessPoolExecutor - if sys.version_info.major == 2: text_type = unicode type_type = types.TypeType @@ -106,6 +105,7 @@ class EmptyConfig(object): This variation is needed because CONFIG uses .get() to access members. Without it, None (instead of our EmptyValue types) would be returned. """ + def get(self, key, default=None): return self[key] @@ -182,6 +182,7 @@ class MozbuildSandbox(Sandbox): metadata is a dict of metadata that can be used during the sandbox evaluation. """ + def __init__(self, context, metadata={}, finder=default_finder): assert isinstance(context, Context) @@ -241,7 +242,7 @@ class MozbuildSandbox(Sandbox): # protection, so it is omitted. if not is_read_allowed(path, self._context.config): raise SandboxLoadError(self._context.source_stack, - sys.exc_info()[2], illegal_path=path) + sys.exc_info()[2], illegal_path=path) Sandbox.exec_file(self, path) @@ -290,14 +291,14 @@ class MozbuildSandbox(Sandbox): if not inspect.isfunction(func): raise Exception('`template` is a function decorator. You must ' - 'use it as `@template` preceding a function declaration.') + 'use it as `@template` preceding a function declaration.') name = func.func_name if name in self.templates: raise KeyError( 'A template named "%s" was already declared in %s.' % (name, - self.templates[name].path)) + self.templates[name].path)) if name.islower() or name.isupper() or name[0].islower(): raise NameError('Template function names must be CamelCase.') @@ -320,6 +321,7 @@ class MozbuildSandbox(Sandbox): The wrapper function does type coercion on the function arguments """ func, args_def, doc = function_def + def function(*args): def coerce(arg, type): if not isinstance(arg, type): @@ -416,7 +418,7 @@ class TemplateFunction(object): # actually never calls __getitem__ and __setitem__, so we need to # modify the AST so that accesses to globals are properly directed # to a dict. - self._global_name = b'_data' # AST wants str for this, not unicode + self._global_name = b'_data' # AST wants str for this, not unicode # In case '_data' is a name used for a variable in the function code, # prepend more underscores until we find an unused name. while (self._global_name in code.co_names or @@ -463,6 +465,7 @@ class TemplateFunction(object): """AST Node Transformer to rewrite variable accesses to go through a dict. """ + def __init__(self, sandbox, global_name): self._sandbox = sandbox self._global_name = global_name @@ -491,6 +494,7 @@ class TemplateFunction(object): class SandboxValidationError(Exception): """Represents an error encountered when validating sandbox results.""" + def __init__(self, message, context): Exception.__init__(self, message) self.context = context @@ -532,9 +536,10 @@ class BuildReaderError(Exception): MozbuildSandbox has over Sandbox (e.g. the concept of included files - which affect error messages, of course). """ + def __init__(self, file_stack, trace, sandbox_exec_error=None, - sandbox_load_error=None, validation_error=None, other_error=None, - sandbox_called_error=None): + sandbox_load_error=None, validation_error=None, other_error=None, + sandbox_called_error=None): self.file_stack = file_stack self.trace = trace @@ -559,7 +564,7 @@ class BuildReaderError(Exception): return self.file_stack[-2] if self.sandbox_error is not None and \ - len(self.sandbox_error.file_stack): + len(self.sandbox_error.file_stack): return self.sandbox_error.file_stack[-1] return self.file_stack[-1] @@ -602,7 +607,7 @@ class BuildReaderError(Exception): s.write('\n') for l in traceback.format_exception(type(self.other), self.other, - self.trace): + self.trace): s.write(unicode(l)) return s.getvalue() @@ -760,7 +765,7 @@ class BuildReaderError(Exception): if inner.args[2] in DEPRECATION_HINTS: s.write('%s\n' % - textwrap.dedent(DEPRECATION_HINTS[inner.args[2]]).strip()) + textwrap.dedent(DEPRECATION_HINTS[inner.args[2]]).strip()) return s.write('Please change the file to not use this variable.\n') @@ -802,7 +807,7 @@ class BuildReaderError(Exception): s.write(' %s\n' % inner.args[4].__name__) else: for t in inner.args[4]: - s.write( ' %s\n' % t.__name__) + s.write(' %s\n' % t.__name__) s.write('\n') s.write('Change the file to write a value of the appropriate type ') s.write('and try again.\n') @@ -1053,23 +1058,23 @@ class BuildReader(object): except SandboxCalledError as sce: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], sandbox_called_error=sce) + sys.exc_info()[2], sandbox_called_error=sce) except SandboxExecutionError as se: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], sandbox_exec_error=se) + sys.exc_info()[2], sandbox_exec_error=se) except SandboxLoadError as sle: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], sandbox_load_error=sle) + sys.exc_info()[2], sandbox_load_error=sle) except SandboxValidationError as ve: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], validation_error=ve) + sys.exc_info()[2], validation_error=ve) except Exception as e: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], other_error=e) + sys.exc_info()[2], other_error=e) def _read_mozbuild(self, path, config, descend, metadata): path = mozpath.normpath(path) @@ -1127,7 +1132,7 @@ class BuildReader(object): for v in ('input', 'variables'): if not getattr(gyp_dir, v): raise SandboxValidationError('Missing value for ' - 'GYP_DIRS["%s"].%s' % (target_dir, v), context) + 'GYP_DIRS["%s"].%s' % (target_dir, v), context) # The make backend assumes contexts for sub-directories are # emitted after their parent, so accumulate the gyp contexts. @@ -1140,7 +1145,7 @@ class BuildReader(object): source = SourcePath(context, s) if not self.finder.get(source.full_path): raise SandboxValidationError('Cannot find %s.' % source, - context) + context) non_unified_sources.add(source) action_overrides = {} for action, script in gyp_dir.action_overrides.iteritems(): @@ -1189,7 +1194,7 @@ class BuildReader(object): if not is_read_allowed(child_path, context.config): raise SandboxValidationError( 'Attempting to process file outside of allowed paths: %s' % - child_path, context) + child_path, context) if not descend: continue @@ -1283,6 +1288,7 @@ class BuildReader(object): # Exporting doesn't work reliably in tree traversal mode. Override # the function to no-op. functions = dict(FUNCTIONS) + def export(sandbox): return lambda varname: None functions['export'] = tuple([export] + list(FUNCTIONS['export'][1:])) @@ -1337,6 +1343,7 @@ class BuildReader(object): # times (once for every path in a directory that doesn't have any # test metadata). So, we cache the function call. defaults_cache = {} + def test_defaults_for_path(ctxs): key = tuple(ctx.current_path or ctx.main_path for ctx in ctxs) @@ -1394,7 +1401,8 @@ class BuildReader(object): test_manifest_contexts = set( ['%s_MANIFESTS' % key for key in TEST_MANIFESTS] + ['%s_MANIFESTS' % flavor.upper() for flavor in REFTEST_FLAVORS] + - ['%s_MANIFESTS' % flavor.upper().replace('-', '_') for flavor in WEB_PLATFORM_TESTS_FLAVORS] + ['%s_MANIFESTS' % flavor.upper().replace('-', '_') + for flavor in WEB_PLATFORM_TESTS_FLAVORS] ) result_context = Files(Context()) diff --git a/python/mozbuild/mozbuild/frontend/sandbox.py b/python/mozbuild/mozbuild/frontend/sandbox.py index b2090802e6ea..564603703206 100644 --- a/python/mozbuild/mozbuild/frontend/sandbox.py +++ b/python/mozbuild/mozbuild/frontend/sandbox.py @@ -53,6 +53,7 @@ class SandboxExecutionError(SandboxError): This is a simple container exception. It's purpose is to capture state so something else can report on it. """ + def __init__(self, file_stack, exc_type, exc_value, trace): SandboxError.__init__(self, file_stack) @@ -69,6 +70,7 @@ class SandboxLoadError(SandboxError): a file. If so, the file_stack will be non-empty and the file that caused the load will be on top of the stack. """ + def __init__(self, file_stack, trace, illegal_path=None, read_error=None): SandboxError.__init__(self, file_stack) @@ -155,7 +157,7 @@ class Sandbox(dict): source = self._finder.get(path).read() except Exception as e: raise SandboxLoadError(self._context.source_stack, - sys.exc_info()[2], read_error=path) + sys.exc_info()[2], read_error=path) self.exec_source(source, path) @@ -288,7 +290,7 @@ class Sandbox(dict): raise KeyError('global_ns', 'reassign', key) if (key not in self._context and isinstance(value, (list, dict)) - and not value): + and not value): raise KeyError('Variable %s assigned an empty value.' % key) self._context[key] = value diff --git a/python/mozbuild/mozbuild/gn_processor.py b/python/mozbuild/mozbuild/gn_processor.py index 917526756c3f..57273757595e 100644 --- a/python/mozbuild/mozbuild/gn_processor.py +++ b/python/mozbuild/mozbuild/gn_processor.py @@ -78,7 +78,8 @@ class MozbuildWriter(object): self.write('\n') self.write(self.indent + key) self.write(' += [\n ' + self.indent) - self.write((',\n ' + self.indent).join(alphabetical_sorted(self.mb_serialize(v) for v in value))) + self.write( + (',\n ' + self.indent).join(alphabetical_sorted(self.mb_serialize(v) for v in value))) self.write('\n') self.write_ln(']') @@ -112,7 +113,6 @@ class MozbuildWriter(object): if not wrote_ln: self.write_ln("%s[%s] = %s" % subst_vals) - def write_condition(self, values): def mk_condition(k, v): if not v: diff --git a/python/mozbuild/mozbuild/jar.py b/python/mozbuild/mozbuild/jar.py index 47a2eff6390c..d9e37a4fc88c 100644 --- a/python/mozbuild/mozbuild/jar.py +++ b/python/mozbuild/mozbuild/jar.py @@ -90,7 +90,8 @@ class JarInfo(object): self.entries = [] -class DeprecatedJarManifest(Exception): pass +class DeprecatedJarManifest(Exception): + pass class JarManifestParser(object): @@ -197,7 +198,7 @@ class JarMaker(object): ''' def __init__(self, outputFormat='flat', useJarfileManifest=True, - useChromeManifest=False): + useChromeManifest=False): self.outputFormat = outputFormat self.useJarfileManifest = useJarfileManifest @@ -224,10 +225,10 @@ class JarMaker(object): p = self.pp.getCommandLineParser(unescapeDefines=True) p.add_option('-f', type='choice', default='jar', - choices=('jar', 'flat', 'symlink'), - help='fileformat used for output', - metavar='[jar, flat, symlink]', - ) + choices=('jar', 'flat', 'symlink'), + help='fileformat used for output', + metavar='[jar, flat, symlink]', + ) p.add_option('-v', action='store_true', dest='verbose', help='verbose output') p.add_option('-q', action='store_false', dest='verbose', @@ -238,13 +239,12 @@ class JarMaker(object): p.add_option('-s', type='string', action='append', default=[], help='source directory') p.add_option('-t', type='string', help='top source directory') - p.add_option('-c', '--l10n-src', type='string', action='append' - , help='localization directory') + p.add_option('-c', '--l10n-src', type='string', + action='append', help='localization directory') p.add_option('--l10n-base', type='string', action='store', help='base directory to be used for localization (requires relativesrcdir)' ) - p.add_option('--locale-mergedir', type='string', action='store' - , + p.add_option('--locale-mergedir', type='string', action='store', help='base directory to be used for l10n-merge (requires l10n-base and relativesrcdir)' ) p.add_option('--relativesrcdir', type='string', @@ -292,18 +292,18 @@ class JarMaker(object): chromeDir = \ os.path.basename(os.path.dirname(os.path.normpath(chromeManifest))) logging.info("adding '%s' entry to root chrome manifest appid=%s" - % (chromeDir, self.rootManifestAppId)) + % (chromeDir, self.rootManifestAppId)) addEntriesToListFile(rootChromeManifest, ['manifest %s/chrome.manifest application=%s' % (chromeDir, - self.rootManifestAppId)]) + self.rootManifestAppId)]) def updateManifest(self, manifestPath, chromebasepath, register): '''updateManifest replaces the % in the chrome registration entries with the given chrome base path, and updates the given manifest file. ''' myregister = dict.fromkeys(map(lambda s: s.replace('%', - chromebasepath), register)) + chromebasepath), register)) addEntriesToListFile(manifestPath, myregister.iterkeys()) def makeJar(self, infile, jardir): @@ -314,7 +314,7 @@ class JarMaker(object): ''' # making paths absolute, guess srcdir if file and add to sourcedirs - _normpath = lambda p: os.path.normpath(os.path.abspath(p)) + def _normpath(p): return os.path.normpath(os.path.abspath(p)) self.topsourcedir = _normpath(self.topsourcedir) self.sourcedirs = [_normpath(p) for p in self.sourcedirs] if self.localedirs: @@ -348,7 +348,7 @@ class JarMaker(object): if self.l10nmerge or not self.l10nbase: # add en-US if we merge, or if it's not l10n locdirs.append(os.path.join(self.topsourcedir, - relativesrcdir, 'en-US')) + relativesrcdir, 'en-US')) return locdirs def processJarSection(self, jarinfo, jardir): @@ -457,7 +457,7 @@ class JarMaker(object): if jf is not None: jf.close() raise RuntimeError('File "{0}" not found in {1}'.format(src, - ', '.join(src_base))) + ', '.join(src_base))) if out in self._seen_output: raise RuntimeError('%s already added' % out) @@ -585,7 +585,7 @@ def main(args=None): jm.l10nmerge = options.locale_mergedir if jm.l10nmerge and not os.path.isdir(jm.l10nmerge): logging.warning("WARNING: --locale-mergedir passed, but '%s' does not exist. " - "Ignore this message if the locale is complete." % jm.l10nmerge) + "Ignore this message if the locale is complete." % jm.l10nmerge) elif options.locale_mergedir: p.error('l10n-base required when using locale-mergedir') jm.localedirs = options.l10n_src diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 221d957867ae..35aa9be0db02 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -140,19 +140,19 @@ class Build(MachCommandBase): @Command('build', category='build', description='Build the tree.') @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int, - help='Number of concurrent jobs to run. Default is the number of CPUs.') + help='Number of concurrent jobs to run. Default is the number of CPUs.') @CommandArgument('-C', '--directory', default=None, - help='Change to a subdirectory of the build directory first.') + help='Change to a subdirectory of the build directory first.') @CommandArgument('what', default=None, nargs='*', help=BUILD_WHAT_HELP) @CommandArgument('-X', '--disable-extra-make-dependencies', default=False, action='store_true', help='Do not add extra make dependencies.') @CommandArgument('-v', '--verbose', action='store_true', - help='Verbose output for what commands the build is running.') + help='Verbose output for what commands the build is running.') @CommandArgument('--keep-going', action='store_true', help='Keep building after an error has occurred') def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, - directory=None, verbose=False, keep_going=False): + directory=None, verbose=False, keep_going=False): """Build the source tree. With no arguments, this will perform a full build. @@ -190,7 +190,7 @@ class Build(MachCommandBase): mach_context=self._mach_context) @Command('configure', category='build', - description='Configure the tree (run configure and config.status).') + description='Configure the tree (run configure and config.status).') @CommandArgument('options', default=None, nargs=argparse.REMAINDER, help='Configure options') def configure(self, options=None, buildstatus_messages=False, line_handler=None): @@ -207,15 +207,15 @@ class Build(MachCommandBase): line_handler=line_handler) @Command('resource-usage', category='post-build', - description='Show information about system resource usage for a build.') + description='Show information about system resource usage for a build.') @CommandArgument('--address', default='localhost', - help='Address the HTTP server should listen on.') + help='Address the HTTP server should listen on.') @CommandArgument('--port', type=int, default=0, - help='Port number the HTTP server should listen on.') + help='Port number the HTTP server should listen on.') @CommandArgument('--browser', default='firefox', - help='Web browser to automatically open. See webbrowser Python module.') + help='Web browser to automatically open. See webbrowser Python module.') @CommandArgument('--url', - help='URL of JSON document to display') + help='URL of JSON document to display') def resource_usage(self, address=None, port=None, browser=None, url=None): import webbrowser from mozbuild.html_build_viewer import BuildViewerServer @@ -228,8 +228,8 @@ class Build(MachCommandBase): last = self._get_state_filename('build_resources.json') if not os.path.exists(last): print('Build resources not available. If you have performed a ' - 'build and receive this message, the psutil Python package ' - 'likely failed to initialize properly.') + 'build and receive this message, the psutil Python package ' + 'likely failed to initialize properly.') return 1 server.add_resource_json_file('last', last) @@ -246,17 +246,17 @@ class Build(MachCommandBase): server.run() @Command('build-backend', category='build', - description='Generate a backend used to build the tree.') + description='Generate a backend used to build the tree.') @CommandArgument('-d', '--diff', action='store_true', - help='Show a diff of changes.') + help='Show a diff of changes.') # It would be nice to filter the choices below based on # conditions, but that is for another day. @CommandArgument('-b', '--backend', nargs='+', choices=sorted(backends), - help='Which backend to build.') + help='Which backend to build.') @CommandArgument('-v', '--verbose', action='store_true', - help='Verbose output.') + help='Verbose output.') @CommandArgument('-n', '--dry-run', action='store_true', - help='Do everything except writing files out.') + help='Do everything except writing files out.') def build_backend(self, backend, diff=False, verbose=False, dry_run=False): python = self.virtualenv_manager.python_path config_status = os.path.join(self.topobjdir, 'config.status') @@ -279,7 +279,7 @@ class Build(MachCommandBase): args.append('--dry-run') return self._run_command_in_objdir(args=args, pass_thru=True, - ensure_exit_code=False) + ensure_exit_code=False) @CommandProvider @@ -295,7 +295,7 @@ class CargoProvider(MachCommandBase): @SubCommand('cargo', 'check', description='Run `cargo check` on a given crate. Defaults to gkrust.') @CommandArgument('--all-crates', default=None, action='store_true', - help='Check all of the crates in the tree.') + help='Check all of the crates in the tree.') @CommandArgument('crates', default=None, nargs='*', help='The crate name(s) to check.') def check(self, all_crates=None, crates=None): # XXX duplication with `mach vendor rust` @@ -335,30 +335,32 @@ class CargoProvider(MachCommandBase): return 0 + @CommandProvider class Doctor(MachCommandBase): """Provide commands for diagnosing common build environment problems""" @Command('doctor', category='devenv', - description='') + description='') @CommandArgument('--fix', default=None, action='store_true', - help='Attempt to fix found problems.') + help='Attempt to fix found problems.') def doctor(self, fix=None): self._activate_virtualenv() from mozbuild.doctor import Doctor doctor = Doctor(self.topsrcdir, self.topobjdir, fix) return doctor.check_all() + @CommandProvider class Clobber(MachCommandBase): NO_AUTO_LOG = True CLOBBER_CHOICES = ['objdir', 'python'] @Command('clobber', category='build', - description='Clobber the tree (delete the object directory).') + description='Clobber the tree (delete the object directory).') @CommandArgument('what', default=['objdir'], nargs='*', - help='Target to clobber, must be one of {{{}}} (default objdir).'.format( + help='Target to clobber, must be one of {{{}}} (default objdir).'.format( ', '.join(CLOBBER_CHOICES))) @CommandArgument('--full', action='store_true', - help='Perform a full clobber') + help='Perform a full clobber') def clobber(self, what, full=False): """Clean up the source and object directories. @@ -392,10 +394,10 @@ class Clobber(MachCommandBase): Clobberer(self.topsrcdir, self.topobjdir).remove_objdir(full) except OSError as e: if sys.platform.startswith('win'): - if isinstance(e, WindowsError) and e.winerror in (5,32): + if isinstance(e, WindowsError) and e.winerror in (5, 32): self.log(logging.ERROR, 'file_access_error', {'error': e}, - "Could not clobber because a file was in use. If the " - "application is running, try closing it. {error}") + "Could not clobber because a file was in use. If the " + "application is running, try closing it. {error}") return 1 raise @@ -421,16 +423,17 @@ class Clobber(MachCommandBase): except BuildEnvironmentNotFoundException: return {} + @CommandProvider class Logs(MachCommandBase): """Provide commands to read mach logs.""" NO_AUTO_LOG = True @Command('show-log', category='post-build', - description='Display mach logs') + description='Display mach logs') @CommandArgument('log_file', nargs='?', type=argparse.FileType('rb'), - help='Filename to read log data from. Defaults to the log of the last ' - 'mach command.') + help='Filename to read log data from. Defaults to the log of the last ' + 'mach command.') def show_log(self, log_file=None): if not log_file: path = self._get_state_filename('last_log.json') @@ -502,12 +505,12 @@ class Warnings(MachCommandBase): return database @Command('warnings-summary', category='post-build', - description='Show a summary of compiler warnings.') + description='Show a summary of compiler warnings.') @CommandArgument('-C', '--directory', default=None, - help='Change to a subdirectory of the build directory first.') + help='Change to a subdirectory of the build directory first.') @CommandArgument('report', default=None, nargs='?', - help='Warnings report to display. If not defined, show the most ' - 'recent report.') + help='Warnings report to display. If not defined, show the most ' + 'recent report.') def summary(self, directory=None, report=None): database = self.database @@ -520,7 +523,7 @@ class Warnings(MachCommandBase): type_counts = database.type_counts(dirpath) sorted_counts = sorted(type_counts.iteritems(), - key=operator.itemgetter(1)) + key=operator.itemgetter(1)) total = 0 for k, v in sorted_counts: @@ -530,14 +533,14 @@ class Warnings(MachCommandBase): print('%d\tTotal' % total) @Command('warnings-list', category='post-build', - description='Show a list of compiler warnings.') + description='Show a list of compiler warnings.') @CommandArgument('-C', '--directory', default=None, - help='Change to a subdirectory of the build directory first.') + help='Change to a subdirectory of the build directory first.') @CommandArgument('--flags', default=None, nargs='+', - help='Which warnings flags to match.') + help='Which warnings flags to match.') @CommandArgument('report', default=None, nargs='?', - help='Warnings report to display. If not defined, show the most ' - 'recent report.') + help='Warnings report to display. If not defined, show the most ' + 'recent report.') def list(self, directory=None, flags=None, report=None): database = self.database @@ -569,10 +572,10 @@ class Warnings(MachCommandBase): if warning['column'] is not None: print('%s:%d:%d [%s] %s' % (filename, warning['line'], - warning['column'], warning['flag'], warning['message'])) + warning['column'], warning['flag'], warning['message'])) else: print('%s:%d [%s] %s' % (filename, warning['line'], - warning['flag'], warning['message'])) + warning['flag'], warning['message'])) def join_ensure_dir(self, dir1, dir2): dir1 = mozpath.normpath(dir1) @@ -583,48 +586,46 @@ class Warnings(MachCommandBase): print('Specified directory not found.') return None + @CommandProvider class GTestCommands(MachCommandBase): @Command('gtest', category='testing', - description='Run GTest unit tests (C++ tests).') + description='Run GTest unit tests (C++ tests).') @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter', - help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," - "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") + help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," + "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") @CommandArgument('--jobs', '-j', default='1', nargs='?', metavar='jobs', type=int, - help='Run the tests in parallel using multiple processes.') + help='Run the tests in parallel using multiple processes.') @CommandArgument('--tbpl-parser', '-t', action='store_true', - help='Output test results in a format that can be parsed by TBPL.') + help='Output test results in a format that can be parsed by TBPL.') @CommandArgument('--shuffle', '-s', action='store_true', - help='Randomize the execution order of tests.') - + help='Randomize the execution order of tests.') @CommandArgument('--package', - default='org.mozilla.geckoview.test', - help='(Android only) Package name of test app.') + default='org.mozilla.geckoview.test', + help='(Android only) Package name of test app.') @CommandArgument('--adbpath', - dest='adb_path', - help='(Android only) Path to adb binary.') + dest='adb_path', + help='(Android only) Path to adb binary.') @CommandArgument('--deviceSerial', - dest='device_serial', - help="(Android only) adb serial number of remote device. " - "Required when more than one device is connected to the host. " - "Use 'adb devices' to see connected devices.") + dest='device_serial', + help="(Android only) adb serial number of remote device. " + "Required when more than one device is connected to the host. " + "Use 'adb devices' to see connected devices.") @CommandArgument('--remoteTestRoot', - dest='remote_test_root', - help='(Android only) Remote directory to use as test root ' - '(eg. /mnt/sdcard/tests or /data/local/tests).') + dest='remote_test_root', + help='(Android only) Remote directory to use as test root ' + '(eg. /mnt/sdcard/tests or /data/local/tests).') @CommandArgument('--libxul', - dest='libxul_path', - help='(Android only) Path to gtest libxul.so.') - + dest='libxul_path', + help='(Android only) Path to gtest libxul.so.') @CommandArgumentGroup('debugging') @CommandArgument('--debug', action='store_true', group='debugging', - help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') + help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') @CommandArgument('--debugger', default=None, type=str, group='debugging', - help='Name of debugger to use.') + help='Name of debugger to use.') @CommandArgument('--debugger-args', default=None, metavar='params', type=str, - group='debugging', - help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') - + group='debugging', + help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') def gtest(self, shuffle, jobs, gtest_filter, tbpl_parser, package, adb_path, device_serial, remote_test_root, libxul_path, debug, debugger, debugger_args): @@ -669,10 +670,10 @@ class GTestCommands(MachCommandBase): print("One or more Android-only options will be ignored") app_path = self.get_binary_path('app') - args = [app_path, '-unittest', '--gtest_death_test_style=threadsafe']; + args = [app_path, '-unittest', '--gtest_death_test_style=threadsafe'] if sys.platform.startswith('win') and \ - 'MOZ_LAUNCHER_PROCESS' in self.defines: + 'MOZ_LAUNCHER_PROCESS' in self.defines: args.append('--wait-for-browser') if debug or debugger or debugger_args: @@ -710,6 +711,7 @@ class GTestCommands(MachCommandBase): from mozprocess import ProcessHandlerMixin import functools + def handle_line(job_id, line): # Prepend the jobId line = '[%d] %s' % (job_id + 1, line.strip()) @@ -720,10 +722,11 @@ class GTestCommands(MachCommandBase): for i in range(0, jobs): gtest_env["GTEST_SHARD_INDEX"] = str(i) processes[i] = ProcessHandlerMixin([app_path, "-unittest"], - cwd=cwd, - env=gtest_env, - processOutputLine=[functools.partial(handle_line, i)], - universal_newlines=True) + cwd=cwd, + env=gtest_env, + processOutputLine=[ + functools.partial(handle_line, i)], + universal_newlines=True) processes[i].run() exit_code = 0 @@ -808,10 +811,11 @@ class GTestCommands(MachCommandBase): args = [debuggerInfo.path] + debuggerInfo.args + args return args + @CommandProvider class ClangCommands(MachCommandBase): @Command('clang-complete', category='devenv', - description='Generate a .clang_complete file.') + description='Generate a .clang_complete file.') def clang_complete(self): import shlex @@ -866,9 +870,9 @@ class Package(MachCommandBase): """Package the built product for distribution.""" @Command('package', category='post-build', - description='Package the built product for distribution as an APK, DMG, etc.') + description='Package the built product for distribution as an APK, DMG, etc.') @CommandArgument('-v', '--verbose', action='store_true', - help='Verbose output for what commands the packaging process is running.') + help='Verbose output for what commands the packaging process is running.') def package(self, verbose=False): ret = self._run_make(directory=".", target='package', silent=not verbose, ensure_exit_code=False) @@ -876,14 +880,15 @@ class Package(MachCommandBase): self.notify('Packaging complete') return ret + @CommandProvider class Install(MachCommandBase): """Install a package.""" @Command('install', category='post-build', - description='Install the package on the machine, or on a device.') + description='Install the package on the machine, or on a device.') @CommandArgument('--verbose', '-v', action='store_true', - help='Print verbose output when installing to an Android emulator.') + help='Print verbose output when installing to an Android emulator.') def install(self, verbose=False): if conditions.is_android(self): from mozrunner.devices.android_device import verify_android_device @@ -893,6 +898,7 @@ class Install(MachCommandBase): self.notify('Install complete') return ret + @SettingsProvider class RunSettings(): config_settings = [ @@ -903,6 +909,7 @@ single quoted to force them to be strings. """.strip()), ] + @CommandProvider class RunProgram(MachCommandBase): """Run the compiled program.""" @@ -910,51 +917,49 @@ class RunProgram(MachCommandBase): prog_group = 'the compiled program' @Command('run', category='post-build', - description='Run the compiled program, possibly under a debugger or DMD.') + description='Run the compiled program, possibly under a debugger or DMD.') @CommandArgument('params', nargs='...', group=prog_group, - help='Command-line arguments to be passed through to the program. Not specifying a --profile or -P option will result in a temporary profile being used.') + help='Command-line arguments to be passed through to the program. Not specifying a --profile or -P option will result in a temporary profile being used.') @CommandArgumentGroup(prog_group) @CommandArgument('--remote', '-r', action='store_true', group=prog_group, - help='Do not pass the --no-remote argument by default.') + help='Do not pass the --no-remote argument by default.') @CommandArgument('--background', '-b', action='store_true', group=prog_group, - help='Do not pass the --foreground argument by default on Mac.') + help='Do not pass the --foreground argument by default on Mac.') @CommandArgument('--noprofile', '-n', action='store_true', group=prog_group, - help='Do not pass the --profile argument by default.') + help='Do not pass the --profile argument by default.') @CommandArgument('--disable-e10s', action='store_true', group=prog_group, - help='Run the program with electrolysis disabled.') + help='Run the program with electrolysis disabled.') @CommandArgument('--enable-crash-reporter', action='store_true', group=prog_group, - help='Run the program with the crash reporter enabled.') + help='Run the program with the crash reporter enabled.') @CommandArgument('--setpref', action='append', default=[], group=prog_group, - help='Set the specified pref before starting the program. Can be set multiple times. Prefs can also be set in ~/.mozbuild/machrc in the [runprefs] section - see `./mach settings` for more information.') + help='Set the specified pref before starting the program. Can be set multiple times. Prefs can also be set in ~/.mozbuild/machrc in the [runprefs] section - see `./mach settings` for more information.') @CommandArgument('--temp-profile', action='store_true', group=prog_group, - help='Run the program using a new temporary profile created inside the objdir.') + help='Run the program using a new temporary profile created inside the objdir.') @CommandArgument('--macos-open', action='store_true', group=prog_group, - help="On macOS, run the program using the open(1) command. Per open(1), the browser is launched \"just as if you had double-clicked the file's icon\". The browser can not be launched under a debugger with this option.") - + help="On macOS, run the program using the open(1) command. Per open(1), the browser is launched \"just as if you had double-clicked the file's icon\". The browser can not be launched under a debugger with this option.") @CommandArgumentGroup('debugging') @CommandArgument('--debug', action='store_true', group='debugging', - help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') + help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') @CommandArgument('--debugger', default=None, type=str, group='debugging', - help='Name of debugger to use.') + help='Name of debugger to use.') @CommandArgument('--debugger-args', default=None, metavar='params', type=str, - group='debugging', - help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') + group='debugging', + help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') @CommandArgument('--debugparams', action=StoreDebugParamsAndWarnAction, - default=None, type=str, dest='debugger_args', group='debugging', - help=argparse.SUPPRESS) - + default=None, type=str, dest='debugger_args', group='debugging', + help=argparse.SUPPRESS) @CommandArgumentGroup('DMD') @CommandArgument('--dmd', action='store_true', group='DMD', - help='Enable DMD. The following arguments have no effect without this.') + help='Enable DMD. The following arguments have no effect without this.') @CommandArgument('--mode', choices=['live', 'dark-matter', 'cumulative', 'scan'], group='DMD', - help='Profiling mode. The default is \'dark-matter\'.') + help='Profiling mode. The default is \'dark-matter\'.') @CommandArgument('--stacks', choices=['partial', 'full'], group='DMD', - help='Allocation stack trace coverage. The default is \'partial\'.') + help='Allocation stack trace coverage. The default is \'partial\'.') @CommandArgument('--show-dump-stats', action='store_true', group='DMD', - help='Show stats when doing dumps.') + help='Show stats when doing dumps.') def run(self, params, remote, background, noprofile, disable_e10s, - enable_crash_reporter, setpref, temp_profile, macos_open, debug, - debugger, debugger_args, dmd, mode, stacks, show_dump_stats): + enable_crash_reporter, setpref, temp_profile, macos_open, debug, + debugger, debugger_args, dmd, mode, stacks, show_dump_stats): if conditions.is_android(self): # Running Firefox for Android is completely different @@ -975,7 +980,7 @@ class RunProgram(MachCommandBase): binpath = self.get_binary_path('app') except Exception as e: print("It looks like your program isn't built.", - "You can run |mach build| to build it.") + "You can run |mach build| to build it.") print(e) return 1 @@ -983,7 +988,7 @@ class RunProgram(MachCommandBase): if macos_open: if debug: print("The browser can not be launched in the debugger " - "when using the macOS open command.") + "when using the macOS open command.") return 1 try: m = re.search(r'^.+\.app', binpath) @@ -991,7 +996,7 @@ class RunProgram(MachCommandBase): args = ['open', apppath, '--args'] except Exception as e: print("Couldn't get the .app path from the binary path. " - "The macOS open option can only be used on macOS") + "The macOS open option can only be used on macOS") print(e) return 1 else: @@ -1110,21 +1115,23 @@ class RunProgram(MachCommandBase): extra_env['DMD'] = '1' return self.run_process(args=args, ensure_exit_code=False, - pass_thru=True, append_env=extra_env) + pass_thru=True, append_env=extra_env) + @CommandProvider class Buildsymbols(MachCommandBase): """Produce a package of debug symbols suitable for use with Breakpad.""" @Command('buildsymbols', category='post-build', - description='Produce a package of Breakpad-format symbols.') + description='Produce a package of Breakpad-format symbols.') def buildsymbols(self): return self._run_make(directory=".", target='buildsymbols', ensure_exit_code=False) + @CommandProvider class Makefiles(MachCommandBase): @Command('empty-makefiles', category='build-dev', - description='Find empty Makefile.in in the tree.') + description='Find empty Makefile.in in the tree.') def empty(self): import pymake.parser import pymake.parserdata @@ -1164,7 +1171,7 @@ class Makefiles(MachCommandBase): relpath = os.path.relpath(path, self.topsrcdir) try: statements = [s for s in pymake.parser.parsefile(path) - if is_statement_relevant(s)] + if is_statement_relevant(s)] if not statements: print(relpath) @@ -1177,17 +1184,18 @@ class Makefiles(MachCommandBase): if f == 'Makefile.in': yield os.path.join(root, f) + @CommandProvider class MachDebug(MachCommandBase): @Command('environment', category='build-dev', - description='Show info about the mach and build environment.') + description='Show info about the mach and build environment.') @CommandArgument('--format', default='pretty', - choices=['pretty', 'configure', 'json'], - help='Print data in the given format.') + choices=['pretty', 'configure', 'json'], + help='Print data in the given format.') @CommandArgument('--output', '-o', type=str, - help='Output to the given file.') + help='Output to the given file.') @CommandArgument('--verbose', '-v', action='store_true', - help='Print verbose output.') + help='Print verbose output.') def environment(self, format, output=None, verbose=False): func = getattr(self, '_environment_%s' % format.replace('.', '_')) @@ -1252,6 +1260,7 @@ class MachDebug(MachCommandBase): def _environment_json(self, out, verbose): import json + class EnvironmentEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, MozbuildObject): @@ -1275,11 +1284,11 @@ class ArtifactSubCommand(SubCommand): after = SubCommand.__call__(self, func) args = [ CommandArgument('--tree', metavar='TREE', type=str, - help='Firefox tree.'), + help='Firefox tree.'), CommandArgument('--job', metavar='JOB', choices=JOB_CHOICES, - help='Build job.'), + help='Build job.'), CommandArgument('--verbose', '-v', action='store_true', - help='Print verbose output.'), + help='Print verbose output.'), ] for arg in args: after = arg(after) @@ -1299,7 +1308,7 @@ class PackageFrontend(MachCommandBase): """Fetch and install binary artifacts from Mozilla automation.""" @Command('artifact', category='post-build', - description='Use pre-built artifacts to build Firefox.') + description='Use pre-built artifacts to build Firefox.') def artifact(self): '''Download, cache, and install pre-built binary artifacts to build Firefox. @@ -1342,15 +1351,15 @@ class PackageFrontend(MachCommandBase): return artifacts @ArtifactSubCommand('artifact', 'install', - 'Install a good pre-built artifact.') + 'Install a good pre-built artifact.') @CommandArgument('source', metavar='SRC', nargs='?', type=str, - help='Where to fetch and install artifacts from. Can be omitted, in ' - 'which case the current hg repository is inspected; an hg revision; ' - 'a remote URL; or a local file.', - default=None) + help='Where to fetch and install artifacts from. Can be omitted, in ' + 'which case the current hg repository is inspected; an hg revision; ' + 'a remote URL; or a local file.', + default=None) @CommandArgument('--skip-cache', action='store_true', - help='Skip all local caches to force re-fetching remote artifacts.', - default=False) + help='Skip all local caches to force re-fetching remote artifacts.', + default=False) @CommandArgument('--no-tests', action='store_true', help="Don't install tests.") @CommandArgument('--symbols', nargs='?', action=SymbolsAction, help='Download symbols.') @CommandArgument('--host-bins', action='store_true', help='Download host binaries.') @@ -1366,7 +1375,7 @@ class PackageFrontend(MachCommandBase): return artifacts.install_from(source, distdir or self.distdir) @ArtifactSubCommand('artifact', 'clear-cache', - 'Delete local artifacts and reset local artifact cache.') + 'Delete local artifacts and reset local artifact cache.') def artifact_clear_cache(self, tree=None, job=None, verbose=False): self._set_log_level(verbose) artifacts = self._make_artifacts(tree=tree, job=job) @@ -1375,35 +1384,35 @@ class PackageFrontend(MachCommandBase): @SubCommand('artifact', 'toolchain') @CommandArgument('--verbose', '-v', action='store_true', - help='Print verbose output.') + help='Print verbose output.') @CommandArgument('--cache-dir', metavar='DIR', - help='Directory where to store the artifacts cache') + help='Directory where to store the artifacts cache') @CommandArgument('--skip-cache', action='store_true', - help='Skip all local caches to force re-fetching remote artifacts.', - default=False) + help='Skip all local caches to force re-fetching remote artifacts.', + default=False) @CommandArgument('--from-build', metavar='BUILD', nargs='+', - help='Download toolchains resulting from the given build(s); ' - 'BUILD is a name of a toolchain task, e.g. linux64-clang') + help='Download toolchains resulting from the given build(s); ' + 'BUILD is a name of a toolchain task, e.g. linux64-clang') @CommandArgument('--tooltool-manifest', metavar='MANIFEST', - help='Explicit tooltool manifest to process') + help='Explicit tooltool manifest to process') @CommandArgument('--authentication-file', metavar='FILE', - help='Use the RelengAPI token found in the given file to authenticate') + help='Use the RelengAPI token found in the given file to authenticate') @CommandArgument('--tooltool-url', metavar='URL', - help='Use the given url as tooltool server') + help='Use the given url as tooltool server') @CommandArgument('--no-unpack', action='store_true', - help='Do not unpack any downloaded file') + help='Do not unpack any downloaded file') @CommandArgument('--retry', type=int, default=4, - help='Number of times to retry failed downloads') + help='Number of times to retry failed downloads') @CommandArgument('--artifact-manifest', metavar='FILE', - help='Store a manifest about the downloaded taskcluster artifacts') + help='Store a manifest about the downloaded taskcluster artifacts') @CommandArgument('files', nargs='*', - help='A list of files to download, in the form path@task-id, in ' - 'addition to the files listed in the tooltool manifest.') + help='A list of files to download, in the form path@task-id, in ' + 'addition to the files listed in the tooltool manifest.') def artifact_toolchain(self, verbose=False, cache_dir=None, - skip_cache=False, from_build=(), - tooltool_manifest=None, authentication_file=None, - tooltool_url=None, no_unpack=False, retry=None, - artifact_manifest=None, files=()): + skip_cache=False, from_build=(), + tooltool_manifest=None, authentication_file=None, + tooltool_url=None, no_unpack=False, retry=None, + artifact_manifest=None, files=()): '''Download, cache and install pre-built toolchains. ''' from mozbuild.artifacts import ArtifactCache @@ -1491,7 +1500,7 @@ class PackageFrontend(MachCommandBase): name = os.path.basename(artifact_name) artifact_url = get_artifact_url(task_id, artifact_name, - use_proxy=not artifact_name.startswith('public/')) + use_proxy=not artifact_name.startswith('public/')) super(ArtifactRecord, self).__init__( artifact_url, name, None, digest, algorithm, unpack=True) @@ -1667,6 +1676,7 @@ class PackageFrontend(MachCommandBase): return 0 + class StaticAnalysisSubCommand(SubCommand): def __call__(self, func): after = SubCommand.__call__(self, func) @@ -1823,7 +1833,7 @@ class StaticAnalysis(MachCommandBase): if self._is_version_eligible() is False: self.log(logging.ERROR, 'static-analysis', {}, "You're using an old version of clang-format binary." - " Please update to a more recent one by running: './mach bootstrap'") + " Please update to a more recent one by running: './mach bootstrap'") return 1 rc = self._build_compile_db(verbose=verbose) @@ -1851,8 +1861,8 @@ class StaticAnalysis(MachCommandBase): if not total: self.log(logging.INFO, 'static-analysis', {}, - "There are no files eligible for analysis. Please note that 'header' files " - "cannot be used for analysis since they do not consist compilation units.") + "There are no files eligible for analysis. Please note that 'header' files " + "cannot be used for analysis since they do not consist compilation units.") return 0 cwd = self.topobjdir @@ -1862,11 +1872,13 @@ class StaticAnalysis(MachCommandBase): args = self._get_clang_tidy_command( checks=checks, header_filter=header_filter, sources=source, jobs=jobs, fix=fix) - monitor = StaticAnalysisMonitor(self.topsrcdir, self.topobjdir, self._clang_tidy_config, total) + monitor = StaticAnalysisMonitor( + self.topsrcdir, self.topobjdir, self._clang_tidy_config, total) footer = StaticAnalysisFooter(self.log_manager.terminal, monitor) with StaticAnalysisOutputManager(self.log_manager, monitor, footer) as output_manager: - rc = self.run_process(args=args, ensure_exit_code=False, line_handler=output_manager.on_line, cwd=cwd) + rc = self.run_process(args=args, ensure_exit_code=False, + line_handler=output_manager.on_line, cwd=cwd) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_db)}, @@ -1904,7 +1916,8 @@ class StaticAnalysis(MachCommandBase): self.log_manager.enable_all_structured_loggers() if 'MOZ_AUTOMATION' not in os.environ: - self.log(logging.INFO, 'static-analysis', {}, 'Coverity based static-analysis cannot be ran outside automation.') + self.log(logging.INFO, 'static-analysis', {}, + 'Coverity based static-analysis cannot be ran outside automation.') return # Use outgoing files instead of source files @@ -1914,7 +1927,8 @@ class StaticAnalysis(MachCommandBase): source = map(os.path.abspath, files) if len(source) == 0: - self.log(logging.ERROR, 'static-analysis', {}, 'There are no files that coverity can use to scan.') + self.log(logging.ERROR, 'static-analysis', {}, + 'There are no files that coverity can use to scan.') return 0 rc = self._build_compile_db(verbose=verbose) @@ -1925,7 +1939,8 @@ class StaticAnalysis(MachCommandBase): commands_list = self.get_files_with_commands(source) if len(commands_list) == 0: - self.log(logging.INFO, 'static-analysis', {}, 'There are no files that need to be analyzed.') + self.log(logging.INFO, 'static-analysis', {}, + 'There are no files that need to be analyzed.') return 0 # Load the configuration file for coverity static-analysis @@ -1939,31 +1954,37 @@ class StaticAnalysis(MachCommandBase): # First run cov-run-desktop --setup in order to setup the analysis env cmd = [self.cov_run_desktop, '--setup'] - self.log(logging.INFO, 'static-analysis', {}, 'Running {} --setup'.format(self.cov_run_desktop)) + self.log(logging.INFO, 'static-analysis', {}, + 'Running {} --setup'.format(self.cov_run_desktop)) rc = self.run_process(args=cmd, cwd=self.cov_path, pass_thru=True) if rc != 0: - self.log(logging.ERROR, 'static-analysis', {}, 'Running {} --setup failed!'.format(self.cov_run_desktop)) + self.log(logging.ERROR, 'static-analysis', {}, + 'Running {} --setup failed!'.format(self.cov_run_desktop)) return rc # Run cov-configure for clang cmd = [self.cov_configure, '--clang'] - self.log(logging.INFO, 'static-analysis', {}, 'Running {} --clang'.format(self.cov_configure)) + self.log(logging.INFO, 'static-analysis', {}, + 'Running {} --clang'.format(self.cov_configure)) rc = self.run_process(args=cmd, cwd=self.cov_path, pass_thru=True) if rc != 0: - self.log(logging.ERROR, 'static-analysis', {}, 'Running {} --clang failed!'.format(self.cov_configure)) + self.log(logging.ERROR, 'static-analysis', {}, + 'Running {} --clang failed!'.format(self.cov_configure)) return rc # For each element in commands_list run `cov-translate` for element in commands_list: cmd = [self.cov_translate, '--dir', self.cov_idir_path] + element['command'].split(' ') - self.log(logging.INFO, 'static-analysis', {}, 'Running Coverity Tranlate for {}'.format(cmd)) + self.log(logging.INFO, 'static-analysis', {}, + 'Running Coverity Tranlate for {}'.format(cmd)) rc = self.run_process(args=cmd, cwd=element['directory'], pass_thru=True) if rc != 0: - self.log(logging.ERROR, 'static-analysis', {}, 'Running Coverity Tranlate failed for {}'.format(cmd)) + self.log(logging.ERROR, 'static-analysis', {}, + 'Running Coverity Tranlate failed for {}'.format(cmd)) return cmd if coverity_output_path is None: @@ -1973,7 +1994,8 @@ class StaticAnalysis(MachCommandBase): # Once the capture is performed we need to do the actual Coverity Desktop analysis cmd = [self.cov_run_desktop, '--json-output-v6', cov_result, '--analyze-captured-source'] - self.log(logging.INFO, 'static-analysis', {}, 'Running Coverity Analysis for {}'.format(cmd)) + self.log(logging.INFO, 'static-analysis', {}, + 'Running Coverity Analysis for {}'.format(cmd)) rc = self.run_process(cmd, cwd=self.cov_state_path, pass_thru=True) if rc != 0: self.log(logging.ERROR, 'static-analysis', {}, 'Coverity Analysis failed!') @@ -1984,22 +2006,22 @@ class StaticAnalysis(MachCommandBase): def get_reliability_index_for_cov_checker(self, checker_name): if self._cov_config is None: self.log(logging.INFO, 'static-analysis', {}, 'Coverity config file not found, ' - 'using default-value \'reliablity\' = medium. for checker {}'.format(checker_name)) + 'using default-value \'reliablity\' = medium. for checker {}'.format(checker_name)) return 'medium' checkers = self._cov_config['coverity_checkers'] if checker_name not in checkers: self.log(logging.INFO, 'static-analysis', {}, - 'Coverity checker {} not found to determine reliability index. ' - 'For the moment we shall use the default \'reliablity\' = medium.'.format(checker_name)) + 'Coverity checker {} not found to determine reliability index. ' + 'For the moment we shall use the default \'reliablity\' = medium.'.format(checker_name)) return 'medium' if 'reliability' not in checkers[checker_name]: # This checker doesn't have a reliability index self.log(logging.INFO, 'static-analysis', {}, - 'Coverity checker {} doesn\'t have a reliability index set, ' - 'field \'reliability is missing\', please cosinder adding it. ' - 'For the moment we shall use the default \'reliablity\' = medium.'.format(checker_name)) + 'Coverity checker {} doesn\'t have a reliability index set, ' + 'field \'reliability is missing\', please cosinder adding it. ' + 'For the moment we shall use the default \'reliablity\' = medium.'.format(checker_name)) return 'medium' return checkers[checker_name]['reliability'] @@ -2016,7 +2038,8 @@ class StaticAnalysis(MachCommandBase): def build_element(issue): # We look only for main event - event_path = next((event for event in issue['events'] if event['main'] is True), None) + event_path = next( + (event for event in issue['events'] if event['main'] is True), None) dict_issue = { 'line': issue['mainEventLineNumber'], @@ -2060,7 +2083,8 @@ class StaticAnalysis(MachCommandBase): secret_name = 'project/relman/coverity' secrets_url = '{}/secrets/v1/secret/{}'.format(get_root_url(True), secret_name) - self.log(logging.INFO, 'static-analysis', {}, 'Using symbol upload token from the secrets service: "{}"'.format(secrets_url)) + self.log(logging.INFO, 'static-analysis', {}, + 'Using symbol upload token from the secrets service: "{}"'.format(secrets_url)) import requests res = requests.get(secrets_url) @@ -2069,7 +2093,8 @@ class StaticAnalysis(MachCommandBase): cov_config = secret['secret'] if 'secret' in secret else None if cov_config is None: - self.log(logging.ERROR, 'static-analysis', {}, 'Ill formatted secret for Coverity. Aborting analysis.') + self.log(logging.ERROR, 'static-analysis', {}, + 'Ill formatted secret for Coverity. Aborting analysis.') return 1 self.cov_analysis_url = cov_config.get('package_url') @@ -2156,7 +2181,8 @@ class StaticAnalysis(MachCommandBase): self.cov_idir_path = mozpath.join(self.cov_work_path, self.cov_package_ver, 'idir') if not os.path.exists(self.cov_path): - self.log(logging.ERROR, 'static-analysis', {}, 'Missing Coverity in {}'.format(self.cov_path)) + self.log(logging.ERROR, 'static-analysis', {}, + 'Missing Coverity in {}'.format(self.cov_path)) return 1 return 0 @@ -2351,8 +2377,8 @@ class StaticAnalysis(MachCommandBase): config = yaml.safe_load(file_handler) except Exception: self.log(logging.ERROR, 'static-analysis', {}, - 'Looks like config.yaml is not valid, we are going to use default' - ' values for the rest of the analysis for clang-tidy.') + 'Looks like config.yaml is not valid, we are going to use default' + ' values for the rest of the analysis for clang-tidy.') return None return config @@ -2362,8 +2388,8 @@ class StaticAnalysis(MachCommandBase): config = yaml.safe_load(file_handler) except Exception: self.log(logging.ERROR, 'static-analysis', {}, - 'Looks like config.yaml is not valid, we are going to use default' - ' values for the rest of the analysis for coverity.') + 'Looks like config.yaml is not valid, we are going to use default' + ' values for the rest of the analysis for coverity.') return None return config @@ -2376,7 +2402,8 @@ class StaticAnalysis(MachCommandBase): if 'package_version' in self._clang_tidy_config: version = self._clang_tidy_config['package_version'] else: - self.log(logging.ERROR, 'static-analysis', {}, "Unable to find 'package_version' in the config.yml") + self.log(logging.ERROR, 'static-analysis', {}, + "Unable to find 'package_version' in the config.yml") return False # Because the fact that we ship together clang-tidy and clang-format @@ -2410,7 +2437,7 @@ class StaticAnalysis(MachCommandBase): # is considered to be the source in order to limit the diagnostic message to # the source files or folders. common_args += ['-header-filter=%s' % (header_filter - if len(header_filter) else '|'.join(sources))] + if len(header_filter) else '|'.join(sources))] # From our configuration file, config.yaml, we build the configuration list, for # the checkers that are used. These configuration options are used to better fit @@ -2571,7 +2598,8 @@ class StaticAnalysis(MachCommandBase): ignored_checker = item['name'] in ['mozilla-*', '-*'] # 4. List checker_names is passed and the current checker is not part of the # list or 'publish' is False - checker_not_in_list = checker_names and (item['name'] not in checker_names or not_published) + checker_not_in_list = checker_names and ( + item['name'] not in checker_names or not_published) if not_published or \ ignored_platform or \ ignored_checker or \ @@ -2592,7 +2620,8 @@ class StaticAnalysis(MachCommandBase): if error_code != self.TOOLS_SUCCESS: - self.log(logging.INFO, 'static-analysis', {}, "FAIL: the following clang-tidy check(s) failed:") + self.log(logging.INFO, 'static-analysis', {}, + "FAIL: the following clang-tidy check(s) failed:") for failure in checkers_results: checker_error = failure['checker-error'] checker_name = failure['checker-name'] @@ -2605,12 +2634,14 @@ class StaticAnalysis(MachCommandBase): message_to_log = "\tChecker {} not present in this clang-tidy version.".format( checker_name) elif checker_error == self.TOOLS_CHECKER_NO_TEST_FILE: - message_to_log = "\tChecker {0} does not have a test file - {0}.cpp".format(checker_name) + message_to_log = "\tChecker {0} does not have a test file - {0}.cpp".format( + checker_name) elif checker_error == self.TOOLS_CHECKER_RETURNED_NO_ISSUES: message_to_log = "\tChecker {0} did not find any issues in its test file, clang-tidy output for the run is:\n{1}".format( checker_name, info1) elif checker_error == self.TOOLS_CHECKER_RESULT_FILE_NOT_FOUND: - message_to_log = "\tChecker {0} does not have a result file - {0}.json".format(checker_name) + message_to_log = "\tChecker {0} does not have a result file - {0}.json".format( + checker_name) elif checker_error == self.TOOLS_CHECKER_DIFF_FAILED: message_to_log = "\tChecker {0}\nExpected: {1}\nGot: {2}\nclang-tidy output for the run is:\n{3}".format( checker_name, info1, info2, info3) @@ -2647,9 +2678,11 @@ class StaticAnalysis(MachCommandBase): return self._parse_issues(clang_output), clang_output def _run_analysis_batch(self, items): - self.log(logging.INFO, 'static-analysis', {},"RUNNING: clang-tidy checker batch analysis.") + self.log(logging.INFO, 'static-analysis', {}, + "RUNNING: clang-tidy checker batch analysis.") if not len(items): - self.log(logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy checker list is empty!") + self.log(logging.ERROR, 'static-analysis', {}, + "ERROR: clang-tidy checker list is empty!") return self.TOOLS_CHECKER_LIST_EMPTY issues, clang_output = self._run_analysis( @@ -2662,7 +2695,8 @@ class StaticAnalysis(MachCommandBase): failed_checks = [] failed_checks_baseline = [] for checker in items: - test_file_path_json = mozpath.join(self._clang_tidy_base_path, "test", checker) + '.json' + test_file_path_json = mozpath.join( + self._clang_tidy_base_path, "test", checker) + '.json' # Read the pre-determined issues baseline_issues = self._get_autotest_stored_issues(test_file_path_json) @@ -2676,10 +2710,12 @@ class StaticAnalysis(MachCommandBase): failed_checks_baseline.append(baseline_issues) if len(failed_checks) > 0: - self.log(logging.ERROR, 'static-analysis', {}, 'The following check(s) failed for bulk analysis: ' + ' '.join(failed_checks)) + self.log(logging.ERROR, 'static-analysis', {}, + 'The following check(s) failed for bulk analysis: ' + ' '.join(failed_checks)) for failed_check, baseline_issue in zip(failed_checks, failed_checks_baseline): - print('\tChecker {0} expect following results: \n\t\t{1}'.format(failed_check, baseline_issue)) + print('\tChecker {0} expect following results: \n\t\t{1}'.format( + failed_check, baseline_issue)) print('This is the output generated by clang-tidy for the bulk build:\n{}'.format(clang_output)) return self.TOOLS_CHECKER_DIFF_FAILED @@ -2922,7 +2958,7 @@ class StaticAnalysis(MachCommandBase): if path: # Create the full path list - path_maker = lambda f_name: os.path.join(self.topsrcdir, f_name) + def path_maker(f_name): return os.path.join(self.topsrcdir, f_name) path = map(path_maker, path) os.chdir(self.topsrcdir) @@ -2955,7 +2991,7 @@ class StaticAnalysis(MachCommandBase): if self._is_version_eligible() is False: self.log(logging.ERROR, 'static-analysis', {}, "You're using an old version of clang-format binary." - " Please update to a more recent one by running: './mach bootstrap'") + " Please update to a more recent one by running: './mach bootstrap'") return 1 if path is None: @@ -3086,7 +3122,7 @@ class StaticAnalysis(MachCommandBase): checks += ',' + item['name'] except Exception: print('Looks like config.yaml is not valid, so we are unable to ' - 'determine default checkers, using \'-checks=-*,mozilla-*\'') + 'determine default checkers, using \'-checks=-*,mozilla-*\'') checks += ',mozilla-*' finally: return checks @@ -3110,7 +3146,7 @@ class StaticAnalysis(MachCommandBase): checker_config['CheckOptions'] = config_list except Exception: print('Looks like config.yaml is not valid, so we are unable to ' - 'determine configuration for checkers, so using default') + 'determine configuration for checkers, so using default') checker_config = None finally: return checker_config @@ -3221,8 +3257,8 @@ class StaticAnalysis(MachCommandBase): # and start from scratch. shutil.rmtree(self._clang_tools_path) return self._get_clang_tools(force=force, skip_cache=skip_cache, - source=source, verbose=verbose, - download_if_needed=download_if_needed) + source=source, verbose=verbose, + download_if_needed=download_if_needed) # Create base directory where we store clang binary os.mkdir(self._clang_tools_path) @@ -3248,10 +3284,10 @@ class StaticAnalysis(MachCommandBase): currentWorkingDir = os.getcwd() os.chdir(self._clang_tools_path) rc = self._artifact_manager.artifact_toolchain(verbose=verbose, - skip_cache=skip_cache, - from_build=[job], - no_unpack=False, - retry=0) + skip_cache=skip_cache, + from_build=[job], + no_unpack=False, + retry=0) # Change back the cwd os.chdir(currentWorkingDir) @@ -3293,7 +3329,7 @@ class StaticAnalysis(MachCommandBase): args += ['--include', 'glob:**{0}'.format(dot_extension)] args += ['--exclude', 'listfile:{0}'.format(self._format_ignore_file)] else: - commit_range = "HEAD" # All uncommitted changes. + commit_range = "HEAD" # All uncommitted changes. if commit: commit_range = commit if ".." in commit else "{}~..{}".format(commit, commit) args = ["git", "diff", "--no-color", "-U0", commit_range, "--"] @@ -3608,11 +3644,11 @@ class Vendor(MachCommandBase): @SubCommand('vendor', 'rust', description='Vendor rust crates from crates.io into third_party/rust') @CommandArgument('--ignore-modified', action='store_true', - help='Ignore modified files in current checkout', - default=False) + help='Ignore modified files in current checkout', + default=False) @CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true', - help='Permit overly-large files to be added to the repository', - default=False) + help='Permit overly-large files to be added to the repository', + default=False) def vendor_rust(self, **kwargs): from mozbuild.vendor_rust import VendorRust vendor_command = self._spawn(VendorRust) @@ -3621,12 +3657,12 @@ class Vendor(MachCommandBase): @SubCommand('vendor', 'aom', description='Vendor av1 video codec reference implementation into the source repository.') @CommandArgument('-r', '--revision', - help='Repository tag or commit to update to.') + help='Repository tag or commit to update to.') @CommandArgument('--repo', - help='Repository url to pull a snapshot from. Supports github and googlesource.') + help='Repository url to pull a snapshot from. Supports github and googlesource.') @CommandArgument('--ignore-modified', action='store_true', - help='Ignore modified files in current checkout', - default=False) + help='Ignore modified files in current checkout', + default=False) def vendor_aom(self, **kwargs): from mozbuild.vendor_aom import VendorAOM vendor_command = self._spawn(VendorAOM) @@ -3634,12 +3670,12 @@ class Vendor(MachCommandBase): @SubCommand('vendor', 'dav1d', description='Vendor dav1d implementation of AV1 into the source repository.') @CommandArgument('-r', '--revision', - help='Repository tag or commit to update to.') + help='Repository tag or commit to update to.') @CommandArgument('--repo', - help='Repository url to pull a snapshot from. Supports gitlab.') + help='Repository url to pull a snapshot from. Supports gitlab.') @CommandArgument('--ignore-modified', action='store_true', - help='Ignore modified files in current checkout', - default=False) + help='Ignore modified files in current checkout', + default=False) def vendor_dav1d(self, **kwargs): from mozbuild.vendor_dav1d import VendorDav1d vendor_command = self._spawn(VendorDav1d) @@ -3648,8 +3684,8 @@ class Vendor(MachCommandBase): @SubCommand('vendor', 'python', description='Vendor Python packages from pypi.org into third_party/python') @CommandArgument('--with-windows-wheel', action='store_true', - help='Vendor a wheel for Windows along with the source package', - default=False) + help='Vendor a wheel for Windows along with the source package', + default=False) @CommandArgument('packages', default=None, nargs='*', help='Packages to vendor. If omitted, packages and their dependencies defined in Pipfile.lock will be vendored. If Pipfile has been modified, then Pipfile.lock will be regenerated. Note that transient dependencies may be updated when running this command.') def vendor_python(self, **kwargs): from mozbuild.vendor_python import VendorPython @@ -3668,21 +3704,22 @@ class Vendor(MachCommandBase): from mozbuild.vendor_manifest import verify_manifests verify_manifests(files) + @CommandProvider class WebRTCGTestCommands(GTestCommands): @Command('webrtc-gtest', category='testing', - description='Run WebRTC.org GTest unit tests.') + description='Run WebRTC.org GTest unit tests.') @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter', - help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," - "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") + help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," + "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") @CommandArgumentGroup('debugging') @CommandArgument('--debug', action='store_true', group='debugging', - help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') + help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') @CommandArgument('--debugger', default=None, type=str, group='debugging', - help='Name of debugger to use.') + help='Name of debugger to use.') @CommandArgument('--debugger-args', default=None, metavar='params', type=str, - group='debugging', - help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') + group='debugging', + help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') def gtest(self, gtest_filter, debug, debugger, debugger_args): app_path = self.get_binary_path('webrtc-gtest') @@ -3714,6 +3751,7 @@ class WebRTCGTestCommands(GTestCommands): ensure_exit_code=False, pass_thru=True) + @CommandProvider class Repackage(MachCommandBase): '''Repackages artifacts into different formats. @@ -3730,9 +3768,9 @@ class Repackage(MachCommandBase): @SubCommand('repackage', 'dmg', description='Repackage a tar file into a .dmg for OSX') @CommandArgument('--input', '-i', type=str, required=True, - help='Input filename') + help='Input filename') @CommandArgument('--output', '-o', type=str, required=True, - help='Output filename') + help='Output filename') def repackage_dmg(self, input, output): if not os.path.exists(input): print('Input file does not exist: %s' % input) @@ -3749,19 +3787,19 @@ class Repackage(MachCommandBase): @SubCommand('repackage', 'installer', description='Repackage into a Windows installer exe') @CommandArgument('--tag', type=str, required=True, - help='The .tag file used to build the installer') + help='The .tag file used to build the installer') @CommandArgument('--setupexe', type=str, required=True, - help='setup.exe file inside the installer') + help='setup.exe file inside the installer') @CommandArgument('--package', type=str, required=False, - help='Optional package .zip for building a full installer') + help='Optional package .zip for building a full installer') @CommandArgument('--output', '-o', type=str, required=True, - help='Output filename') + help='Output filename') @CommandArgument('--package-name', type=str, required=False, - help='Name of the package being rebuilt') + help='Name of the package being rebuilt') @CommandArgument('--sfx-stub', type=str, required=True, - help='Path to the self-extraction stub.') + help='Path to the self-extraction stub.') @CommandArgument('--use-upx', required=False, action='store_true', - help='Run UPX on the self-extraction stub.') + help='Run UPX on the self-extraction stub.') def repackage_installer(self, tag, setupexe, package, output, package_name, sfx_stub, use_upx): from mozbuild.repackaging.installer import repackage_installer repackage_installer( @@ -3778,21 +3816,21 @@ class Repackage(MachCommandBase): @SubCommand('repackage', 'msi', description='Repackage into a MSI') @CommandArgument('--wsx', type=str, required=True, - help='The wsx file used to build the installer') + help='The wsx file used to build the installer') @CommandArgument('--version', type=str, required=True, - help='The Firefox version used to create the installer') + help='The Firefox version used to create the installer') @CommandArgument('--locale', type=str, required=True, - help='The locale of the installer') + help='The locale of the installer') @CommandArgument('--arch', type=str, required=True, - help='The archtecture you are building.') + help='The archtecture you are building.') @CommandArgument('--setupexe', type=str, required=True, - help='setup.exe installer') + help='setup.exe installer') @CommandArgument('--candle', type=str, required=False, - help='location of candle binary') + help='location of candle binary') @CommandArgument('--light', type=str, required=False, - help='location of light binary') + help='location of light binary') @CommandArgument('--output', '-o', type=str, required=True, - help='Output filename') + help='Output filename') def repackage_msi(self, wsx, version, locale, arch, setupexe, candle, light, output): from mozbuild.repackaging.msi import repackage_msi repackage_msi( @@ -3810,32 +3848,33 @@ class Repackage(MachCommandBase): @SubCommand('repackage', 'mar', description='Repackage into complete MAR file') @CommandArgument('--input', '-i', type=str, required=True, - help='Input filename') + help='Input filename') @CommandArgument('--mar', type=str, required=True, - help='Mar binary path') + help='Mar binary path') @CommandArgument('--output', '-o', type=str, required=True, - help='Output filename') + help='Output filename') @CommandArgument('--format', type=str, default='lzma', - choices=('lzma', 'bz2'), - help='Mar format') + choices=('lzma', 'bz2'), + help='Mar format') @CommandArgument('--arch', type=str, required=True, - help='The archtecture you are building.') + help='The archtecture you are building.') def repackage_mar(self, input, mar, output, format, arch): from mozbuild.repackaging.mar import repackage_mar repackage_mar(self.topsrcdir, input, mar, output, format, arch=arch) + @CommandProvider class Analyze(MachCommandBase): """ Get information about a file in the build graph """ @Command('analyze', category='misc', - description='Analyze the build graph.') + description='Analyze the build graph.') def analyze(self): print("Usage: ./mach analyze [files|report] [args...]") @SubCommand('analyze', 'files', - description='Get incremental build cost for file(s) from the tup database.') + description='Get incremental build cost for file(s) from the tup database.') @CommandArgument('--path', help='Path to tup db', - default=None) + default=None) @CommandArgument('files', nargs='*', help='Files to analyze') def analyze_files(self, path, files): from mozbuild.analyze.graph import Graph @@ -3851,16 +3890,16 @@ class Analyze(MachCommandBase): return 1 @SubCommand('analyze', 'all', - description='Get a report of files changed within the last n days and their corresponding build cost.') + description='Get a report of files changed within the last n days and their corresponding build cost.') @CommandArgument('--days', '-d', type=int, default=14, - help='Number of days to include in the report.') + help='Number of days to include in the report.') @CommandArgument('--format', default='pretty', - choices=['pretty', 'csv', 'json', 'html'], - help='Print or export data in the given format.') + choices=['pretty', 'csv', 'json', 'html'], + help='Print or export data in the given format.') @CommandArgument('--limit', type=int, default=None, - help='Get the top n most expensive files from the report.') + help='Get the top n most expensive files from the report.') @CommandArgument('--path', help='Path to cost_dict.gz', - default=None) + default=None) def analyze_report(self, days, format, limit, path): from mozbuild.analyze.hg import Report self._activate_virtualenv() @@ -3969,7 +4008,8 @@ class L10NCommands(MachCommandBase): self.log(logging.INFO, 'package-multi-locale', {}, 'Invoking `mach android archive-geckoview`') self.run_process( - [mozpath.join(self.topsrcdir, 'mach'), 'android', 'archive-geckoview'.format(locale)], + [mozpath.join(self.topsrcdir, 'mach'), 'android', + 'archive-geckoview'.format(locale)], append_env=append_env, pass_thru=True, ensure_exit_code=True, diff --git a/python/mozbuild/mozbuild/makeutil.py b/python/mozbuild/mozbuild/makeutil.py index fcd45bed23aa..5e238df09565 100644 --- a/python/mozbuild/mozbuild/makeutil.py +++ b/python/mozbuild/mozbuild/makeutil.py @@ -62,6 +62,7 @@ class _SimpleOrderedSet(object): It doesn't expose a complete API, and normalizes path separators at insertion. ''' + def __init__(self): self._list = [] self._set = set() @@ -95,6 +96,7 @@ class Rule(object): command2 ... ''' + def __init__(self, targets=[]): self._targets = _SimpleOrderedSet() self._dependencies = _SimpleOrderedSet() @@ -175,6 +177,7 @@ def read_dep_makefile(fh): if rule: raise Exception('Makefile finishes with a backslash. Expected more input.') + def write_dep_makefile(fh, target, deps): ''' Write a Makefile containing only target's dependencies to the file handle diff --git a/python/mozbuild/mozbuild/moz_yaml.py b/python/mozbuild/mozbuild/moz_yaml.py index c8805e1f9bee..2ddf7674a351 100644 --- a/python/mozbuild/mozbuild/moz_yaml.py +++ b/python/mozbuild/mozbuild/moz_yaml.py @@ -306,6 +306,7 @@ def _schema_1_additional(filename, manifest, require_license_file=True): class License(object): """Voluptuous validator which verifies the license(s) are valid as per our whitelist.""" + def __call__(self, values): if isinstance(values, str): values = [values] diff --git a/python/mozbuild/mozbuild/mozconfig.py b/python/mozbuild/mozbuild/mozconfig.py index 2eb8d74615fa..ae3846dd47cb 100644 --- a/python/mozbuild/mozbuild/mozconfig.py +++ b/python/mozbuild/mozbuild/mozconfig.py @@ -66,7 +66,7 @@ class MozconfigLoader(object): \s* [?:]?= \s* # Assignment operator surrounded by optional # spaces (?P.*$)''', # Everything else (likely the value) - re.VERBOSE) + re.VERBOSE) # Default mozconfig files in the topsrcdir. DEFAULT_TOPSRCDIR_PATHS = ('.mozconfig', 'mozconfig') @@ -145,7 +145,7 @@ class MozconfigLoader(object): 'does not exist in any of ' + ', '.join(potential_roots)) env_path = os.path.join(existing[0], env_path) - elif not os.path.exists(env_path): # non-relative path + elif not os.path.exists(env_path): # non-relative path raise MozconfigFindException( 'MOZCONFIG environment variable refers to a path that ' 'does not exist: ' + env_path) @@ -156,12 +156,12 @@ class MozconfigLoader(object): 'non-file: ' + env_path) srcdir_paths = [os.path.join(self.topsrcdir, p) for p in - self.DEFAULT_TOPSRCDIR_PATHS] + self.DEFAULT_TOPSRCDIR_PATHS] existing = [p for p in srcdir_paths if os.path.isfile(p)] if env_path is None and len(existing) > 1: raise MozconfigFindException('Multiple default mozconfig files ' - 'present. Remove all but one. ' + ', '.join(existing)) + 'present. Remove all but one. ' + ', '.join(existing)) path = None @@ -175,12 +175,12 @@ class MozconfigLoader(object): return os.path.abspath(path) deprecated_paths = [os.path.join(self.topsrcdir, s) for s in - self.DEPRECATED_TOPSRCDIR_PATHS] + self.DEPRECATED_TOPSRCDIR_PATHS] home = env.get('HOME', None) if home is not None: deprecated_paths.extend([os.path.join(home, s) for s in - self.DEPRECATED_HOME_PATHS]) + self.DEPRECATED_HOME_PATHS]) for path in deprecated_paths: if os.path.exists(path): @@ -243,7 +243,7 @@ class MozconfigLoader(object): # We need to capture stderr because that's where the shell sends # errors if execution fails. output = subprocess.check_output(command, stderr=subprocess.STDOUT, - cwd=self.topsrcdir, env=env) + cwd=self.topsrcdir, env=env) except subprocess.CalledProcessError as e: lines = e.output.splitlines() @@ -306,7 +306,7 @@ class MozconfigLoader(object): # Environment variables also appear as shell variables, but that's # uninteresting duplication of information. Filter them out. - filt = lambda x, y: {k: v for k, v in x.items() if k not in y} + def filt(x, y): return {k: v for k, v in x.items() if k not in y} result['vars'] = diff_vars( filt(parsed['vars_before'], parsed['env_before']), filt(parsed['vars_after'], parsed['env_after']) diff --git a/python/mozbuild/mozbuild/mozinfo.py b/python/mozbuild/mozbuild/mozinfo.py index e74eef261da9..56a1df51a124 100755 --- a/python/mozbuild/mozbuild/mozinfo.py +++ b/python/mozbuild/mozbuild/mozinfo.py @@ -58,7 +58,7 @@ def build_dict(config, env=os.environ): # processor p = substs["TARGET_CPU"] # do some slight massaging for some values - #TODO: retain specific values in case someone wants them? + # TODO: retain specific values in case someone wants them? if p.startswith("arm"): p = "arm" elif re.match("i[3-9]86", p): diff --git a/python/mozbuild/mozbuild/preprocessor.py b/python/mozbuild/mozbuild/preprocessor.py index 200651712ab5..0732e68ecd88 100644 --- a/python/mozbuild/mozbuild/preprocessor.py +++ b/python/mozbuild/mozbuild/preprocessor.py @@ -198,6 +198,7 @@ class Expression: rv = not rv return rv # Helper function to evaluate __get_logical_and and __get_logical_or results + def eval_logical_op(tok): left = opmap[tok[0].type](tok[0]) right = opmap[tok[2].type](tok[2]) @@ -217,12 +218,13 @@ class Expression: 'defined': lambda tok: tok.value in context, 'int': lambda tok: tok.value} - return opmap[self.e.type](self.e); + return opmap[self.e.type](self.e) class __AST(list): """ Internal class implementing Abstract Syntax Tree nodes """ + def __init__(self, type): self.type = type super(self.__class__, self).__init__(self) @@ -231,11 +233,14 @@ class Expression: """ Internal class implementing Abstract Syntax Tree leafs """ + def __init__(self, type, value): self.value = value self.type = type + def __str__(self): return self.value.__str__() + def __repr__(self): return self.value.__repr__() @@ -245,13 +250,16 @@ class Expression: It has two members, offset and content, which give the offset of the error and the offending content. """ + def __init__(self, expression): self.offset = expression.offset self.content = expression.content[:3] + def __str__(self): return 'Unexpected content at offset {0}, "{1}"'.format(self.offset, self.content) + class Context(dict): """ This class holds variable values by subclassing dict, and while it @@ -266,6 +274,7 @@ class Context(dict): to reflect the ambiguity between string literals and preprocessor variables. """ + def __getitem__(self, key): if key in self: return super(self.__class__, self).__getitem__(key) @@ -285,9 +294,9 @@ class Preprocessor: def __init__(self, defines=None, marker='#'): self.context = Context() - for k,v in {'FILE': '', - 'LINE': 0, - 'DIRECTORY': os.path.abspath('.')}.iteritems(): + for k, v in {'FILE': '', + 'LINE': 0, + 'DIRECTORY': os.path.abspath('.')}.iteritems(): self.context[k] = v try: # Can import globally because of bootstrapping issues. @@ -341,7 +350,8 @@ class Preprocessor: elif self.actionLevel == 1: msg = 'no useful preprocessor directives found' if msg: - class Fake(object): pass + class Fake(object): + pass fake = Fake() fake.context = { 'FILE': file, @@ -454,7 +464,7 @@ class Preprocessor: self.actionLevel = 2 self.out.write(filteredLine) - def handleCommandLine(self, args, defaultToStdin = False): + def handleCommandLine(self, args, defaultToStdin=False): """ Parse a commandline into this parser. Uses OptionParser internally, no args mean sys.argv[1:]. @@ -505,9 +515,10 @@ class Preprocessor: if options.output: out.close() - def getCommandLineParser(self, unescapeDefines = False): + def getCommandLineParser(self, unescapeDefines=False): escapedValue = re.compile('".*"$') numberValue = re.compile('\d+$') + def handleD(option, opt, value, parser): vals = value.split('=', 1) if len(vals) == 1: @@ -518,12 +529,16 @@ class Preprocessor: elif numberValue.match(vals[1]): vals[1] = int(vals[1]) self.context[vals[0]] = vals[1] + def handleU(option, opt, value, parser): del self.context[value] + def handleF(option, opt, value, parser): self.do_filter(value) + def handleMarker(option, opt, value, parser): self.setMarker(value) + def handleSilenceDirectiveWarnings(option, opt, value, parse): self.setSilenceDirectiveWarnings(True) p = OptionParser() @@ -534,7 +549,7 @@ class Preprocessor: p.add_option('-F', action='callback', callback=handleF, type="string", metavar="FILTER", help='Enable the specified filter') p.add_option('-o', '--output', type="string", default=None, - metavar="FILENAME", help='Output to the specified file '+ + metavar="FILENAME", help='Output to the specified file ' + 'instead of stdout') p.add_option('--depend', type="string", default=None, metavar="FILENAME", help='Generate dependencies in the given file') @@ -586,6 +601,7 @@ class Preprocessor: except: pass self.context[m.group('name')] = val + def do_undef(self, args): m = re.match('(?P\w+)$', args, re.U) if not m: @@ -593,9 +609,11 @@ class Preprocessor: if args in self.context: del self.context[args] # Logic + def ensure_not_else(self): if len(self.ifStates) == 0 or self.ifStates[-1] == 2: sys.stderr.write('WARNING: bad nesting of #else in %s\n' % self.context['FILE']) + def do_if(self, args, replace=False): if self.disableLevel and not replace: self.disableLevel += 1 @@ -619,6 +637,7 @@ class Preprocessor: else: self.ifStates.append(self.disableLevel) pass + def do_ifdef(self, args, replace=False): if self.disableLevel and not replace: self.disableLevel += 1 @@ -634,6 +653,7 @@ class Preprocessor: else: self.ifStates.append(self.disableLevel) pass + def do_ifndef(self, args, replace=False): if self.disableLevel and not replace: self.disableLevel += 1 @@ -649,51 +669,60 @@ class Preprocessor: else: self.ifStates.append(self.disableLevel) pass - def do_else(self, args, ifState = 2): + + def do_else(self, args, ifState=2): self.ensure_not_else() hadTrue = self.ifStates[-1] == 0 - self.ifStates[-1] = ifState # in-else + self.ifStates[-1] = ifState # in-else if hadTrue: self.disableLevel = 1 return self.disableLevel = 0 + def do_elif(self, args): if self.disableLevel == 1: if self.ifStates[-1] == 1: self.do_if(args, replace=True) else: self.do_else(None, self.ifStates[-1]) + def do_elifdef(self, args): if self.disableLevel == 1: if self.ifStates[-1] == 1: self.do_ifdef(args, replace=True) else: self.do_else(None, self.ifStates[-1]) + def do_elifndef(self, args): if self.disableLevel == 1: if self.ifStates[-1] == 1: self.do_ifndef(args, replace=True) else: self.do_else(None, self.ifStates[-1]) + def do_endif(self, args): if self.disableLevel > 0: self.disableLevel -= 1 if self.disableLevel == 0: self.ifStates.pop() # output processing + def do_expand(self, args): lst = re.split('__(\w+)__', args, re.U) do_replace = False + def vsubst(v): if v in self.context: return str(self.context[v]) return '' for i in range(1, len(lst), 2): lst[i] = vsubst(lst[i]) - lst.append('\n') # add back the newline + lst.append('\n') # add back the newline self.write(reduce(lambda x, y: x+y, lst, '')) + def do_literal(self, args): self.write(args + '\n') + def do_filter(self, args): filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)] if len(filters) == 0: @@ -705,6 +734,7 @@ class Preprocessor: filterNames.sort() self.filters = [(fn, current[fn]) for fn in filterNames] return + def do_unfilter(self, args): filters = args.split(' ') current = dict(self.filters) @@ -719,12 +749,14 @@ class Preprocessor: # # emptyLines # Strips blank lines from the output. + def filter_emptyLines(self, aLine): if aLine == '\n': return '' return aLine # slashslash # Strips everything after // + def filter_slashslash(self, aLine): if (aLine.find('//') == -1): return aLine @@ -734,10 +766,12 @@ class Preprocessor: return aLine # spaces # Collapses sequences of spaces into a single space + def filter_spaces(self, aLine): return re.sub(' +', ' ', aLine).strip(' ') # substition # helper to be used by both substition and attemptSubstitution + def filter_substitution(self, aLine, fatal=True): def repl(matchobj): varname = matchobj.group('VAR') @@ -747,9 +781,11 @@ class Preprocessor: raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname) return matchobj.group(0) return self.varsubst.sub(repl, aLine) + def filter_attemptSubstitution(self, aLine): return self.filter_substitution(aLine, fatal=False) # File ops + def do_include(self, args, filters=True): """ Preprocess a given file. @@ -806,15 +842,17 @@ class Preprocessor: self.context['LINE'] = oldLine self.context['DIRECTORY'] = oldDir self.curdir = oldCurdir + def do_includesubst(self, args): args = self.filter_substitution(args) self.do_include(args) + def do_error(self, args): raise Preprocessor.Error(self, 'Error: ', str(args)) def preprocess(includes=[sys.stdin], defines={}, - output = sys.stdout, + output=sys.stdout, marker='#'): pp = Preprocessor(defines=defines, marker=marker) diff --git a/python/mozbuild/mozbuild/repackaging/dmg.py b/python/mozbuild/mozbuild/repackaging/dmg.py index 9868e85697b1..089361cafb04 100644 --- a/python/mozbuild/mozbuild/repackaging/dmg.py +++ b/python/mozbuild/mozbuild/repackaging/dmg.py @@ -11,6 +11,7 @@ import mozpack.path as mozpath from mozpack.dmg import create_dmg from application_ini import get_application_ini_value + def repackage_dmg(infile, output): if not tarfile.is_tarfile(infile): diff --git a/python/mozbuild/mozbuild/repackaging/msi.py b/python/mozbuild/mozbuild/repackaging/msi.py index 14228c83c96a..b1dcedf22803 100644 --- a/python/mozbuild/mozbuild/repackaging/msi.py +++ b/python/mozbuild/mozbuild/repackaging/msi.py @@ -16,16 +16,17 @@ _MSI_ARCH = { 'x86_64': 'x64', } + def update_wsx(wfile, pvalues): parsed = minidom.parse(wfile) # construct a dictinary for the pre-processing options # iterate over that list and add them to the wsx xml doc - for k,v in pvalues.items(): - entry = parsed.createProcessingInstruction('define', k + ' = "' + v + '"') - root = parsed.firstChild - parsed.insertBefore(entry, root) + for k, v in pvalues.items(): + entry = parsed.createProcessingInstruction('define', k + ' = "' + v + '"') + root = parsed.firstChild + parsed.insertBefore(entry, root) # write out xml to new wfile new_w_file = wfile + ".new" fh = open(new_w_file, "wb") @@ -76,8 +77,8 @@ def repackage_msi(topsrcdir, wsx, version, locale, arch, setupexe, candle, light try: wsx_file = os.path.split(wsx)[1] shutil.copy(wsx, tmpdir) - temp_wsx_file = os.path.join(tmpdir, wsx_file) - temp_wsx_file = mozpath.realpath(temp_wsx_file) + temp_wsx_file = os.path.join(tmpdir, wsx_file) + temp_wsx_file = mozpath.realpath(temp_wsx_file) pre_values = {'Vendor': 'Mozilla', 'BrandFullName': 'Mozilla Firefox', 'Version': version, @@ -100,7 +101,7 @@ def repackage_msi(topsrcdir, wsx, version, locale, arch, setupexe, candle, light '-out', wix_installer, wix_object_file] subprocess.check_call(light_cmd, env=env) os.remove(wix_object_file) - #mv file to output dir + # mv file to output dir shutil.move(wix_installer, output) finally: os.chdir(old_cwd) diff --git a/python/mozbuild/mozbuild/shellutil.py b/python/mozbuild/mozbuild/shellutil.py index eaa9510605cb..ba9bb217dc9c 100644 --- a/python/mozbuild/mozbuild/shellutil.py +++ b/python/mozbuild/mozbuild/shellutil.py @@ -22,6 +22,7 @@ def _tokens2re(**tokens): # backslash, captured in the "escape" match group. return re.compile('(?:%s|%s)' % (nonescaped, r'(?P\\\\)')) + UNQUOTED_TOKENS_RE = _tokens2re( whitespace=r'[\t\r\n ]+', quote=r'[\'"]', @@ -54,6 +55,7 @@ class _ClineSplitter(object): Parses a given command line string and creates a list of command and arguments, with wildcard expansion. ''' + def __init__(self, cline): self.arg = None self.cline = cline diff --git a/python/mozbuild/mozbuild/sphinx.py b/python/mozbuild/mozbuild/sphinx.py index 2d9032ec0c5e..a3bafa4394c6 100644 --- a/python/mozbuild/mozbuild/sphinx.py +++ b/python/mozbuild/mozbuild/sphinx.py @@ -187,9 +187,9 @@ def setup(app): # properly. We leverage the in-tree virtualenv for this. topsrcdir = manager.topsrcdir ve = VirtualenvManager(topsrcdir, - os.path.join(topsrcdir, 'dummy-objdir'), - os.path.join(app.outdir, '_venv'), - sys.stderr, - os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt')) + os.path.join(topsrcdir, 'dummy-objdir'), + os.path.join(app.outdir, '_venv'), + sys.stderr, + os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt')) ve.ensure() ve.activate() diff --git a/python/mozbuild/mozbuild/telemetry.py b/python/mozbuild/mozbuild/telemetry.py index 49bd398d84ec..ceb2a31e28af 100644 --- a/python/mozbuild/mozbuild/telemetry.py +++ b/python/mozbuild/mozbuild/telemetry.py @@ -238,6 +238,7 @@ def get_build_attrs(attrs): res['cpu_percent'] = int(round(usage['cpu_percent'])) return res + def filter_args(command, argv, paths): ''' Given the full list of command-line arguments, remove anything up to and including `command`, diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/mozbuild/mozbuild/test/action/test_buildlist.py index 9c2631812d41..7ce07d42d837 100644 --- a/python/mozbuild/mozbuild/test/action/test_buildlist.py +++ b/python/mozbuild/mozbuild/test/action/test_buildlist.py @@ -4,7 +4,10 @@ import unittest -import os, sys, os.path, time +import os +import sys +import os.path +import time from tempfile import mkdtemp from shutil import rmtree import mozunit @@ -13,77 +16,79 @@ from mozbuild.action.buildlist import addEntriesToListFile class TestBuildList(unittest.TestCase): - """ - Unit tests for buildlist.py - """ - def setUp(self): - self.tmpdir = mkdtemp() + """ + Unit tests for buildlist.py + """ - def tearDown(self): - rmtree(self.tmpdir) + def setUp(self): + self.tmpdir = mkdtemp() - # utility methods for tests - def touch(self, file, dir=None): - if dir is None: - dir = self.tmpdir - f = os.path.join(dir, file) - open(f, 'w').close() - return f + def tearDown(self): + rmtree(self.tmpdir) - def assertFileContains(self, filename, l): - """Assert that the lines in the file |filename| are equal - to the contents of the list |l|, in order.""" - l = l[:] - f = open(filename, 'r') - lines = [line.rstrip() for line in f.readlines()] - f.close() - for line in lines: - self.assert_(len(l) > 0, - "ran out of expected lines! (expected '{0}', got '{1}')" - .format(l, lines)) - self.assertEqual(line, l.pop(0)) - self.assert_(len(l) == 0, - "not enough lines in file! (expected '{0}'," - " got '{1}'".format(l, lines)) + # utility methods for tests + def touch(self, file, dir=None): + if dir is None: + dir = self.tmpdir + f = os.path.join(dir, file) + open(f, 'w').close() + return f - def test_basic(self): - "Test that addEntriesToListFile works when file doesn't exist." - testfile = os.path.join(self.tmpdir, "test.list") - l = ["a", "b", "c"] - addEntriesToListFile(testfile, l) - self.assertFileContains(testfile, l) - # ensure that attempting to add the same entries again doesn't change it - addEntriesToListFile(testfile, l) - self.assertFileContains(testfile, l) + def assertFileContains(self, filename, l): + """Assert that the lines in the file |filename| are equal + to the contents of the list |l|, in order.""" + l = l[:] + f = open(filename, 'r') + lines = [line.rstrip() for line in f.readlines()] + f.close() + for line in lines: + self.assert_(len(l) > 0, + "ran out of expected lines! (expected '{0}', got '{1}')" + .format(l, lines)) + self.assertEqual(line, l.pop(0)) + self.assert_(len(l) == 0, + "not enough lines in file! (expected '{0}'," + " got '{1}'".format(l, lines)) - def test_append(self): - "Test adding new entries." - testfile = os.path.join(self.tmpdir, "test.list") - l = ["a", "b", "c"] - addEntriesToListFile(testfile, l) - self.assertFileContains(testfile, l) - l2 = ["x","y","z"] - addEntriesToListFile(testfile, l2) - l.extend(l2) - self.assertFileContains(testfile, l) + def test_basic(self): + "Test that addEntriesToListFile works when file doesn't exist." + testfile = os.path.join(self.tmpdir, "test.list") + l = ["a", "b", "c"] + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + # ensure that attempting to add the same entries again doesn't change it + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) - def test_append_some(self): - "Test adding new entries mixed with existing entries." - testfile = os.path.join(self.tmpdir, "test.list") - l = ["a", "b", "c"] - addEntriesToListFile(testfile, l) - self.assertFileContains(testfile, l) - addEntriesToListFile(testfile, ["a", "x", "c", "z"]) - self.assertFileContains(testfile, ["a", "b", "c", "x", "z"]) + def test_append(self): + "Test adding new entries." + testfile = os.path.join(self.tmpdir, "test.list") + l = ["a", "b", "c"] + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + l2 = ["x", "y", "z"] + addEntriesToListFile(testfile, l2) + l.extend(l2) + self.assertFileContains(testfile, l) + + def test_append_some(self): + "Test adding new entries mixed with existing entries." + testfile = os.path.join(self.tmpdir, "test.list") + l = ["a", "b", "c"] + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + addEntriesToListFile(testfile, ["a", "x", "c", "z"]) + self.assertFileContains(testfile, ["a", "b", "c", "x", "z"]) + + def test_add_multiple(self): + """Test that attempting to add the same entry multiple times results in + only one entry being added.""" + testfile = os.path.join(self.tmpdir, "test.list") + addEntriesToListFile(testfile, ["a", "b", "a", "a", "b"]) + self.assertFileContains(testfile, ["a", "b"]) + addEntriesToListFile(testfile, ["c", "a", "c", "b", "c"]) + self.assertFileContains(testfile, ["a", "b", "c"]) - def test_add_multiple(self): - """Test that attempting to add the same entry multiple times results in - only one entry being added.""" - testfile = os.path.join(self.tmpdir, "test.list") - addEntriesToListFile(testfile, ["a","b","a","a","b"]) - self.assertFileContains(testfile, ["a","b"]) - addEntriesToListFile(testfile, ["c","a","c","b","c"]) - self.assertFileContains(testfile, ["a","b","c"]) if __name__ == '__main__': - mozunit.main() + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/action/test_node.py b/python/mozbuild/mozbuild/test/action/test_node.py index 0e4323a91f46..6eaa07ca9785 100644 --- a/python/mozbuild/mozbuild/test/action/test_node.py +++ b/python/mozbuild/mozbuild/test/action/test_node.py @@ -21,6 +21,7 @@ test_data_path = mozpath.join(test_data_path, 'data', 'node') def data(name): return os.path.join(test_data_path, name) + TEST_SCRIPT = data("node-test-script.js") NONEXISTENT_TEST_SCRIPT = data("non-existent-test-script.js") diff --git a/python/mozbuild/mozbuild/test/action/test_process_install_manifest.py b/python/mozbuild/mozbuild/test/action/test_process_install_manifest.py index a912e3be29b9..7f2cf03befa6 100644 --- a/python/mozbuild/mozbuild/test/action/test_process_install_manifest.py +++ b/python/mozbuild/mozbuild/test/action/test_process_install_manifest.py @@ -70,5 +70,6 @@ class TestGenerateManifest(TestWithTmpDir): self.assertFalse(os.path.exists(self.tmppath('dest/foo/file2'))) self.assertFalse(os.path.exists(self.tmppath('dest/foo/file3'))) + if __name__ == '__main__': mozunit.main() diff --git a/python/mozbuild/mozbuild/test/analyze/test_graph.py b/python/mozbuild/mozbuild/test/analyze/test_graph.py index fe9b99c5679a..fd2c13ec59bd 100644 --- a/python/mozbuild/mozbuild/test/analyze/test_graph.py +++ b/python/mozbuild/mozbuild/test/analyze/test_graph.py @@ -20,40 +20,40 @@ CREATE_NORMAL_LINK = """CREATE TABLE normal_link (from_id integer, to_id integer, unique(from_id, to_id));""" -NODE_DATA = [(1, 0 ,2, -1, '.'), - (2, 100, 0, 1, 'Base64.cpp'), - (3, 200, 0, 1, 'nsArray.cpp'), - (4, 100, 0, 1, 'nsWildCard.h'), - (5, -1, 1, 9426, 'CDD Unified_cpp_xpcom_io0.cpp'), - (6, -1, 1, 5921, 'CXX Unified_cpp_xpcom_ds0.cpp'), - (7, -1, 1, 11077, 'CXX /builds/worker/workspace/build/src/dom/\ +NODE_DATA = [(1, 0, 2, -1, '.'), + (2, 100, 0, 1, 'Base64.cpp'), + (3, 200, 0, 1, 'nsArray.cpp'), + (4, 100, 0, 1, 'nsWildCard.h'), + (5, -1, 1, 9426, 'CDD Unified_cpp_xpcom_io0.cpp'), + (6, -1, 1, 5921, 'CXX Unified_cpp_xpcom_ds0.cpp'), + (7, -1, 1, 11077, 'CXX /builds/worker/workspace/build/src/dom/\ plugins/base/snNPAPIPlugin.cpp'), - (8, -1, 1, 7677, 'CXX Unified_cpp_xpcom_io1.cpp'), - (9, -1, 1, 8672, 'CXX Unified_cpp_modules_libjar0.cpp'), - (10, -1, 4, 1, 'Unified_cpp_xpcom_io0.o'), - (11, -1, 4, 1, 'Unified_cpp_xpcom_dso.o'), - (12, -1, 4, 1, 'nsNPAPIPlugin.o'), - (13, -1, 4, 1, 'Unified_cpp_xpcom_io1.o'), - (14, -1, 4, 1, 'Unified_cpp_modules_libjar0.o'), - (15, -1, 1, 52975, 'LINK libxul.so'), - (16, -1, 4, 1, 'libxul.so'), - (17, -1, 1, 180, 'LINK libtestcrasher.so'), - (18, -1, 1, 944, 'python /builds/worker/workspace/build/src/toolkit/\ + (8, -1, 1, 7677, 'CXX Unified_cpp_xpcom_io1.cpp'), + (9, -1, 1, 8672, 'CXX Unified_cpp_modules_libjar0.cpp'), + (10, -1, 4, 1, 'Unified_cpp_xpcom_io0.o'), + (11, -1, 4, 1, 'Unified_cpp_xpcom_dso.o'), + (12, -1, 4, 1, 'nsNPAPIPlugin.o'), + (13, -1, 4, 1, 'Unified_cpp_xpcom_io1.o'), + (14, -1, 4, 1, 'Unified_cpp_modules_libjar0.o'), + (15, -1, 1, 52975, 'LINK libxul.so'), + (16, -1, 4, 1, 'libxul.so'), + (17, -1, 1, 180, 'LINK libtestcrasher.so'), + (18, -1, 1, 944, 'python /builds/worker/workspace/build/src/toolkit/\ library/dependentlibs.py:gen_list -> [dependentlibs.list, \ dependentlibs.list.gtest, dependentlibs.list.pp]'), - (19, -1, 1, 348, 'LINK ../../dist/bin/plugin-container'), - (20, -1, 1, 342, 'LINK ../../../dist/bin/xpcshell'), - (21, -1, 4, 1, 'libtestcrasher.so'), - (22, -1, 4, 1, 'dependentlibs.list'), - (23, -1, 4, 1, 'dependentlibs.list.gtest'), - (24, -1, 4, 1, 'dependentlibs.list.pp'), - (25, -1, 4, 1, 'plugin-container'), - (26, -1, 4, 1, 'xpcshell'), - (27, -1, 6, 1, ''), - (28, 1, 0, 1, 'dummy node'), - (100, 300, 2, -1, 'io'), - (200, 300, 2, -1, 'ds'), - (300, 1, 2, -1, 'xpcom')] + (19, -1, 1, 348, 'LINK ../../dist/bin/plugin-container'), + (20, -1, 1, 342, 'LINK ../../../dist/bin/xpcshell'), + (21, -1, 4, 1, 'libtestcrasher.so'), + (22, -1, 4, 1, 'dependentlibs.list'), + (23, -1, 4, 1, 'dependentlibs.list.gtest'), + (24, -1, 4, 1, 'dependentlibs.list.pp'), + (25, -1, 4, 1, 'plugin-container'), + (26, -1, 4, 1, 'xpcshell'), + (27, -1, 6, 1, ''), + (28, 1, 0, 1, 'dummy node'), + (100, 300, 2, -1, 'io'), + (200, 300, 2, -1, 'ds'), + (300, 1, 2, -1, 'xpcom')] NORMAL_LINK_DATA = [(2, 5), (3, 6), (4, 7), (4, 8), (4, 9), (5, 10), (6, 11), (7, 12), (8, 13), (9, 14), (10, 15), (11, 15), (12, 15), @@ -63,6 +63,7 @@ NORMAL_LINK_DATA = [(2, 5), (3, 6), (4, 7), (4, 8), (4, 9), (5, 10), (6, 11), PATH_TO_TEST_DB = ':memory:' + class TestGraph(unittest.TestCase): @classmethod def setUpClass(cls): @@ -93,10 +94,10 @@ class TestGraph(unittest.TestCase): self.assertEqual(len(g.get_node(21).cmds), 0) self.assertEqual(len(g.get_node(28).cmds), 0) # one immediate command child - self.assertItemsEqual(g.get_node(2).get_cmd_ids(),[5] + libxul) - self.assertItemsEqual(g.get_node(3).get_cmd_ids(),[6] + libxul) + self.assertItemsEqual(g.get_node(2).get_cmd_ids(), [5] + libxul) + self.assertItemsEqual(g.get_node(3).get_cmd_ids(), [6] + libxul) # multiple immediate command children - self.assertItemsEqual(g.get_node(4).get_cmd_ids(),[7, 8, 9] + libxul) + self.assertItemsEqual(g.get_node(4).get_cmd_ids(), [7, 8, 9] + libxul) # node is not a file or command self.assertItemsEqual(g.get_node(16).get_cmd_ids(), libxul[1:]) self.assertItemsEqual(g.get_node(11).get_cmd_ids(), libxul) @@ -131,5 +132,6 @@ class TestGraph(unittest.TestCase): self.assertEqual(g.get_node(4).path, 'xpcom/io/nsWildCard.h') self.assertEqual(g.get_node(28).path, 'dummy node') + if __name__ == '__main__': - mozunit.main() \ No newline at end of file + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/backend/test_build.py b/python/mozbuild/mozbuild/test/backend/test_build.py index ada6d6e75b7b..1f69a8bc8a5c 100644 --- a/python/mozbuild/mozbuild/test/backend/test_build.py +++ b/python/mozbuild/mozbuild/test/backend/test_build.py @@ -233,5 +233,6 @@ class TestBuild(unittest.TestCase): 'bin/app/modules/foo.jsm': 'foo.jsm\n', }) + if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/backend/test_configenvironment.py b/python/mozbuild/mozbuild/test/backend/test_configenvironment.py index 95593e1869e8..8550ec674b8f 100644 --- a/python/mozbuild/mozbuild/test/backend/test_configenvironment.py +++ b/python/mozbuild/mozbuild/test/backend/test_configenvironment.py @@ -2,7 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import os, posixpath +import os +import posixpath from StringIO import StringIO import unittest from mozunit import main, MockedOpen @@ -39,15 +40,16 @@ class TestEnvironment(unittest.TestCase): and ALLEMPTYSUBSTS. ''' env = ConfigEnvironment('.', '.', - defines = { 'foo': 'bar', 'baz': 'qux 42', - 'abc': "d'e'f", 'extra': 'foobar' }, - non_global_defines = ['extra', 'ignore'], - substs = { 'FOO': 'bar', 'FOOBAR': '', 'ABC': 'def', - 'bar': 'baz qux', 'zzz': '"abc def"', - 'qux': '' }) + defines={'foo': 'bar', 'baz': 'qux 42', + 'abc': "d'e'f", 'extra': 'foobar'}, + non_global_defines=['extra', 'ignore'], + substs={'FOO': 'bar', 'FOOBAR': '', 'ABC': 'def', + 'bar': 'baz qux', 'zzz': '"abc def"', + 'qux': ''}) # non_global_defines should be filtered out in ACDEFINES. # Original order of the defines need to be respected in ACDEFINES - self.assertEqual(env.substs['ACDEFINES'], """-Dabc='d'\\''e'\\''f' -Dbaz='qux 42' -Dfoo=bar""") + self.assertEqual(env.substs['ACDEFINES'], + """-Dabc='d'\\''e'\\''f' -Dbaz='qux 42' -Dfoo=bar""") # Likewise for ALLSUBSTS, which also must contain ACDEFINES self.assertEqual(env.substs['ALLSUBSTS'], '''ABC = def ACDEFINES = -Dabc='d'\\''e'\\''f' -Dbaz='qux 42' -Dfoo=bar diff --git a/python/mozbuild/mozbuild/test/backend/test_fastermake.py b/python/mozbuild/mozbuild/test/backend/test_fastermake.py index eba9110a0ab3..8a9207a2e5f1 100644 --- a/python/mozbuild/mozbuild/test/backend/test_fastermake.py +++ b/python/mozbuild/mozbuild/test/backend/test_fastermake.py @@ -25,15 +25,15 @@ class TestFasterMakeBackend(BackendTester): """Ensure the FasterMakeBackend works without error.""" env = self._consume('stub0', FasterMakeBackend) self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, - 'backend.FasterMakeBackend'))) + 'backend.FasterMakeBackend'))) self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, - 'backend.FasterMakeBackend.in'))) + 'backend.FasterMakeBackend.in'))) def test_final_target_files_wildcard(self): """Ensure that wildcards in FINAL_TARGET_FILES work properly.""" env = self._consume('final-target-files-wildcard', FasterMakeBackend) m = InstallManifest(path=mozpath.join(env.topobjdir, - 'faster', 'install_dist_bin')) + 'faster', 'install_dist_bin')) self.assertEqual(len(m), 1) reg = FileRegistry() m.populate_registry(reg) diff --git a/python/mozbuild/mozbuild/test/backend/test_gn_processor.py b/python/mozbuild/mozbuild/test/backend/test_gn_processor.py index d98766318b95..498543aa1bfc 100644 --- a/python/mozbuild/mozbuild/test/backend/test_gn_processor.py +++ b/python/mozbuild/mozbuild/test/backend/test_gn_processor.py @@ -30,6 +30,7 @@ from mozbuild.frontend.data import ( UnifiedSources, ) + class TestGnMozbuildWriter(BackendTester): def setUp(self): diff --git a/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py b/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py index 26d5c474f7a1..daf2634f2efb 100644 --- a/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py +++ b/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py @@ -82,7 +82,8 @@ class TestPartial(unittest.TestCase): self.assertTrue(os.path.exists(path)) def _assert_deps(self, env, deps): - deps = sorted(['$(wildcard %s)' % (mozpath.join(env.topobjdir, 'config.statusd', d)) for d in deps]) + deps = sorted(['$(wildcard %s)' % + (mozpath.join(env.topobjdir, 'config.statusd', d)) for d in deps]) self.assertEqual(sorted(env.get_dependencies()), deps) def test_dependencies(self): @@ -107,7 +108,8 @@ class TestPartial(unittest.TestCase): with self.assertRaises(KeyError): x = env.substs['NON_EXISTENT'] - self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', 'substs/MOZ_SUBST_1', 'substs/NON_EXISTENT']) + self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', + 'substs/MOZ_SUBST_1', 'substs/NON_EXISTENT']) self.assertEqual(env.substs.get('NON_EXISTENT'), None) def test_set_subst(self): @@ -159,5 +161,6 @@ class TestPartial(unittest.TestCase): self.assertEqual(mydefines['DEBUG'], '1') self.assertEqual(mydefines['MOZ_FOO'], '1') + if __name__ == "__main__": main() diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 0d6fd5214fef..64b5222d2112 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -59,9 +59,10 @@ class TestRecursiveMakeTraversal(unittest.TestCase): traversal.add('X') parallels = set(('G', 'H', 'I', 'J', 'O', 'P', 'Q', 'R', 'U')) + def filter(current, subdirs): return (current, [d for d in subdirs.dirs if d in parallels], - [d for d in subdirs.dirs if d not in parallels]) + [d for d in subdirs.dirs if d not in parallels]) start, deps = traversal.compute_dependencies(filter) self.assertEqual(start, ('X',)) @@ -95,8 +96,8 @@ class TestRecursiveMakeTraversal(unittest.TestCase): self.assertEqual(list(traversal.traverse('', filter)), ['', 'A', 'B', 'E', 'F', 'C', 'G', 'H', 'D', 'I', - 'M', 'N', 'T', 'J', 'O', 'P', 'U', 'K', 'Q', 'R', - 'V', 'L', 'S', 'W', 'X']) + 'M', 'N', 'T', 'J', 'O', 'P', 'U', 'K', 'Q', 'R', + 'V', 'L', 'S', 'W', 'X']) self.assertEqual(list(traversal.traverse('C', filter)), ['C', 'G', 'H']) @@ -191,14 +192,15 @@ class TestRecursiveMakeTraversal(unittest.TestCase): 'J': ('',), }) + class TestRecursiveMakeBackend(BackendTester): def test_basic(self): """Ensure the RecursiveMakeBackend works without error.""" env = self._consume('stub0', RecursiveMakeBackend) self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, - 'backend.RecursiveMakeBackend'))) + 'backend.RecursiveMakeBackend'))) self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, - 'backend.RecursiveMakeBackend.in'))) + 'backend.RecursiveMakeBackend.in'))) def test_output_files(self): """Ensure proper files are generated.""" @@ -385,7 +387,7 @@ class TestRecursiveMakeBackend(BackendTester): # EXPORTS files should appear in the dist_include install manifest. m = InstallManifest(path=mozpath.join(env.topobjdir, - '_build_manifests', 'install', 'dist_include')) + '_build_manifests', 'install', 'dist_include')) self.assertEqual(len(m), 7) self.assertIn('foo.h', m) self.assertIn('mozilla/mozilla1.h', m) @@ -567,7 +569,7 @@ class TestRecursiveMakeBackend(BackendTester): # EXPORTS files should appear in the dist_include install manifest. m = InstallManifest(path=mozpath.join(env.topobjdir, - '_build_manifests', 'install', 'dist_include')) + '_build_manifests', 'install', 'dist_include')) self.assertEqual(len(m), 8) self.assertIn('foo.h', m) self.assertIn('mozilla/mozilla1.h', m) @@ -605,7 +607,7 @@ class TestRecursiveMakeBackend(BackendTester): # RESOURCE_FILES should appear in the dist_bin install manifest. m = InstallManifest(path=os.path.join(env.topobjdir, - '_build_manifests', 'install', 'dist_bin')) + '_build_manifests', 'install', 'dist_bin')) self.assertEqual(len(m), 10) self.assertIn('res/foo.res', m) self.assertIn('res/fonts/font1.ttf', m) @@ -637,7 +639,7 @@ class TestRecursiveMakeBackend(BackendTester): """Pattern matches in test manifests' support-files should be recorded.""" env = self._consume('test-manifests-written', RecursiveMakeBackend) m = InstallManifest(path=mozpath.join(env.topobjdir, - '_build_manifests', 'install', '_test_files')) + '_build_manifests', 'install', '_test_files')) # This is not the most robust test in the world, but it gets the job # done. @@ -690,7 +692,7 @@ class TestRecursiveMakeBackend(BackendTester): # Install manifests should contain entries. install_dir = mozpath.join(env.topobjdir, '_build_manifests', - 'install') + 'install') self.assertTrue(os.path.isfile(mozpath.join(install_dir, 'xpidl'))) m = InstallManifest(path=mozpath.join(install_dir, 'xpidl')) @@ -710,7 +712,7 @@ class TestRecursiveMakeBackend(BackendTester): def test_test_support_files_tracked(self): env = self._consume('test-support-binaries-tracked', RecursiveMakeBackend) m = InstallManifest(path=mozpath.join(env.topobjdir, - '_build_manifests', 'install', '_tests')) + '_build_manifests', 'install', '_tests')) self.assertEqual(len(m), 4) self.assertIn('xpcshell/tests/mozbuildtest/test-library.dll', m) self.assertIn('xpcshell/tests/mozbuildtest/test-one.exe', m) @@ -772,7 +774,8 @@ class TestRecursiveMakeBackend(BackendTester): topsrcdir = env.topsrcdir.replace(os.sep, '/') expected = [ - "ALL_IPDLSRCS := bar1.ipdl foo1.ipdl %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" % tuple([topsrcdir] * 4), + "ALL_IPDLSRCS := bar1.ipdl foo1.ipdl %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" % tuple([ + topsrcdir] * 4), "CPPSRCS := UnifiedProtocols0.cpp", "IPDLDIRS := %s %s/bar %s/foo" % (env.topobjdir, topsrcdir, topsrcdir), ] @@ -949,8 +952,8 @@ class TestRecursiveMakeBackend(BackendTester): backend_path = mozpath.join(key, 'backend.mk') lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] found = [str for str in lines if - str.startswith('FINAL_TARGET') or str.startswith('XPI_NAME') or - str.startswith('DIST_SUBDIR')] + str.startswith('FINAL_TARGET') or str.startswith('XPI_NAME') or + str.startswith('DIST_SUBDIR')] self.assertEqual(found, expected_rules) def test_final_target_pp_files(self): @@ -1149,7 +1152,7 @@ class TestRecursiveMakeBackend(BackendTester): def test_test_manifests_duplicate_support_files(self): """Ensure duplicate support-files in test manifests work.""" env = self._consume('test-manifests-duplicate-support-files', - RecursiveMakeBackend) + RecursiveMakeBackend) p = os.path.join(env.topobjdir, '_build_manifests', 'install', '_test_files') m = InstallManifest(p) diff --git a/python/mozbuild/mozbuild/test/backend/test_visualstudio.py b/python/mozbuild/mozbuild/test/backend/test_visualstudio.py index bfc95e55271a..d93249b8b7d8 100644 --- a/python/mozbuild/mozbuild/test/backend/test_visualstudio.py +++ b/python/mozbuild/mozbuild/test/backend/test_visualstudio.py @@ -41,7 +41,7 @@ class TestVisualStudioBackend(BackendTester): els = d.getElementsByTagName('NMakeForcedIncludes') self.assertEqual(len(els), 1) self.assertEqual(els[0].firstChild.nodeValue, - '$(TopObjDir)\\dist\\include\\mozilla-config.h') + '$(TopObjDir)\\dist\\include\\mozilla-config.h') # LOCAL_INCLUDES get added to the include search path. els = d.getElementsByTagName('NMakeIncludeSearchPath') diff --git a/python/mozbuild/mozbuild/test/codecoverage/test_lcov_rewrite.py b/python/mozbuild/mozbuild/test/codecoverage/test_lcov_rewrite.py index b1b72438a6d4..91e21d32fe4d 100644 --- a/python/mozbuild/mozbuild/test/codecoverage/test_lcov_rewrite.py +++ b/python/mozbuild/mozbuild/test/codecoverage/test_lcov_rewrite.py @@ -108,6 +108,7 @@ LH:2 end_of_record """ + class TempFile(): def __init__(self, content): self.file = NamedTemporaryFile(delete=False) @@ -142,6 +143,7 @@ class TestLcovParser(unittest.TestCase): output = self.parser_roundtrip(fn_with_multiple_commas) self.assertEqual(fn_with_multiple_commas, output) + multiple_included_files = """//@line 1 "/src/dir/foo.js" bazfoobar //@line 2 "/src/dir/path/bar.js" @@ -158,6 +160,7 @@ baz fin """ + class TestLineRemapping(unittest.TestCase): def setUp(self): chrome_map_file = os.path.join(buildconfig.topobjdir, 'chrome-map.json') @@ -237,6 +240,7 @@ class TestLineRemapping(unittest.TestCase): # Rewrite preprocessed entries. lcov_file = lcov_rewriter.LcovFile([fpath]) r_num = [] + def rewrite_source(s): r_num.append(1) return s, pp_remap @@ -262,6 +266,7 @@ class TestLineRemapping(unittest.TestCase): self.assertEqual(original_covered_function_count, sum(r.covered_function_count for r in records)) + class TestUrlFinder(unittest.TestCase): def setUp(self): chrome_map_file = os.path.join(buildconfig.topobjdir, 'chrome-map.json') @@ -329,8 +334,10 @@ class TestUrlFinder(unittest.TestCase): omnijar_name = buildconfig.substs.get('OMNIJAR_NAME') paths = [ - ('jar:file:///home/worker/workspace/build/application/' + app_name + '/' + omnijar_name + '!/components/MainProcessSingleton.js', 'path1'), - ('jar:file:///home/worker/workspace/build/application/' + app_name + '/browser/features/firefox@getpocket.com.xpi!/bootstrap.js', 'path4'), + ('jar:file:///home/worker/workspace/build/application/' + app_name + + '/' + omnijar_name + '!/components/MainProcessSingleton.js', 'path1'), + ('jar:file:///home/worker/workspace/build/application/' + app_name + + '/browser/features/firefox@getpocket.com.xpi!/bootstrap.js', 'path4'), ] url_finder = lcov_rewriter.UrlFinder(self._chrome_map_file, '', '', []) @@ -356,31 +363,37 @@ class TestUrlFinder(unittest.TestCase): def test_chrome_resource_paths(self): paths = [ # Path with default url prefix - ('resource://gre/modules/osfile/osfile_async_worker.js', ('toolkit/components/osfile/modules/osfile_async_worker.js', None)), + ('resource://gre/modules/osfile/osfile_async_worker.js', + ('toolkit/components/osfile/modules/osfile_async_worker.js', None)), # Path with url prefix that is in chrome map - ('resource://activity-stream/lib/PrefsFeed.jsm', ('browser/components/newtab/lib/PrefsFeed.jsm', None)), + ('resource://activity-stream/lib/PrefsFeed.jsm', + ('browser/components/newtab/lib/PrefsFeed.jsm', None)), # Path which is in url overrides - ('chrome://global/content/netError.xhtml', ('browser/base/content/aboutNetError.xhtml', None)), + ('chrome://global/content/netError.xhtml', + ('browser/base/content/aboutNetError.xhtml', None)), # Path which ends with > eval ('resource://gre/modules/osfile/osfile_async_worker.js line 3 > eval', None), # Path which ends with > Function ('resource://gre/modules/osfile/osfile_async_worker.js line 3 > Function', None), # Path which contains "->" - ('resource://gre/modules/addons/XPIProvider.jsm -> resource://gre/modules/osfile/osfile_async_worker.js', ('toolkit/components/osfile/modules/osfile_async_worker.js', None)), + ('resource://gre/modules/addons/XPIProvider.jsm -> resource://gre/modules/osfile/osfile_async_worker.js', + ('toolkit/components/osfile/modules/osfile_async_worker.js', None)), # Path with pp_info ('resource://gre/modules/AppConstants.jsm', ('toolkit/modules/AppConstants.jsm', { '101,102': [ - 'toolkit/modules/AppConstants.jsm', + 'toolkit/modules/AppConstants.jsm', 135 ], })), # Path with query - ('resource://activity-stream/lib/PrefsFeed.jsm?q=0.9098419174803978', ('browser/components/newtab/lib/PrefsFeed.jsm', None)), + ('resource://activity-stream/lib/PrefsFeed.jsm?q=0.9098419174803978', + ('browser/components/newtab/lib/PrefsFeed.jsm', None)), ] url_finder = lcov_rewriter.UrlFinder(self._chrome_map_file, '', 'dist/bin/', []) for path, expected in paths: self.assertEqual(url_finder.rewrite_url(path), expected) + if __name__ == '__main__': mozunit.main() diff --git a/python/mozbuild/mozbuild/test/common.py b/python/mozbuild/mozbuild/test/common.py index 3b258d2af1f5..4afce0e41a6a 100644 --- a/python/mozbuild/mozbuild/test/common.py +++ b/python/mozbuild/mozbuild/test/common.py @@ -46,7 +46,7 @@ class MockConfig(object): topsrcdir='/path/to/topsrcdir', extra_substs={}, error_is_fatal=True, - ): + ): self.topsrcdir = mozpath.abspath(topsrcdir) self.topobjdir = mozpath.abspath('/path/to/topobjdir') diff --git a/python/mozbuild/mozbuild/test/compilation/test_warnings.py b/python/mozbuild/mozbuild/test/compilation/test_warnings.py index 1aa1fd1d6f9d..65e457da2385 100644 --- a/python/mozbuild/mozbuild/test/compilation/test_warnings.py +++ b/python/mozbuild/mozbuild/test/compilation/test_warnings.py @@ -27,11 +27,12 @@ MSVC_TESTS = [ "conversion from 'double' to 'uint32_t', possible loss of data", 'C:/mozilla-central/test/foo.cpp', 793, 'C4244', "'return' : conversion from 'double' to 'uint32_t', possible loss of " - 'data') + 'data') ] CURRENT_LINE = 1 + def get_warning(): global CURRENT_LINE @@ -45,6 +46,7 @@ def get_warning(): return w + class TestCompilerWarning(unittest.TestCase): def test_equivalence(self): w1 = CompilerWarning() @@ -122,6 +124,7 @@ class TestCompilerWarning(unittest.TestCase): self.assertGreaterEqual(w2, w1) self.assertGreaterEqual(w1, w2) + class TestWarningsParsing(unittest.TestCase): def test_clang_parsing(self): for source, filename, line, column, message, flag in CLANG_TESTS: @@ -148,6 +151,7 @@ class TestWarningsParsing(unittest.TestCase): self.assertEqual(warning['flag'], flag) self.assertEqual(warning['message'], message) + class TestWarningsDatabase(unittest.TestCase): def test_basic(self): db = WarningsDatabase() diff --git a/python/mozbuild/mozbuild/test/configure/common.py b/python/mozbuild/mozbuild/test/configure/common.py index 91682fbd48aa..97760b0b958a 100644 --- a/python/mozbuild/mozbuild/test/configure/common.py +++ b/python/mozbuild/mozbuild/test/configure/common.py @@ -31,6 +31,7 @@ def fake_short_path(path): for p in mozpath.split(path)) return path + def ensure_exe_extension(path): if sys.platform.startswith('win'): return path + '.exe' @@ -73,6 +74,7 @@ class ConfigureTestSandbox(ConfigureSandbox): This class is only meant to implement the minimal things to make moz.configure testing possible. As such, it takes shortcuts. ''' + def __init__(self, paths, config, environ, *args, **kwargs): self._search_path = environ.get('PATH', '').split(os.pathsep) @@ -153,7 +155,6 @@ class ConfigureTestSandbox(ConfigureSandbox): def __call__(self, *args, **kwargs): return self._func(*args, **kwargs) - return ReadOnlyNamespace( create_unicode_buffer=self.create_unicode_buffer, windll=ReadOnlyNamespace( diff --git a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py index 671c9582d496..34a3f3044d9c 100644 --- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py @@ -233,7 +233,6 @@ class TestChecksConfigure(unittest.TestCase): self.assertEqual(config, {'FOO': self.KNOWN_A}) self.assertEqual(out, 'checking for foo... %s\n' % self.KNOWN_A) - def test_check_prog_with_args(self): config, out, status = self.get_result( 'check_prog("FOO", ("unknown", "known-b", "known c"))', @@ -431,7 +430,8 @@ class TestChecksConfigure(unittest.TestCase): 'single element, or a string') def test_check_prog_with_path(self): - config, out, status = self.get_result('check_prog("A", ("known-a",), paths=["/some/path"])') + config, out, status = self.get_result( + 'check_prog("A", ("known-a",), paths=["/some/path"])') self.assertEqual(status, 1) self.assertEqual(config, {}) self.assertEqual(out, textwrap.dedent('''\ @@ -675,7 +675,6 @@ class TestChecksConfigure(unittest.TestCase): *** to the full path to pkg-config. ''')) - config, output, status = get_result("pkg_check_modules('MOZ_VALID', 'valid')", extra_paths=extra_paths) self.assertEqual(status, 0) diff --git a/python/mozbuild/mozbuild/test/configure/test_compile_checks.py b/python/mozbuild/mozbuild/test/configure/test_compile_checks.py index 618a30bdd868..ce2c7938d26a 100644 --- a/python/mozbuild/mozbuild/test/configure/test_compile_checks.py +++ b/python/mozbuild/mozbuild/test/configure/test_compile_checks.py @@ -21,6 +21,7 @@ from test_toolchain_helpers import FakeCompiler class BaseCompileChecks(unittest.TestCase): def get_mock_compiler(self, expected_test_content=None, expected_flags=None): expected_flags = expected_flags or [] + def mock_compiler(stdin, args): args, test_file = args[:-1], args[-1] self.assertIn('-c', args) @@ -220,7 +221,7 @@ class TestHeaderChecks(BaseCompileChecks): config, out, status = self.do_compile_test(cmd) self.assertEqual(status, 0) self.assertEqual(out, '') - self.assertEqual(config, {'DEFINES':{}}) + self.assertEqual(config, {'DEFINES': {}}) def test_check_header_include(self): expected_test_content = textwrap.dedent('''\ diff --git a/python/mozbuild/mozbuild/test/configure/test_configure.py b/python/mozbuild/mozbuild/test/configure/test_configure.py index 3ae4bbdd8245..b7da1d32ca96 100644 --- a/python/mozbuild/mozbuild/test/configure/test_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_configure.py @@ -247,16 +247,16 @@ class TestConfigure(unittest.TestCase): def foo(): import sys foo()'''), - sandbox - ) + sandbox + ) exec_(textwrap.dedent(''' @template @imports('sys') def foo(): return sys'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), sys) @@ -265,8 +265,8 @@ class TestConfigure(unittest.TestCase): @imports(_from='os', _import='path') def foo(): return path'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), os.path) @@ -275,8 +275,8 @@ class TestConfigure(unittest.TestCase): @imports(_from='os', _import='path', _as='os_path') def foo(): return os_path'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), os.path) @@ -285,8 +285,8 @@ class TestConfigure(unittest.TestCase): @imports('__builtin__') def foo(): return __builtin__'''), - sandbox - ) + sandbox + ) import __builtin__ self.assertIs(sandbox['foo'](), __builtin__) @@ -296,8 +296,8 @@ class TestConfigure(unittest.TestCase): @imports(_from='__builtin__', _import='open') def foo(): return open('%s')''' % os.devnull), - sandbox - ) + sandbox + ) f = sandbox['foo']() self.assertEquals(f.name, os.devnull) @@ -310,8 +310,8 @@ class TestConfigure(unittest.TestCase): def foo(): import sys return sys'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), sys) @@ -320,8 +320,8 @@ class TestConfigure(unittest.TestCase): @imports('__sandbox__') def foo(): return __sandbox__'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), sandbox) @@ -330,8 +330,8 @@ class TestConfigure(unittest.TestCase): @imports(_import='__sandbox__', _as='s') def foo(): return s'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), sandbox) @@ -348,8 +348,8 @@ class TestConfigure(unittest.TestCase): return sys return bar bar = foo()'''), - sandbox - ) + sandbox + ) with self.assertRaises(NameError) as e: sandbox._depends[sandbox['bar']].result() @@ -377,8 +377,8 @@ class TestConfigure(unittest.TestCase): return sys foo() foo()'''), - sandbox - ) + sandbox + ) self.assertEquals(len(imports), 1) @@ -587,7 +587,7 @@ class TestConfigure(unittest.TestCase): config = get_config(['--enable-foo=a,b']) self.assertIn('BAR', config) - self.assertEquals(config['BAR'], PositiveOptionValue(('a','b'))) + self.assertEquals(config['BAR'], PositiveOptionValue(('a', 'b'))) with self.assertRaises(InvalidOptionError) as e: get_config(['--enable-foo=a,b', '--disable-bar']) @@ -639,18 +639,18 @@ class TestConfigure(unittest.TestCase): mozpath.join(test_data_path, 'imply_option', 'imm.configure')) with self.assertRaisesRegexp(InvalidOptionError, - "--enable-foo' implied by 'imply_option at %s:7' conflicts with " - "'--disable-foo' from the command-line" % config_path): + "--enable-foo' implied by 'imply_option at %s:7' conflicts with " + "'--disable-foo' from the command-line" % config_path): get_config(['--disable-foo']) with self.assertRaisesRegexp(InvalidOptionError, - "--enable-bar=foo,bar' implied by 'imply_option at %s:16' conflicts" - " with '--enable-bar=a,b,c' from the command-line" % config_path): + "--enable-bar=foo,bar' implied by 'imply_option at %s:16' conflicts" + " with '--enable-bar=a,b,c' from the command-line" % config_path): get_config(['--enable-bar=a,b,c']) with self.assertRaisesRegexp(InvalidOptionError, - "--enable-baz=BAZ' implied by 'imply_option at %s:25' conflicts" - " with '--enable-baz=QUUX' from the command-line" % config_path): + "--enable-baz=BAZ' implied by 'imply_option at %s:25' conflicts" + " with '--enable-baz=QUUX' from the command-line" % config_path): get_config(['--enable-baz=QUUX']) def test_imply_option_failures(self): diff --git a/python/mozbuild/mozbuild/test/configure/test_options.py b/python/mozbuild/mozbuild/test/configure/test_options.py index 9defccb2c5de..5ddffbe748b9 100644 --- a/python/mozbuild/mozbuild/test/configure/test_options.py +++ b/python/mozbuild/mozbuild/test/configure/test_options.py @@ -250,7 +250,7 @@ class TestOption(unittest.TestCase): self.assertEquals(PositiveOptionValue(('c',)), value) value = option.get_value('--with-option=-b,+d') - self.assertEquals(PositiveOptionValue(('c','d')), value) + self.assertEquals(PositiveOptionValue(('c', 'd')), value) # Adding something that is in the default is fine value = option.get_value('--with-option=+b') @@ -876,13 +876,13 @@ class TestCommandLineHelper(unittest.TestCase): bar = Option('--bar', possible_origins=('mozconfig',)) with self.assertRaisesRegexp(InvalidOptionError, - "--bar can not be set by command-line. Values are accepted from: mozconfig"): + "--bar can not be set by command-line. Values are accepted from: mozconfig"): helper.handle(bar) baz = Option(env='BAZ', possible_origins=('implied',)) with self.assertRaisesRegexp(InvalidOptionError, - "BAZ=1 can not be set by environment. Values are accepted from: implied"): + "BAZ=1 can not be set by environment. Values are accepted from: implied"): helper.handle(baz) diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py index 5bdbdcfd2e0a..edfcd6cbc60b 100644 --- a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py +++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py @@ -42,6 +42,7 @@ class CompilerPreprocessor(Preprocessor): # different handling than what our Preprocessor does out of the box. # Hack around it enough that the configure tests work properly. context = self.context + def normalize_numbers(value): if isinstance(value, types.StringTypes): if value[-1:] == 'L' and value[:-1].isdigit(): @@ -49,6 +50,7 @@ class CompilerPreprocessor(Preprocessor): return value # Our Preprocessor doesn't handle macros with parameters, so we hack # around that for __has_feature()-like things. + def normalize_has_feature_or_builtin(expr): return self.HAS_FEATURE_OR_BUILTIN.sub(r'\1\2', expr) self.context = self.Context( @@ -162,6 +164,7 @@ class FakeCompiler(dict): For convenience, FakeCompiler instances can be added (+) to one another. ''' + def __init__(self, *definitions): for definition in definitions: if all(not isinstance(d, dict) for d in definition.itervalues()): diff --git a/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py index 2233cff559aa..2ebf5186b795 100644 --- a/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py @@ -82,7 +82,7 @@ class TestToolkitMozConfigure(BaseConfigureTest): self.assertEqual(get_value(environ={'MOZILLA_OFFICIAL': 1}), None) self.assertEqual(get_value(['--enable-release'], - environ={'MOZILLA_OFFICIAL': 1}), None) + environ={'MOZILLA_OFFICIAL': 1}), None) with self.assertRaises(InvalidOptionError): get_value(['--disable-release'], diff --git a/python/mozbuild/mozbuild/test/configure/test_util.py b/python/mozbuild/mozbuild/test/configure/test_util.py index 9f9575fd0606..122a405c9316 100644 --- a/python/mozbuild/mozbuild/test/configure/test_util.py +++ b/python/mozbuild/mozbuild/test/configure/test_util.py @@ -57,7 +57,7 @@ class TestConfigureOutputHandler(unittest.TestCase): name = '%s.test_format' % self.__class__.__name__ logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) - handler = ConfigureOutputHandler(out, err) + handler = ConfigureOutputHandler(out, err) handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) logger.addHandler(handler) @@ -79,7 +79,7 @@ class TestConfigureOutputHandler(unittest.TestCase): name = '%s.test_continuation' % self.__class__.__name__ logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) - handler = ConfigureOutputHandler(out, out) + handler = ConfigureOutputHandler(out, out) handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) logger.addHandler(handler) @@ -138,7 +138,7 @@ class TestConfigureOutputHandler(unittest.TestCase): err = StringIO() logger.removeHandler(handler) - handler = ConfigureOutputHandler(out, err) + handler = ConfigureOutputHandler(out, err) handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) logger.addHandler(handler) @@ -167,7 +167,7 @@ class TestConfigureOutputHandler(unittest.TestCase): name = '%s.test_queue_debug' % self.__class__.__name__ logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) - handler = ConfigureOutputHandler(out, out, maxlen=3) + handler = ConfigureOutputHandler(out, out, maxlen=3) handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) logger.addHandler(handler) @@ -268,7 +268,7 @@ class TestConfigureOutputHandler(unittest.TestCase): name = '%s.test_queue_debug_reentrant' % self.__class__.__name__ logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) - handler = ConfigureOutputHandler(out, out, maxlen=10) + handler = ConfigureOutputHandler(out, out, maxlen=10) handler.setFormatter(logging.Formatter('%(levelname)s| %(message)s')) logger.addHandler(handler) @@ -424,7 +424,7 @@ class TestLogSubprocessOutput(unittest.TestCase): sandbox = ConfigureSandbox({}, {}, ['configure'], out, out) sandbox.include_file(mozpath.join(topsrcdir, 'build', - 'moz.configure', 'util.configure')) + 'moz.configure', 'util.configure')) sandbox.include_file(mozpath.join(topsrcdir, 'python', 'mozbuild', 'mozbuild', 'test', 'configure', 'data', 'subprocess.configure')) @@ -474,6 +474,7 @@ class TestVersion(unittest.TestCase): self.assertEqual(v.minor, 0) self.assertEqual(v.patch, 0) + class TestCheckCmdOutput(unittest.TestCase): def get_result(self, command='', paths=None): @@ -483,7 +484,7 @@ class TestCheckCmdOutput(unittest.TestCase): sandbox = ConfigureTestSandbox(paths, config, {}, ['/bin/configure'], out, out) sandbox.include_file(mozpath.join(topsrcdir, 'build', - 'moz.configure', 'util.configure')) + 'moz.configure', 'util.configure')) status = 0 try: exec_(command, sandbox) diff --git a/python/mozbuild/mozbuild/test/controller/test_ccachestats.py b/python/mozbuild/mozbuild/test/controller/test_ccachestats.py index 60d99c82d5cd..75b0d2c5bd5c 100644 --- a/python/mozbuild/mozbuild/test/controller/test_ccachestats.py +++ b/python/mozbuild/mozbuild/test/controller/test_ccachestats.py @@ -291,5 +291,6 @@ class TestCcacheStats(unittest.TestCase): stat9 = CCacheStats(self.STAT9) self.assertTrue(stat9) + if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/controller/test_clobber.py b/python/mozbuild/mozbuild/test/controller/test_clobber.py index 5aa080337c57..44f2c48caab2 100644 --- a/python/mozbuild/mozbuild/test/controller/test_clobber.py +++ b/python/mozbuild/mozbuild/test/controller/test_clobber.py @@ -123,7 +123,7 @@ class TestClobberer(unittest.TestCase): self.assertFalse(os.path.exists(dummy_path)) self.assertTrue(os.path.exists(c.obj_clobber)) self.assertGreaterEqual(os.path.getmtime(c.obj_clobber), - os.path.getmtime(c.src_clobber)) + os.path.getmtime(c.src_clobber)) def test_objdir_is_srcdir(self): """If topobjdir is the topsrcdir, refuse to clobber.""" @@ -181,7 +181,6 @@ class TestClobberer(unittest.TestCase): self.assertFalse(performed) self.assertIn('Cannot clobber while the shell is inside', reason) - def test_mozconfig_opt_in(self): """Auto clobber iff AUTOCLOBBER is in the environment.""" diff --git a/python/mozbuild/mozbuild/test/frontend/test_context.py b/python/mozbuild/mozbuild/test/frontend/test_context.py index dcb1b65bdabb..e11bfb5ccbfd 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_context.py +++ b/python/mozbuild/mozbuild/test/frontend/test_context.py @@ -40,20 +40,20 @@ class TestContext(unittest.TestCase): self.assertEqual(test['foo'], 0) - self.assertEqual(set(test.keys()), { 'foo' }) + self.assertEqual(set(test.keys()), {'foo'}) self.assertEqual(test['bar'], False) - self.assertEqual(set(test.keys()), { 'foo', 'bar' }) + self.assertEqual(set(test.keys()), {'foo', 'bar'}) self.assertEqual(test['baz'], {}) - self.assertEqual(set(test.keys()), { 'foo', 'bar', 'baz' }) + self.assertEqual(set(test.keys()), {'foo', 'bar', 'baz'}) with self.assertRaises(KeyError): test['qux'] - self.assertEqual(set(test.keys()), { 'foo', 'bar', 'baz' }) + self.assertEqual(set(test.keys()), {'foo', 'bar', 'baz'}) def test_type_check(self): test = Context({ @@ -75,7 +75,7 @@ class TestContext(unittest.TestCase): test['baz'] = [('a', 1), ('b', 2)] - self.assertEqual(test['baz'], { 'a': 1, 'b': 2 }) + self.assertEqual(test['baz'], {'a': 1, 'b': 2}) def test_update(self): test = Context({ @@ -93,7 +93,7 @@ class TestContext(unittest.TestCase): test.update(bar=True, foo=1) - self.assertEqual(set(test.keys()), { 'foo', 'bar' }) + self.assertEqual(set(test.keys()), {'foo', 'bar'}) self.assertEqual(test['foo'], 1) self.assertEqual(test['bar'], True) @@ -101,13 +101,13 @@ class TestContext(unittest.TestCase): self.assertEqual(test['foo'], 2) self.assertEqual(test['bar'], False) - test.update([('foo', 0), ('baz', { 'a': 1, 'b': 2 })]) + test.update([('foo', 0), ('baz', {'a': 1, 'b': 2})]) self.assertEqual(test['foo'], 0) - self.assertEqual(test['baz'], { 'a': 1, 'b': 2 }) + self.assertEqual(test['baz'], {'a': 1, 'b': 2}) test.update([('foo', 42), ('baz', [('c', 3), ('d', 4)])]) self.assertEqual(test['foo'], 42) - self.assertEqual(test['baz'], { 'c': 3, 'd': 4 }) + self.assertEqual(test['baz'], {'c': 3, 'd': 4}) def test_context_paths(self): test = Context() @@ -214,7 +214,8 @@ class TestContext(unittest.TestCase): self.assertEqual(test.source_stack, [foo, bar, bar, foo]) def test_context_dirs(self): - class Config(object): pass + class Config(object): + pass config = Config() config.topsrcdir = mozpath.abspath(os.curdir) config.topobjdir = mozpath.abspath('obj') @@ -277,7 +278,8 @@ class TestSymbols(unittest.TestCase): class TestPaths(unittest.TestCase): @classmethod def setUpClass(cls): - class Config(object): pass + class Config(object): + pass cls.config = config = Config() config.topsrcdir = mozpath.abspath(os.curdir) config.topobjdir = mozpath.abspath('obj') @@ -723,7 +725,7 @@ class TestFiles(unittest.TestCase): f2 = Files(c, 'b/**', 'a/bar') f2['BUG_COMPONENT'] = (u'Product2', u'Component2') - files = {'a/foo': f1, 'a/bar' : f2, 'b/foo' : f2 } + files = {'a/foo': f1, 'a/bar': f2, 'b/foo': f2} self.assertEqual(Files.aggregate(files), { 'bug_component_counts': [ ((u'Product2', u'Component2'), 2), diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py index 5e5ab7ff28b7..99c745c913ba 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -174,9 +174,9 @@ class TestEmitterBasic(unittest.TestCase): topobjdir = mozpath.abspath(reader.config.topobjdir) self.assertEqual(objs[0].relpath, 'foo') self.assertEqual(mozpath.normpath(objs[0].output_path), - mozpath.normpath(mozpath.join(topobjdir, 'foo'))) + mozpath.normpath(mozpath.join(topobjdir, 'foo'))) self.assertEqual(mozpath.normpath(objs[1].output_path), - mozpath.normpath(mozpath.join(topobjdir, 'bar'))) + mozpath.normpath(mozpath.join(topobjdir, 'bar'))) def test_variable_passthru(self): reader = self.reader('variable-passthru') @@ -403,7 +403,7 @@ class TestEmitterBasic(unittest.TestCase): def test_resolved_flags_error(self): reader = self.reader('resolved-flags-error') with self.assertRaisesRegexp(BuildReaderError, - "`DEFINES` may not be set in COMPILE_FLAGS from moz.build"): + "`DEFINES` may not be set in COMPILE_FLAGS from moz.build"): self.read_topsrcdir(reader) def test_includes_in_flags(self): @@ -436,7 +436,7 @@ class TestEmitterBasic(unittest.TestCase): # When yasm is not available, this should raise. reader = self.reader('use-yasm') with self.assertRaisesRegexp(SandboxValidationError, - 'yasm is not available'): + 'yasm is not available'): self.read_topsrcdir(reader) # When yasm is available, this should work. @@ -534,17 +534,16 @@ class TestEmitterBasic(unittest.TestCase): """ reader = self.reader('localized-files-not-localized-generated') with self.assertRaisesRegexp(SandboxValidationError, - 'Objdir file listed in LOCALIZED_FILES not in LOCALIZED_GENERATED_FILES:'): + 'Objdir file listed in LOCALIZED_FILES not in LOCALIZED_GENERATED_FILES:'): objs = self.read_topsrcdir(reader) - def test_localized_generated_files_final_target_files(self): """Test that using LOCALIZED_GENERATED_FILES and then putting the output in FINAL_TARGET_FILES as an objdir path produces an error. """ reader = self.reader('localized-generated-files-final-target-files') with self.assertRaisesRegexp(SandboxValidationError, - 'Outputs of LOCALIZED_GENERATED_FILES cannot be used in FINAL_TARGET_FILES:'): + 'Outputs of LOCALIZED_GENERATED_FILES cannot be used in FINAL_TARGET_FILES:'): objs = self.read_topsrcdir(reader) def test_generated_files_method_names(self): @@ -578,19 +577,19 @@ class TestEmitterBasic(unittest.TestCase): def test_generated_files_no_script(self): reader = self.reader('generated-files-no-script') with self.assertRaisesRegexp(SandboxValidationError, - 'Script for generating bar.c does not exist'): + 'Script for generating bar.c does not exist'): self.read_topsrcdir(reader) def test_generated_files_no_inputs(self): reader = self.reader('generated-files-no-inputs') with self.assertRaisesRegexp(SandboxValidationError, - 'Input for generating foo.c does not exist'): + 'Input for generating foo.c does not exist'): self.read_topsrcdir(reader) def test_generated_files_no_python_script(self): reader = self.reader('generated-files-no-python-script') with self.assertRaisesRegexp(SandboxValidationError, - 'Script for generating bar.c does not end in .py'): + 'Script for generating bar.c does not end in .py'): self.read_topsrcdir(reader) def test_exports(self): @@ -619,7 +618,7 @@ class TestEmitterBasic(unittest.TestCase): ''' reader = self.reader('exports-missing') with self.assertRaisesRegexp(SandboxValidationError, - 'File listed in EXPORTS does not exist:'): + 'File listed in EXPORTS does not exist:'): self.read_topsrcdir(reader) def test_exports_missing_generated(self): @@ -628,7 +627,7 @@ class TestEmitterBasic(unittest.TestCase): ''' reader = self.reader('exports-missing-generated') with self.assertRaisesRegexp(SandboxValidationError, - 'Objdir file listed in EXPORTS not in GENERATED_FILES:'): + 'Objdir file listed in EXPORTS not in GENERATED_FILES:'): self.read_topsrcdir(reader) def test_exports_generated(self): @@ -665,7 +664,7 @@ class TestEmitterBasic(unittest.TestCase): def test_test_harness_files_root(self): reader = self.reader('test-harness-files-root') with self.assertRaisesRegexp(SandboxValidationError, - 'Cannot install files to the root of TEST_HARNESS_FILES'): + 'Cannot install files to the root of TEST_HARNESS_FILES'): self.read_topsrcdir(reader) def test_program(self): @@ -736,7 +735,6 @@ class TestEmitterBasic(unittest.TestCase): with self.assertRaisesRegexp(SandboxValidationError, 'Empty test manifest'): self.read_topsrcdir(reader) - def test_test_manifest_just_support_files(self): """A test manifest with no tests but support-files is not supported.""" reader = self.reader('test-manifest-just-support') @@ -751,7 +749,7 @@ class TestEmitterBasic(unittest.TestCase): reader = self.reader('test-manifest-dupes') with self.assertRaisesRegexp(SandboxValidationError, 'bar.js appears multiple times ' - 'in a test manifest under a support-files field, please omit the duplicate entry.'): + 'in a test manifest under a support-files field, please omit the duplicate entry.'): self.read_topsrcdir(reader) def test_test_manifest_absolute_support_files(self): @@ -930,7 +928,7 @@ class TestEmitterBasic(unittest.TestCase): self.assertEqual(o.dupe_manifest, m.get('dupe', False)) external_normalized = set(mozpath.basename(p) for p in - o.external_installs) + o.external_installs) self.assertEqual(external_normalized, m.get('external', set())) self.assertEqual(len(o.installs), len(m['installs'])) @@ -948,7 +946,7 @@ class TestEmitterBasic(unittest.TestCase): reader = self.reader('test-manifest-unmatched-generated') with self.assertRaisesRegexp(SandboxValidationError, - 'entry in generated-files not present elsewhere'): + 'entry in generated-files not present elsewhere'): self.read_topsrcdir(reader), def test_test_manifest_parent_support_files_dir(self): @@ -965,14 +963,14 @@ class TestEmitterBasic(unittest.TestCase): expected = mozpath.join(o.srcdir, 'support-file.txt') self.assertIn(expected, o.installs) self.assertEqual(o.installs[expected], - ('testing/mochitest/tests/child/support-file.txt', False)) + ('testing/mochitest/tests/child/support-file.txt', False)) def test_test_manifest_missing_test_error(self): """Missing test files should result in error.""" reader = self.reader('test-manifest-missing-test-file') with self.assertRaisesRegexp(SandboxValidationError, - 'lists test that does not exist: test_missing.html'): + 'lists test that does not exist: test_missing.html'): self.read_topsrcdir(reader) def test_test_manifest_missing_test_error_unfiltered(self): @@ -980,7 +978,7 @@ class TestEmitterBasic(unittest.TestCase): reader = self.reader('test-manifest-missing-test-file-unfiltered') with self.assertRaisesRegexp(SandboxValidationError, - 'lists test that does not exist: missing.js'): + 'lists test that does not exist: missing.js'): self.read_topsrcdir(reader) def test_ipdl_sources(self): @@ -1028,7 +1026,6 @@ class TestEmitterBasic(unittest.TestCase): ]) self.assertEqual(generated_sources, expected) - def test_local_includes(self): """Test that LOCAL_INCLUDES is emitted correctly.""" reader = self.reader('local_includes') @@ -1136,21 +1133,21 @@ class TestEmitterBasic(unittest.TestCase): def test_xpidl_module_no_sources(self): """XPIDL_MODULE without XPIDL_SOURCES should be rejected.""" with self.assertRaisesRegexp(SandboxValidationError, 'XPIDL_MODULE ' - 'cannot be defined'): + 'cannot be defined'): reader = self.reader('xpidl-module-no-sources') self.read_topsrcdir(reader) def test_xpidl_module_no_sources(self): """Missing XPIDL_SOURCES should be rejected.""" with self.assertRaisesRegexp(SandboxValidationError, 'File .* ' - 'from XPIDL_SOURCES does not exist'): + 'from XPIDL_SOURCES does not exist'): reader = self.reader('missing-xpidl') self.read_topsrcdir(reader) def test_missing_local_includes(self): """LOCAL_INCLUDES containing non-existent directories should be rejected.""" with self.assertRaisesRegexp(SandboxValidationError, 'Path specified in ' - 'LOCAL_INCLUDES does not exist'): + 'LOCAL_INCLUDES does not exist'): reader = self.reader('missing-local-includes') self.read_topsrcdir(reader) @@ -1159,7 +1156,7 @@ class TestEmitterBasic(unittest.TestCase): reader = self.reader('library-defines') objs = self.read_topsrcdir(reader) - libraries = [o for o in objs if isinstance(o,StaticLibrary)] + libraries = [o for o in objs if isinstance(o, StaticLibrary)] library_flags = [o for o in objs if isinstance(o, ComputedFlags) and 'LIBRARY_DEFINES' in o.flags] expected = { @@ -1334,7 +1331,6 @@ class TestEmitterBasic(unittest.TestCase): reader.config.substs['OBJ_SUFFIX'])), linkable.objs) - def test_unified_sources(self): """Test that UNIFIED_SOURCES works properly.""" reader = self.reader('unified-sources') @@ -1369,7 +1365,6 @@ class TestEmitterBasic(unittest.TestCase): reader.config.substs['OBJ_SUFFIX'])), linkable.objs) - def test_unified_sources_non_unified(self): """Test that UNIFIED_SOURCES with FILES_PER_UNIFIED_FILE=1 works properly.""" reader = self.reader('unified-sources-non-unified') @@ -1418,7 +1413,7 @@ class TestEmitterBasic(unittest.TestCase): def test_missing_final_target_pp_files(self): """Test that FINAL_TARGET_PP_FILES with missing files throws errors.""" with self.assertRaisesRegexp(SandboxValidationError, 'File listed in ' - 'FINAL_TARGET_PP_FILES does not exist'): + 'FINAL_TARGET_PP_FILES does not exist'): reader = self.reader('dist-files-missing') self.read_topsrcdir(reader) @@ -1426,7 +1421,7 @@ class TestEmitterBasic(unittest.TestCase): '''Test that non-srcdir paths in FINAL_TARGET_PP_FILES throws errors.''' reader = self.reader('final-target-pp-files-non-srcdir') with self.assertRaisesRegexp(SandboxValidationError, - 'Only source directory paths allowed in FINAL_TARGET_PP_FILES:'): + 'Only source directory paths allowed in FINAL_TARGET_PP_FILES:'): self.read_topsrcdir(reader) def test_localized_files(self): @@ -1450,7 +1445,7 @@ class TestEmitterBasic(unittest.TestCase): `en-US/` or contain `locales/en-US/`.""" reader = self.reader('localized-files-no-en-us') with self.assertRaisesRegexp(SandboxValidationError, - 'LOCALIZED_FILES paths must start with `en-US/` or contain `locales/en-US/`: foo.js'): + 'LOCALIZED_FILES paths must start with `en-US/` or contain `locales/en-US/`: foo.js'): objs = self.read_topsrcdir(reader) def test_localized_pp_files(self): @@ -1473,28 +1468,28 @@ class TestEmitterBasic(unittest.TestCase): '''Test that defining a RustLibrary without a Cargo.toml fails.''' reader = self.reader('rust-library-no-cargo-toml') with self.assertRaisesRegexp(SandboxValidationError, - 'No Cargo.toml file found'): + 'No Cargo.toml file found'): self.read_topsrcdir(reader) def test_rust_library_name_mismatch(self): '''Test that defining a RustLibrary that doesn't match Cargo.toml fails.''' reader = self.reader('rust-library-name-mismatch') with self.assertRaisesRegexp(SandboxValidationError, - 'library.*does not match Cargo.toml-defined package'): + 'library.*does not match Cargo.toml-defined package'): self.read_topsrcdir(reader) def test_rust_library_no_lib_section(self): '''Test that a RustLibrary Cargo.toml with no [lib] section fails.''' reader = self.reader('rust-library-no-lib-section') with self.assertRaisesRegexp(SandboxValidationError, - 'Cargo.toml for.* has no \\[lib\\] section'): + 'Cargo.toml for.* has no \\[lib\\] section'): self.read_topsrcdir(reader) def test_rust_library_invalid_crate_type(self): '''Test that a RustLibrary Cargo.toml has a permitted crate-type.''' reader = self.reader('rust-library-invalid-crate-type') with self.assertRaisesRegexp(SandboxValidationError, - 'crate-type.* is not permitted'): + 'crate-type.* is not permitted'): self.read_topsrcdir(reader) def test_rust_library_dash_folding(self): @@ -1517,7 +1512,7 @@ class TestEmitterBasic(unittest.TestCase): reader = self.reader('multiple-rust-libraries', extra_substs=dict(RUST_TARGET='i686-pc-windows-msvc')) with self.assertRaisesRegexp(LinkageMultipleRustLibrariesError, - 'Cannot link multiple Rust libraries'): + 'Cannot link multiple Rust libraries'): self.read_topsrcdir(reader) def test_rust_library_features(self): @@ -1537,21 +1532,21 @@ class TestEmitterBasic(unittest.TestCase): '''Test that duplicate RustLibrary features are rejected.''' reader = self.reader('rust-library-duplicate-features') with self.assertRaisesRegexp(SandboxValidationError, - 'features for .* should not contain duplicates'): + 'features for .* should not contain duplicates'): self.read_topsrcdir(reader) def test_rust_program_no_cargo_toml(self): '''Test that specifying RUST_PROGRAMS without a Cargo.toml fails.''' reader = self.reader('rust-program-no-cargo-toml') with self.assertRaisesRegexp(SandboxValidationError, - 'No Cargo.toml file found'): + 'No Cargo.toml file found'): self.read_topsrcdir(reader) def test_host_rust_program_no_cargo_toml(self): '''Test that specifying HOST_RUST_PROGRAMS without a Cargo.toml fails.''' reader = self.reader('host-rust-program-no-cargo-toml') with self.assertRaisesRegexp(SandboxValidationError, - 'No Cargo.toml file found'): + 'No Cargo.toml file found'): self.read_topsrcdir(reader) def test_rust_program_nonexistent_name(self): @@ -1559,7 +1554,7 @@ class TestEmitterBasic(unittest.TestCase): correctly throws an error.''' reader = self.reader('rust-program-nonexistent-name') with self.assertRaisesRegexp(SandboxValidationError, - 'Cannot find Cargo.toml definition for'): + 'Cannot find Cargo.toml definition for'): self.read_topsrcdir(reader) def test_host_rust_program_nonexistent_name(self): @@ -1567,7 +1562,7 @@ class TestEmitterBasic(unittest.TestCase): Cargo.toml correctly throws an error.''' reader = self.reader('host-rust-program-nonexistent-name') with self.assertRaisesRegexp(SandboxValidationError, - 'Cannot find Cargo.toml definition for'): + 'Cannot find Cargo.toml definition for'): self.read_topsrcdir(reader) def test_rust_programs(self): @@ -1671,7 +1666,7 @@ class TestEmitterBasic(unittest.TestCase): """ reader = self.reader('test-symbols-file-objdir-missing-generated') with self.assertRaisesRegexp(SandboxValidationError, - 'Objdir file specified in SYMBOLS_FILE not in GENERATED_FILES:'): + 'Objdir file specified in SYMBOLS_FILE not in GENERATED_FILES:'): self.read_topsrcdir(reader) diff --git a/python/mozbuild/mozbuild/test/frontend/test_namespaces.py b/python/mozbuild/mozbuild/test/frontend/test_namespaces.py index 71cc634e1bdb..f6e707c186c4 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_namespaces.py +++ b/python/mozbuild/mozbuild/test/frontend/test_namespaces.py @@ -52,7 +52,7 @@ VARIABLES = { 'FUGA': (Fuga, unicode, None), 'PIYO': (Piyo, unicode, None), 'HOGERA': (ContextDerivedTypedList(Piyo, StrictOrderingOnAppendList), - list, None), + list, None), 'HOGEHOGE': (ContextDerivedTypedListWithItems( Piyo, StrictOrderingOnAppendListWithFlagsFactory({ @@ -60,6 +60,7 @@ VARIABLES = { })), list, None), } + class TestContext(unittest.TestCase): def test_key_rejection(self): # Lowercase keys should be rejected during normal operation. @@ -203,5 +204,6 @@ class TestContext(unittest.TestCase): with self.assertRaises(UnsortedError): ns['HOGEHOGE'] += ['f', 'e', 'd'] + if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_reader.py b/python/mozbuild/mozbuild/test/frontend/test_reader.py index dc4edb69d63c..6337f7be4e25 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_reader.py +++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py @@ -72,7 +72,7 @@ class TestBuildReader(unittest.TestCase): self.assertTrue(os.path.exists(path)) contexts = list(reader.read_mozbuild(path, reader.config, - descend=False)) + descend=False)) self.assertEqual(len(contexts), 1) @@ -116,7 +116,7 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertEqual(e.actual_file, self.file_path('reader-error-basic', - 'moz.build')) + 'moz.build')) self.assertIn('The error occurred while processing the', str(e)) @@ -128,9 +128,9 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertEqual(e.actual_file, - self.file_path('reader-error-included-from', 'child.build')) + self.file_path('reader-error-included-from', 'child.build')) self.assertEqual(e.main_file, - self.file_path('reader-error-included-from', 'moz.build')) + self.file_path('reader-error-included-from', 'moz.build')) self.assertIn('This file was included as part of processing', str(e)) @@ -176,13 +176,13 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertIn('The error was triggered on line 5', str(e)) self.assertIn('is an attempt to write an illegal value to a special', - str(e)) + str(e)) self.assertIn('variable whose value was rejected is:\n\n DIRS', - str(e)) + str(e)) self.assertIn('written to it was of the following type:\n\n %s' % text_type, - str(e)) + str(e)) self.assertIn('expects the following type(s):\n\n list', str(e)) @@ -194,7 +194,7 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertIn('The underlying problem is an illegal file access', - str(e)) + str(e)) def test_error_missing_include_path(self): reader = self.reader('reader-error-missing-include') @@ -213,7 +213,7 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertIn('The error appears to be the fault of the script', - str(e)) + str(e)) self.assertIn(' ["TypeError: unsupported operand', str(e)) def test_error_bad_dir(self): @@ -265,9 +265,9 @@ class TestBuildReader(unittest.TestCase): self.assertEqual(len(contexts), 4) self.assertEqual([context.relsrcdir for context in contexts], - ['', 'foo', 'foo/baz', 'bar']) + ['', 'foo', 'foo/baz', 'bar']) self.assertEqual([context['XPIDL_MODULE'] for context in contexts], - ['foobar', 'foobar', 'baz', 'foobar']) + ['foobar', 'foobar', 'baz', 'foobar']) def test_find_relevant_mozbuilds(self): reader = self.reader('reader-relevant-mozbuild') @@ -334,16 +334,16 @@ class TestBuildReader(unittest.TestCase): reader = self.reader('reader-relevant-mozbuild') paths, contexts = reader.read_relevant_mozbuilds(['d1/every-level/a/file', - 'd1/every-level/b/file', 'd2/file']) + 'd1/every-level/b/file', 'd2/file']) self.assertEqual(len(paths), 3) self.assertEqual(len(contexts), 6) self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/a/file']], - ['', 'd1', 'd1/every-level', 'd1/every-level/a']) + ['', 'd1', 'd1/every-level', 'd1/every-level/a']) self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/b/file']], - ['', 'd1', 'd1/every-level', 'd1/every-level/b']) + ['', 'd1', 'd1/every-level', 'd1/every-level/b']) self.assertEqual([ctx.relsrcdir for ctx in paths['d2/file']], - ['', 'd2']) + ['', 'd2']) def test_files_bad_bug_component(self): reader = self.reader('files-info') @@ -389,7 +389,8 @@ class TestBuildReader(unittest.TestCase): self.assertEqual(js_flags['BUG_COMPONENT'], BugzillaComponent('Firefox', 'JS')) self.assertEqual(cpp_flags['BUG_COMPONENT'], BugzillaComponent('Firefox', 'C++')) - self.assertEqual(misc_flags['BUG_COMPONENT'], BugzillaComponent('default_product', 'default_component')) + self.assertEqual(misc_flags['BUG_COMPONENT'], BugzillaComponent( + 'default_product', 'default_component')) def test_files_bug_component_final(self): reader = self.reader('files-info') @@ -401,13 +402,13 @@ class TestBuildReader(unittest.TestCase): 'bug_component/final/subcomponent/bar']) self.assertEqual(v['bug_component/final/foo']['BUG_COMPONENT'], - BugzillaComponent('default_product', 'default_component')) + BugzillaComponent('default_product', 'default_component')) self.assertEqual(v['bug_component/final/Makefile.in']['BUG_COMPONENT'], - BugzillaComponent('Firefox Build System', 'General')) + BugzillaComponent('Firefox Build System', 'General')) self.assertEqual(v['bug_component/final/subcomponent/Makefile.in']['BUG_COMPONENT'], - BugzillaComponent('Firefox Build System', 'General')) + BugzillaComponent('Firefox Build System', 'General')) self.assertEqual(v['bug_component/final/subcomponent/bar']['BUG_COMPONENT'], - BugzillaComponent('Another', 'Component')) + BugzillaComponent('Another', 'Component')) def test_file_test_deps(self): reader = self.reader('files-test-metadata') @@ -506,7 +507,8 @@ class TestBuildReader(unittest.TestCase): self.assertEqual(info['subd/aa.py']['SCHEDULES'].exclusive, schedules.EXCLUSIVE_COMPONENTS) # Files('yaml.py') in subd/moz.build combines with Files('subdir/**.py') self.assertEqual(info['subd/yaml.py']['SCHEDULES'].inclusive, ['py-lint', 'yaml-lint']) - self.assertEqual(info['subd/yaml.py']['SCHEDULES'].exclusive, schedules.EXCLUSIVE_COMPONENTS) + self.assertEqual(info['subd/yaml.py']['SCHEDULES'].exclusive, + schedules.EXCLUSIVE_COMPONENTS) # .. but exlusive does not override inclusive self.assertEqual(info['subd/win.js']['SCHEDULES'].inclusive, ['js-lint']) self.assertEqual(info['subd/win.js']['SCHEDULES'].exclusive, ['windows']) @@ -519,5 +521,6 @@ class TestBuildReader(unittest.TestCase): self.assertEqual(set(info['win.and.osx']['SCHEDULES'].exclusive), set(['macosx', 'windows'])) + if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py index d24c5d9ea624..954776c09ce4 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py +++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py @@ -99,7 +99,7 @@ class TestSandbox(unittest.TestCase): sandbox.exec_source('DIRS = ["foo"]') with self.assertRaises(SandboxExecutionError) as se: - sandbox.exec_source('DIRS = ["bar"]') + sandbox.exec_source('DIRS = ["bar"]') self.assertEqual(sandbox['DIRS'], ['foo']) e = se.exception @@ -129,6 +129,7 @@ class TestedSandbox(MozbuildSandbox): It automatically normalizes paths given to exec_file and exec_source. This helps simplify the test code. ''' + def normalize_path(self, path): return mozpath.normpath( mozpath.join(self._context.config.topsrcdir, path)) @@ -141,7 +142,7 @@ class TestedSandbox(MozbuildSandbox): def exec_source(self, source, path=''): super(TestedSandbox, self).exec_source(source, - self.normalize_path(path) if path else '') + self.normalize_path(path) if path else '') class TestMozbuildSandbox(unittest.TestCase): @@ -188,9 +189,9 @@ class TestMozbuildSandbox(unittest.TestCase): self.assertEqual(sandbox['TOPOBJDIR'], config.topobjdir) self.assertEqual(sandbox['RELATIVEDIR'], 'foo/bar') self.assertEqual(sandbox['SRCDIR'], - mozpath.join(config.topsrcdir, 'foo/bar')) + mozpath.join(config.topsrcdir, 'foo/bar')) self.assertEqual(sandbox['OBJDIR'], - mozpath.join(config.topobjdir, 'foo/bar')) + mozpath.join(config.topobjdir, 'foo/bar')) def test_config_access(self): sandbox = self.sandbox() @@ -238,7 +239,7 @@ class TestMozbuildSandbox(unittest.TestCase): sandbox.exec_source('DIST_SUBDIR = "foo"') with self.assertRaises(SandboxExecutionError) as se: - sandbox.exec_source('DIST_SUBDIR = "bar"') + sandbox.exec_source('DIST_SUBDIR = "bar"') self.assertEqual(sandbox['DIST_SUBDIR'], 'foo') e = se.exception @@ -259,7 +260,7 @@ class TestMozbuildSandbox(unittest.TestCase): sandbox.source_path('bar'), ]) self.assertEqual(sandbox._context.main_path, - sandbox.normalize_path('moz.build')) + sandbox.normalize_path('moz.build')) self.assertEqual(len(sandbox._context.all_paths), 2) def test_include_outside_topsrcdir(self): @@ -269,7 +270,7 @@ class TestMozbuildSandbox(unittest.TestCase): sandbox.exec_file('relative.build') self.assertEqual(se.exception.illegal_path, - sandbox.normalize_path('../moz.build')) + sandbox.normalize_path('../moz.build')) def test_include_error_stack(self): # Ensure the path stack is reported properly in exceptions. @@ -345,7 +346,7 @@ class TestMozbuildSandbox(unittest.TestCase): sandbox = MozbuildSandbox(Context(VARIABLES, config)) self.assertEqual(sandbox['CONFIG']['BAD_UTF8'], - u'\ufffd\ufffd\ufffd\ufffd:') + u'\ufffd\ufffd\ufffd\ufffd:') def test_invalid_exports_set_base(self): sandbox = self.sandbox() @@ -472,7 +473,7 @@ def foo(): e = se.exception.exc_value self.assertEqual(e.message, - 'Template function names must be CamelCase.') + 'Template function names must be CamelCase.') # Template names must not already be registered. sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) @@ -489,11 +490,12 @@ def Template(): e = se.exception.exc_value self.assertEqual(e.message, - 'A template named "Template" was already declared in %s.' % - sandbox.normalize_path('templates.mozbuild')) + 'A template named "Template" was already declared in %s.' % + sandbox.normalize_path('templates.mozbuild')) def test_function_args(self): - class Foo(int): pass + class Foo(int): + pass def foo(a, b): return type(a), type(b) diff --git a/python/mozbuild/mozbuild/test/test_android_version_code.py b/python/mozbuild/mozbuild/test/test_android_version_code.py index 80fa91ad69a3..c0798172014c 100644 --- a/python/mozbuild/mozbuild/test/test_android_version_code.py +++ b/python/mozbuild/mozbuild/test/test_android_version_code.py @@ -10,33 +10,41 @@ from mozbuild.android_version_code import ( android_version_code_v1, ) + class TestAndroidVersionCode(unittest.TestCase): def test_android_version_code_v0(self): # From https://treeherder.mozilla.org/#/jobs?repo=mozilla-central&revision=e25de9972a77. buildid = '20150708104620' - arm_api9 = 2015070819 + arm_api9 = 2015070819 arm_api11 = 2015070821 - x86_api9 = 2015070822 - self.assertEqual(android_version_code_v0(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) - self.assertEqual(android_version_code_v0(buildid, cpu_arch='armeabi-v7a', min_sdk=11, max_sdk=None), arm_api11) - self.assertEqual(android_version_code_v0(buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) + x86_api9 = 2015070822 + self.assertEqual(android_version_code_v0( + buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) + self.assertEqual(android_version_code_v0( + buildid, cpu_arch='armeabi-v7a', min_sdk=11, max_sdk=None), arm_api11) + self.assertEqual(android_version_code_v0( + buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) def test_android_version_code_v1(self): buildid = '20150825141628' - arm_api16 = 0b01111000001000000001001001110001 + arm_api16 = 0b01111000001000000001001001110001 arm64_api21 = 0b01111000001000000001001001110100 - x86_api9 = 0b01111000001000000001001001110100 - self.assertEqual(android_version_code_v1(buildid, cpu_arch='armeabi-v7a', min_sdk=16, max_sdk=None), arm_api16) - self.assertEqual(android_version_code_v1(buildid, cpu_arch='arm64-v8a', min_sdk=21, max_sdk=None), arm64_api21) - self.assertEqual(android_version_code_v1(buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) + x86_api9 = 0b01111000001000000001001001110100 + self.assertEqual(android_version_code_v1( + buildid, cpu_arch='armeabi-v7a', min_sdk=16, max_sdk=None), arm_api16) + self.assertEqual(android_version_code_v1( + buildid, cpu_arch='arm64-v8a', min_sdk=21, max_sdk=None), arm64_api21) + self.assertEqual(android_version_code_v1( + buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) def test_android_version_code_v1_underflow(self): '''Verify that it is an error to ask for v1 codes predating the cutoff.''' - buildid = '201508010000' # Earliest possible. + buildid = '201508010000' # Earliest possible. arm_api9 = 0b01111000001000000000000000000000 - self.assertEqual(android_version_code_v1(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) + self.assertEqual(android_version_code_v1( + buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) with self.assertRaises(ValueError) as cm: - underflow = '201507310000' # Latest possible (valid) underflowing date. + underflow = '201507310000' # Latest possible (valid) underflowing date. android_version_code_v1(underflow, cpu_arch='armeabi', min_sdk=9, max_sdk=None) self.assertTrue('underflow' in cm.exception.message) diff --git a/python/mozbuild/mozbuild/test/test_artifact_cache.py b/python/mozbuild/mozbuild/test/test_artifact_cache.py index 780473a0462e..54886bb95c48 100644 --- a/python/mozbuild/mozbuild/test/test_artifact_cache.py +++ b/python/mozbuild/mozbuild/test/test_artifact_cache.py @@ -24,6 +24,7 @@ CONTENTS = { 'http://server/larger': b'larger' * 3000, } + class FakeResponse(object): def __init__(self, content): self._content = content diff --git a/python/mozbuild/mozbuild/test/test_base.py b/python/mozbuild/mozbuild/test/test_base.py index e728bfa395ec..ec8a0f11d934 100644 --- a/python/mozbuild/mozbuild/test/test_base.py +++ b/python/mozbuild/mozbuild/test/test_base.py @@ -76,7 +76,7 @@ class TestMozbuildObject(unittest.TestCase): os.environ[b'MOZCONFIG'] = mozconfig.name self.assertEqual(base.topobjdir, mozpath.join(base.topsrcdir, - 'foo')) + 'foo')) self.assertTrue(base.topobjdir.endswith('foo')) def test_objdir_config_status(self): @@ -214,7 +214,7 @@ class TestMozbuildObject(unittest.TestCase): context.topdir = topsrcdir context.settings = None context.log_manager = None - context.detect_virtualenv_mozinfo=False + context.detect_virtualenv_mozinfo = False o = MachCommandBase(context) @@ -280,7 +280,7 @@ class TestMozbuildObject(unittest.TestCase): context.topdir = topsrcdir context.settings = None context.log_manager = None - context.detect_virtualenv_mozinfo=False + context.detect_virtualenv_mozinfo = False stdout = sys.stdout sys.stdout = StringIO() @@ -332,7 +332,7 @@ class TestMozbuildObject(unittest.TestCase): substs.append(('BIN_SUFFIX', '')) base._config_environment = ConfigEnvironment(base.topsrcdir, - base.topobjdir, substs=substs) + base.topobjdir, substs=substs) p = base.get_binary_path('xpcshell', False) if platform.startswith('darwin'): @@ -366,6 +366,7 @@ class TestMozbuildObject(unittest.TestCase): else: self.assertTrue(p.endswith('foobar')) + class TestPathArgument(unittest.TestCase): def test_path_argument(self): # Absolute path @@ -404,5 +405,6 @@ class TestPathArgument(unittest.TestCase): self.assertEqual(p.srcdir_path(), "/src/foo/bar") self.assertEqual(p.objdir_path(), "/src/obj/foo/bar") + if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/test_containers.py b/python/mozbuild/mozbuild/test/test_containers.py index 3d46f86a90fc..98db6329d4c9 100644 --- a/python/mozbuild/mozbuild/test/test_containers.py +++ b/python/mozbuild/mozbuild/test/test_containers.py @@ -142,6 +142,7 @@ class TestList(unittest.TestCase): with self.assertRaises(ValueError): test = test + False + class TestOrderedDefaultDict(unittest.TestCase): def test_simple(self): original = OrderedDict(foo=1, bar=2) @@ -152,21 +153,21 @@ class TestOrderedDefaultDict(unittest.TestCase): self.assertEqual(test['foo'], 1) - self.assertEqual(test.keys(), ['foo', 'bar' ]) + self.assertEqual(test.keys(), ['foo', 'bar']) def test_defaults(self): - test = OrderedDefaultDict(bool, {'foo': 1 }) + test = OrderedDefaultDict(bool, {'foo': 1}) self.assertEqual(test['foo'], 1) self.assertEqual(test['qux'], False) - self.assertEqual(test.keys(), ['foo', 'qux' ]) + self.assertEqual(test.keys(), ['foo', 'qux']) class TestKeyedDefaultDict(unittest.TestCase): def test_simple(self): - original = {'foo': 1, 'bar': 2 } + original = {'foo': 1, 'bar': 2} test = KeyedDefaultDict(lambda x: x, original) @@ -175,7 +176,7 @@ class TestKeyedDefaultDict(unittest.TestCase): self.assertEqual(test['foo'], 1) def test_defaults(self): - test = KeyedDefaultDict(lambda x: x, {'foo': 1 }) + test = KeyedDefaultDict(lambda x: x, {'foo': 1}) self.assertEqual(test['foo'], 1) @@ -196,7 +197,7 @@ class TestKeyedDefaultDict(unittest.TestCase): class TestReadOnlyKeyedDefaultDict(unittest.TestCase): def test_defaults(self): - test = ReadOnlyKeyedDefaultDict(lambda x: x, {'foo': 1 }) + test = ReadOnlyKeyedDefaultDict(lambda x: x, {'foo': 1}) self.assertEqual(test['foo'], 1) diff --git a/python/mozbuild/mozbuild/test/test_dotproperties.py b/python/mozbuild/mozbuild/test/test_dotproperties.py index a03f85b0d179..772c2febc902 100644 --- a/python/mozbuild/mozbuild/test/test_dotproperties.py +++ b/python/mozbuild/mozbuild/test/test_dotproperties.py @@ -31,7 +31,6 @@ key=value self.assertEqual(p.get('missing', 'default'), 'default') self.assertEqual(p.get('key'), 'value') - def test_update(self): contents = StringIO(''' old=old value @@ -48,7 +47,6 @@ key=new value self.assertEqual(p.get('old'), 'old value') self.assertEqual(p.get('key'), 'new value') - def test_get_list(self): contents = StringIO(''' list.0=A @@ -64,7 +62,6 @@ order.2=C self.assertEqual(p.get_list('list'), ['A', 'B', 'C']) self.assertEqual(p.get_list('order'), ['A', 'B', 'C']) - def test_get_list_with_shared_prefix(self): contents = StringIO(''' list.0=A @@ -85,7 +82,6 @@ list.other.0=H self.assertEqual(p.get_list('list.sublist.second'), ['G']) self.assertEqual(p.get_list('list.other'), ['H']) - def test_get_dict(self): contents = StringIO(''' A.title=title A @@ -108,7 +104,6 @@ C=value with self.assertRaises(ValueError): p.get_dict('C', required_keys=['missing_key']) - def test_get_dict_with_shared_prefix(self): contents = StringIO(''' A.title=title A @@ -123,9 +118,9 @@ B.subdict.url=url B subdict self.assertEqual(p.get_dict('A'), {'title': 'title A'}) self.assertEqual(p.get_dict('B'), {'title': 'title B', 'url': 'url B'}) self.assertEqual(p.get_dict('A.subdict'), - {'title': 'title A subdict'}) + {'title': 'title A subdict'}) self.assertEqual(p.get_dict('B.subdict'), - {'title': 'title B subdict', 'url': 'url B subdict'}) + {'title': 'title B subdict', 'url': 'url B subdict'}) def test_get_dict_with_value_prefix(self): contents = StringIO(''' @@ -143,7 +138,6 @@ A.default.C.ignored=C ignored self.assertEqual(p.get('A.default.B'), 'B') self.assertEqual(p.get('A.default.C'), 'C') - def test_unicode(self): contents = StringIO(''' # Danish. diff --git a/python/mozbuild/mozbuild/test/test_expression.py b/python/mozbuild/mozbuild/test/test_expression.py index fb3c45894c81..e042d903885a 100644 --- a/python/mozbuild/mozbuild/test/test_expression.py +++ b/python/mozbuild/mozbuild/test/test_expression.py @@ -6,77 +6,80 @@ import mozunit from mozbuild.preprocessor import Expression, Context + class TestContext(unittest.TestCase): - """ - Unit tests for the Context class - """ + """ + Unit tests for the Context class + """ - def setUp(self): - self.c = Context() - self.c['FAIL'] = 'PASS' + def setUp(self): + self.c = Context() + self.c['FAIL'] = 'PASS' - def test_string_literal(self): - """test string literal, fall-through for undefined var in a Context""" - self.assertEqual(self.c['PASS'], 'PASS') + def test_string_literal(self): + """test string literal, fall-through for undefined var in a Context""" + self.assertEqual(self.c['PASS'], 'PASS') - def test_variable(self): - """test value for defined var in the Context class""" - self.assertEqual(self.c['FAIL'], 'PASS') + def test_variable(self): + """test value for defined var in the Context class""" + self.assertEqual(self.c['FAIL'], 'PASS') + + def test_in(self): + """test 'var in context' to not fall for fallback""" + self.assert_('FAIL' in self.c) + self.assert_('PASS' not in self.c) - def test_in(self): - """test 'var in context' to not fall for fallback""" - self.assert_('FAIL' in self.c) - self.assert_('PASS' not in self.c) class TestExpression(unittest.TestCase): - """ - Unit tests for the Expression class - evaluate() is called with a context {FAIL: 'PASS'} - """ + """ + Unit tests for the Expression class + evaluate() is called with a context {FAIL: 'PASS'} + """ - def setUp(self): - self.c = Context() - self.c['FAIL'] = 'PASS' + def setUp(self): + self.c = Context() + self.c['FAIL'] = 'PASS' - def test_string_literal(self): - """Test for a string literal in an Expression""" - self.assertEqual(Expression('PASS').evaluate(self.c), 'PASS') + def test_string_literal(self): + """Test for a string literal in an Expression""" + self.assertEqual(Expression('PASS').evaluate(self.c), 'PASS') - def test_variable(self): - """Test for variable value in an Expression""" - self.assertEqual(Expression('FAIL').evaluate(self.c), 'PASS') + def test_variable(self): + """Test for variable value in an Expression""" + self.assertEqual(Expression('FAIL').evaluate(self.c), 'PASS') - def test_not(self): - """Test for the ! operator""" - self.assert_(Expression('!0').evaluate(self.c)) - self.assert_(not Expression('!1').evaluate(self.c)) + def test_not(self): + """Test for the ! operator""" + self.assert_(Expression('!0').evaluate(self.c)) + self.assert_(not Expression('!1').evaluate(self.c)) - def test_equals(self): - """ Test for the == operator""" - self.assert_(Expression('FAIL == PASS').evaluate(self.c)) + def test_equals(self): + """ Test for the == operator""" + self.assert_(Expression('FAIL == PASS').evaluate(self.c)) - def test_notequals(self): - """ Test for the != operator""" - self.assert_(Expression('FAIL != 1').evaluate(self.c)) + def test_notequals(self): + """ Test for the != operator""" + self.assert_(Expression('FAIL != 1').evaluate(self.c)) - def test_logical_and(self): - """ Test for the && operator""" - self.assertTrue(Expression('PASS == PASS && PASS != NOTPASS').evaluate(self.c)) + def test_logical_and(self): + """ Test for the && operator""" + self.assertTrue(Expression('PASS == PASS && PASS != NOTPASS').evaluate(self.c)) - def test_logical_or(self): - """ Test for the || operator""" - self.assertTrue(Expression('PASS == NOTPASS || PASS != NOTPASS').evaluate(self.c)) + def test_logical_or(self): + """ Test for the || operator""" + self.assertTrue(Expression('PASS == NOTPASS || PASS != NOTPASS').evaluate(self.c)) - def test_logical_ops(self): - """ Test for the && and || operators precedence""" - # Would evaluate to false if precedence was wrong - self.assertTrue(Expression('PASS == PASS || PASS != NOTPASS && PASS == NOTPASS').evaluate(self.c)) + def test_logical_ops(self): + """ Test for the && and || operators precedence""" + # Would evaluate to false if precedence was wrong + self.assertTrue(Expression( + 'PASS == PASS || PASS != NOTPASS && PASS == NOTPASS').evaluate(self.c)) - def test_defined(self): - """ Test for the defined() value""" - self.assertTrue(Expression('defined(FAIL)').evaluate(self.c)) - self.assertTrue(Expression('!defined(PASS)').evaluate(self.c)) + def test_defined(self): + """ Test for the defined() value""" + self.assertTrue(Expression('defined(FAIL)').evaluate(self.c)) + self.assertTrue(Expression('!defined(PASS)').evaluate(self.c)) if __name__ == '__main__': - mozunit.main() + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_jarmaker.py b/python/mozbuild/mozbuild/test/test_jarmaker.py index 1a01294cfb52..2dc8544ed31b 100644 --- a/python/mozbuild/mozbuild/test/test_jarmaker.py +++ b/python/mozbuild/mozbuild/test/test_jarmaker.py @@ -5,7 +5,11 @@ from __future__ import print_function import unittest -import os, sys, os.path, time, inspect +import os +import sys +import os.path +import time +import inspect from filecmp import dircmp from tempfile import mkdtemp from shutil import rmtree, copy2 @@ -66,6 +70,7 @@ if sys.platform == "win32": DWORD] GetVolumeInformation.restype = ctypes.c_int + def symlinks_supported(path): if sys.platform == "win32": # Add 1 for a trailing backslash if necessary, and 1 for the terminating @@ -86,6 +91,7 @@ def symlinks_supported(path): else: return True + def _getfileinfo(path): """Return information for the given file. This only works on Windows.""" fh = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, None, OPEN_EXISTING, 0, None) @@ -97,6 +103,7 @@ def _getfileinfo(path): raise WinError() return info + def is_symlink_to(dest, src): if sys.platform == "win32": # Check if both are on the same volume and have the same file ID @@ -113,9 +120,11 @@ def is_symlink_to(dest, src): abssrc = os.path.abspath(src) return target == abssrc + class _TreeDiff(dircmp): """Helper to report rich results on difference between two directories. """ + def _fillDiff(self, dc, rv, basepath="{0}"): rv['right_only'] += map(lambda l: basepath.format(l), dc.right_only) rv['left_only'] += map(lambda l: basepath.format(l), dc.left_only) @@ -124,9 +133,10 @@ class _TreeDiff(dircmp): rv['funny'] += map(lambda l: basepath.format(l), dc.funny_files) for subdir, _dc in dc.subdirs.iteritems(): self._fillDiff(_dc, rv, basepath.format(subdir + "/{0}")) + def allResults(self, left, right): - rv = {'right_only':[], 'left_only':[], - 'diff_files':[], 'funny': []} + rv = {'right_only': [], 'left_only': [], + 'diff_files': [], 'funny': []} self._fillDiff(self, rv) chunks = [] if rv['right_only']: @@ -141,11 +151,13 @@ class _TreeDiff(dircmp): chunks.append("{0} don't compare".format(', '.join(rv['funny']))) return '; '.join(chunks) + class TestJarMaker(unittest.TestCase): """ Unit tests for JarMaker.py """ - debug = False # set to True to debug failing tests on disk + debug = False # set to True to debug failing tests on disk + def setUp(self): self.tmpdir = mkdtemp() self.srcdir = os.path.join(self.tmpdir, 'src') @@ -215,18 +227,18 @@ class TestJarMaker(unittest.TestCase): dir/foo (bar) ''') jarf.close() - open(os.path.join(self.srcdir,'bar'),'w').write('content\n') + open(os.path.join(self.srcdir, 'bar'), 'w').write('content\n') # create reference refpath = os.path.join(self.refdir, 'chrome', 'test.jar', 'dir') os.makedirs(refpath) - open(os.path.join(refpath, 'foo'), 'w').write('content\n') + open(os.path.join(refpath, 'foo'), 'w').write('content\n') def test_a_simple_jar(self): '''Test a simple jar.mn''' self._create_simple_setup() # call JarMaker - rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'), - sourcedirs = [self.srcdir]) + rv = self._jar_and_compare(os.path.join(self.srcdir, 'jar.mn'), + sourcedirs=[self.srcdir]) self.assertTrue(not rv, rv) def test_a_simple_symlink(self): @@ -238,7 +250,7 @@ class TestJarMaker(unittest.TestCase): jm = JarMaker(outputFormat='symlink') jm.sourcedirs = [self.srcdir] jm.topsourcedir = self.srcdir - jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir) + jm.makeJar(os.path.join(self.srcdir, 'jar.mn'), self.builddir) # All we do is check that srcdir/bar points to builddir/chrome/test/dir/foo srcbar = os.path.join(self.srcdir, 'bar') destfoo = os.path.join(self.builddir, 'chrome', 'test', 'dir', 'foo') @@ -253,12 +265,12 @@ class TestJarMaker(unittest.TestCase): dir/hoge (qux/*) ''') jarf.close() - open(os.path.join(self.srcdir,'foo.js'),'w').write('foo.js\n') - open(os.path.join(self.srcdir,'bar.js'),'w').write('bar.js\n') + open(os.path.join(self.srcdir, 'foo.js'), 'w').write('foo.js\n') + open(os.path.join(self.srcdir, 'bar.js'), 'w').write('bar.js\n') os.makedirs(os.path.join(self.srcdir, 'qux', 'foo')) - open(os.path.join(self.srcdir,'qux', 'foo', '1'),'w').write('1\n') - open(os.path.join(self.srcdir,'qux', 'foo', '2'),'w').write('2\n') - open(os.path.join(self.srcdir,'qux', 'baz'),'w').write('baz\n') + open(os.path.join(self.srcdir, 'qux', 'foo', '1'), 'w').write('1\n') + open(os.path.join(self.srcdir, 'qux', 'foo', '2'), 'w').write('2\n') + open(os.path.join(self.srcdir, 'qux', 'baz'), 'w').write('baz\n') # create reference refpath = os.path.join(self.refdir, 'chrome', 'test.jar', 'dir') os.makedirs(os.path.join(refpath, 'bar')) @@ -273,8 +285,8 @@ class TestJarMaker(unittest.TestCase): '''Test a wildcard in jar.mn''' self._create_wildcard_setup() # call JarMaker - rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'), - sourcedirs = [self.srcdir]) + rv = self._jar_and_compare(os.path.join(self.srcdir, 'jar.mn'), + sourcedirs=[self.srcdir]) self.assertTrue(not rv, rv) def test_a_wildcard_symlink(self): @@ -286,7 +298,7 @@ class TestJarMaker(unittest.TestCase): jm = JarMaker(outputFormat='symlink') jm.sourcedirs = [self.srcdir] jm.topsourcedir = self.srcdir - jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir) + jm.makeJar(os.path.join(self.srcdir, 'jar.mn'), self.builddir) expected_symlinks = { ('bar', 'foo.js'): ('foo.js',), @@ -311,9 +323,11 @@ class Test_relativesrcdir(unittest.TestCase): self.jm.relativesrcdir = 'browser/locales' self.fake_empty_file = StringIO() self.fake_empty_file.name = 'fake_empty_file' + def tearDown(self): del self.jm del self.fake_empty_file + def test_en_US(self): jm = self.jm jm.makeJar(self.fake_empty_file, '/NO_OUTPUT_REQUIRED') @@ -322,11 +336,13 @@ class Test_relativesrcdir(unittest.TestCase): os.path.join(os.path.abspath('/TOPSOURCEDIR'), 'browser/locales', 'en-US') ]) + def test_l10n_no_merge(self): jm = self.jm jm.l10nbase = '/L10N_BASE' jm.makeJar(self.fake_empty_file, '/NO_OUTPUT_REQUIRED') self.assertEquals(jm.localedirs, [os.path.join('/L10N_BASE', 'browser')]) + def test_l10n_merge(self): jm = self.jm jm.l10nbase = '/L10N_BASE' @@ -338,6 +354,7 @@ class Test_relativesrcdir(unittest.TestCase): os.path.join(os.path.abspath('/TOPSOURCEDIR'), 'browser/locales', 'en-US') ]) + def test_override(self): jm = self.jm jm.outputFormat = 'flat' # doesn't touch chrome dir without files @@ -351,6 +368,7 @@ relativesrcdir dom/locales: os.path.join(os.path.abspath('/TOPSOURCEDIR'), 'dom/locales', 'en-US') ]) + def test_override_l10n(self): jm = self.jm jm.l10nbase = '/L10N_BASE' diff --git a/python/mozbuild/mozbuild/test/test_licenses.py b/python/mozbuild/mozbuild/test/test_licenses.py index e3b2ae8abc9d..13ec0adb010d 100644 --- a/python/mozbuild/mozbuild/test/test_licenses.py +++ b/python/mozbuild/mozbuild/test/test_licenses.py @@ -10,28 +10,30 @@ from mozbuild.preprocessor import Preprocessor from mozbuild.vendor_rust import VendorRust + class TestLicenses(unittest.TestCase): - """ - Unit tests for the Rust Vendoring stuff - """ + """ + Unit tests for the Rust Vendoring stuff + """ - def setUp(self): - pass + def setUp(self): + pass - def tearDown(self): - pass + def tearDown(self): + pass + + def testLicense(self): + self.assertEquals(VendorRust.runtime_license('', 'Apache-2.0'), True) + self.assertEquals(VendorRust.runtime_license('', 'MIT'), True) + self.assertEquals(VendorRust.runtime_license('', 'GPL'), False) + self.assertEquals(VendorRust.runtime_license('', 'MIT /GPL'), True) + self.assertEquals(VendorRust.runtime_license('', 'GPL/ Proprietary'), False) + self.assertEquals(VendorRust.runtime_license('', 'GPL AND MIT'), False) + self.assertEquals(VendorRust.runtime_license('', 'ISC\tAND\tMIT'), False) + self.assertEquals(VendorRust.runtime_license('', 'GPL OR MIT'), True) + self.assertEquals(VendorRust.runtime_license('', 'ALLIGATOR MIT'), False) + pass - def testLicense(self): - self.assertEquals(VendorRust.runtime_license('', 'Apache-2.0'), True) - self.assertEquals(VendorRust.runtime_license('', 'MIT'), True) - self.assertEquals(VendorRust.runtime_license('', 'GPL'), False) - self.assertEquals(VendorRust.runtime_license('', 'MIT /GPL'), True) - self.assertEquals(VendorRust.runtime_license('', 'GPL/ Proprietary'), False) - self.assertEquals(VendorRust.runtime_license('', 'GPL AND MIT'), False) - self.assertEquals(VendorRust.runtime_license('', 'ISC\tAND\tMIT'), False) - self.assertEquals(VendorRust.runtime_license('', 'GPL OR MIT'), True) - self.assertEquals(VendorRust.runtime_license('', 'ALLIGATOR MIT'), False) - pass if __name__ == '__main__': - mozunit.main() + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_line_endings.py b/python/mozbuild/mozbuild/test/test_line_endings.py index 565abc8c93ff..5c0bac40c2d9 100644 --- a/python/mozbuild/mozbuild/test/test_line_endings.py +++ b/python/mozbuild/mozbuild/test/test_line_endings.py @@ -8,39 +8,41 @@ import mozunit from mozbuild.preprocessor import Preprocessor + class TestLineEndings(unittest.TestCase): - """ - Unit tests for the Context class - """ + """ + Unit tests for the Context class + """ - def setUp(self): - self.pp = Preprocessor() - self.pp.out = StringIO() - self.tempnam = os.tempnam('.') + def setUp(self): + self.pp = Preprocessor() + self.pp.out = StringIO() + self.tempnam = os.tempnam('.') - def tearDown(self): - os.remove(self.tempnam) + def tearDown(self): + os.remove(self.tempnam) - def createFile(self, lineendings): - f = open(self.tempnam, 'wb') - for line, ending in zip(['a', '#literal b', 'c'], lineendings): - f.write(line+ending) - f.close() + def createFile(self, lineendings): + f = open(self.tempnam, 'wb') + for line, ending in zip(['a', '#literal b', 'c'], lineendings): + f.write(line+ending) + f.close() - def testMac(self): - self.createFile(['\x0D']*3) - self.pp.do_include(self.tempnam) - self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') + def testMac(self): + self.createFile(['\x0D']*3) + self.pp.do_include(self.tempnam) + self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') - def testUnix(self): - self.createFile(['\x0A']*3) - self.pp.do_include(self.tempnam) - self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') + def testUnix(self): + self.createFile(['\x0A']*3) + self.pp.do_include(self.tempnam) + self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') + + def testWindows(self): + self.createFile(['\x0D\x0A']*3) + self.pp.do_include(self.tempnam) + self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') - def testWindows(self): - self.createFile(['\x0D\x0A']*3) - self.pp.do_include(self.tempnam) - self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') if __name__ == '__main__': - mozunit.main() + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_makeutil.py b/python/mozbuild/mozbuild/test/test_makeutil.py index 6fffa0e0e1fa..75232b32d438 100644 --- a/python/mozbuild/mozbuild/test/test_makeutil.py +++ b/python/mozbuild/mozbuild/test/test_makeutil.py @@ -39,10 +39,10 @@ class TestMakefile(unittest.TestCase): rule.add_commands(['$(BAZ) -o $@ $<', '$(TOUCH) $@']) rule.dump(out) self.assertEqual(out.getvalue(), - 'foo bar: baz\n' + - '\techo $@\n' + - '\t$(BAZ) -o $@ $<\n' + - '\t$(TOUCH) $@\n') + 'foo bar: baz\n' + + '\techo $@\n' + + '\t$(BAZ) -o $@ $<\n' + + '\t$(TOUCH) $@\n') out.truncate(0) rule = Rule(['foo']) @@ -96,19 +96,19 @@ class TestMakefile(unittest.TestCase): rule.add_commands(['echo $@']) mk.dump(out, removal_guard=False) self.assertEqual(out.getvalue(), - 'foo: bar baz qux\n' + - '\techo foo\n' + - 'bar baz: hoge\n' + - '\techo $@\n') + 'foo: bar baz qux\n' + + '\techo foo\n' + + 'bar baz: hoge\n' + + '\techo $@\n') out.truncate(0) mk.dump(out) self.assertEqual(out.getvalue(), - 'foo: bar baz qux\n' + - '\techo foo\n' + - 'bar baz: hoge\n' + - '\techo $@\n' + - 'hoge qux:\n') + 'foo: bar baz qux\n' + + '\techo foo\n' + + 'bar baz: hoge\n' + + '\techo $@\n' + + 'hoge qux:\n') def test_statement(self): out = StringIO() @@ -119,11 +119,11 @@ class TestMakefile(unittest.TestCase): mk.create_rule(['$(BAR)']).add_commands(['echo $@']) mk.dump(out, removal_guard=False) self.assertEqual(out.getvalue(), - 'foo: bar\n' + - '\techo foo\n' + - 'BAR = bar\n' + - '$(BAR):\n' + - '\techo $@\n') + 'foo: bar\n' + + '\techo foo\n' + + 'BAR = bar\n' + + '$(BAR):\n' + + '\techo $@\n') @unittest.skipIf(os.name != 'nt', 'Test only applicable on Windows.') def test_path_normalization(self): @@ -134,9 +134,9 @@ class TestMakefile(unittest.TestCase): rule.add_commands(['echo c:\\foo']) mk.dump(out) self.assertEqual(out.getvalue(), - 'c:/foo: c:/bar c:/baz/qux\n' + - '\techo c:\\foo\n' + - 'c:/bar c:/baz/qux:\n') + 'c:/foo: c:/bar c:/baz/qux\n' + + '\techo c:\\foo\n' + + 'c:/bar c:/baz/qux:\n') def test_read_dep_makefile(self): input = StringIO( @@ -161,5 +161,6 @@ class TestMakefile(unittest.TestCase): 'target: b c a\n' + 'a b c:\n') + if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/test_mozconfig.py b/python/mozbuild/mozbuild/test/test_mozconfig.py index 7178b8942796..5b4449ab002a 100644 --- a/python/mozbuild/mozbuild/test/test_mozconfig.py +++ b/python/mozbuild/mozbuild/test/test_mozconfig.py @@ -199,7 +199,7 @@ class TestMozconfigLoader(unittest.TestCase): MozconfigLoader(d).find_mozconfig() self.assertIn('Multiple default mozconfig files present', - e.exception.message) + e.exception.message) def test_find_deprecated_path_srcdir(self): """Ensure we error when deprecated path locations are present.""" @@ -212,7 +212,7 @@ class TestMozconfigLoader(unittest.TestCase): MozconfigLoader(d).find_mozconfig() self.assertIn('This implicit location is no longer', - e.exception.message) + e.exception.message) self.assertIn(d, e.exception.message) def test_find_deprecated_home_paths(self): @@ -230,7 +230,7 @@ class TestMozconfigLoader(unittest.TestCase): self.get_loader().find_mozconfig() self.assertIn('This implicit location is no longer', - e.exception.message) + e.exception.message) self.assertIn(path, e.exception.message) def test_read_no_mozconfig(self): @@ -327,7 +327,7 @@ class TestMozconfigLoader(unittest.TestCase): result = loader.read_mozconfig(mozconfig.name) self.assertEqual(result['topobjdir'], '%s/some-objdir' % - loader.topsrcdir) + loader.topsrcdir) def test_read_new_variables(self): """New variables declared in mozconfig file are detected.""" @@ -427,7 +427,7 @@ class TestMozconfigLoader(unittest.TestCase): result = loader.read_mozconfig(mozconfig.name) self.assertEqual(result['vars']['added']['TEST'], - loader.topsrcdir.replace(os.sep, '/')) + loader.topsrcdir.replace(os.sep, '/')) self.assertEqual(result['env']['added'], {}) def test_read_empty_variable_value(self): @@ -459,7 +459,7 @@ class TestMozconfigLoader(unittest.TestCase): self.assertTrue(e.exception.message.startswith( 'Evaluation of your mozconfig exited with an error')) self.assertEquals(e.exception.path, - mozconfig.name.replace(os.sep, '/')) + mozconfig.name.replace(os.sep, '/')) self.assertEquals(e.exception.output, ['hello world']) diff --git a/python/mozbuild/mozbuild/test/test_mozinfo.py b/python/mozbuild/mozbuild/test/test_mozinfo.py index 6ae3da566ec4..b69b4b8fe306 100755 --- a/python/mozbuild/mozbuild/test/test_mozinfo.py +++ b/python/mozbuild/mozbuild/test/test_mozinfo.py @@ -201,6 +201,7 @@ class TestWriteMozinfo(unittest.TestCase, Base): """ Test the write_mozinfo function. """ + def setUp(self): fd, self.f = tempfile.mkstemp() os.close(fd) diff --git a/python/mozbuild/mozbuild/test/test_util.py b/python/mozbuild/mozbuild/test/test_util.py index cc0761e0e21a..1af1c5959545 100644 --- a/python/mozbuild/mozbuild/test/test_util.py +++ b/python/mozbuild/mozbuild/test/test_util.py @@ -108,6 +108,7 @@ class TestFileAvoidWrite(unittest.TestCase): '''MockedOpen extension to raise an exception if something attempts to write in an opened file. ''' + def __call__(self, name, mode): if 'w' in mode: raise Exception('Unexpected open with write mode') @@ -161,6 +162,7 @@ class TestFileAvoidWrite(unittest.TestCase): finally: shutil.rmtree(tmpdir) + class TestResolveTargetToMake(unittest.TestCase): def setUp(self): self.topobjdir = data_path @@ -195,20 +197,25 @@ class TestResolveTargetToMake(unittest.TestCase): def test_regular_file(self): self.assertResolve('test-dir/with/file', ('test-dir/with', 'file')) self.assertResolve('test-dir/with/without/file', ('test-dir/with', 'without/file')) - self.assertResolve('test-dir/with/without/with/file', ('test-dir/with/without/with', 'file')) + self.assertResolve('test-dir/with/without/with/file', + ('test-dir/with/without/with', 'file')) self.assertResolve('test-dir/without/file', ('test-dir', 'without/file')) self.assertResolve('test-dir/without/with/file', ('test-dir/without/with', 'file')) - self.assertResolve('test-dir/without/with/without/file', ('test-dir/without/with', 'without/file')) + self.assertResolve('test-dir/without/with/without/file', + ('test-dir/without/with', 'without/file')) def test_Makefile(self): self.assertResolve('test-dir/with/Makefile', ('test-dir', 'with/Makefile')) self.assertResolve('test-dir/with/without/Makefile', ('test-dir/with', 'without/Makefile')) - self.assertResolve('test-dir/with/without/with/Makefile', ('test-dir/with', 'without/with/Makefile')) + self.assertResolve('test-dir/with/without/with/Makefile', + ('test-dir/with', 'without/with/Makefile')) self.assertResolve('test-dir/without/Makefile', ('test-dir', 'without/Makefile')) self.assertResolve('test-dir/without/with/Makefile', ('test-dir', 'without/with/Makefile')) - self.assertResolve('test-dir/without/with/without/Makefile', ('test-dir/without/with', 'without/Makefile')) + self.assertResolve('test-dir/without/with/without/Makefile', + ('test-dir/without/with', 'without/Makefile')) + class TestHierarchicalStringList(unittest.TestCase): def setUp(self): @@ -224,18 +231,18 @@ class TestHierarchicalStringList(unittest.TestCase): def test_exports_subdir(self): self.assertEqual(self.EXPORTS._children, {}) self.EXPORTS.foo += ["foo.h"] - self.assertItemsEqual(self.EXPORTS._children, {"foo" : True}) + self.assertItemsEqual(self.EXPORTS._children, {"foo": True}) self.assertEqual(self.EXPORTS.foo._strings, ["foo.h"]) self.EXPORTS.bar += ["bar.h"] self.assertItemsEqual(self.EXPORTS._children, - {"foo" : True, "bar" : True}) + {"foo": True, "bar": True}) self.assertEqual(self.EXPORTS.foo._strings, ["foo.h"]) self.assertEqual(self.EXPORTS.bar._strings, ["bar.h"]) def test_exports_multiple_subdir(self): self.EXPORTS.foo.bar = ["foobar.h"] - self.assertItemsEqual(self.EXPORTS._children, {"foo" : True}) - self.assertItemsEqual(self.EXPORTS.foo._children, {"bar" : True}) + self.assertItemsEqual(self.EXPORTS._children, {"foo": True}) + self.assertItemsEqual(self.EXPORTS.foo._children, {"bar": True}) self.assertItemsEqual(self.EXPORTS.foo.bar._children, {}) self.assertEqual(self.EXPORTS._strings, []) self.assertEqual(self.EXPORTS.foo._strings, []) @@ -866,6 +873,7 @@ class TestMisc(unittest.TestCase): 'before abc between a b c after' ) + class TestEnumString(unittest.TestCase): def test_string(self): CompilerType = EnumString.subclass('gcc', 'clang', 'clang-cl') diff --git a/python/mozbuild/mozbuild/testing.py b/python/mozbuild/mozbuild/testing.py index 2e5a6b6efafe..d1fe1d7114e5 100644 --- a/python/mozbuild/mozbuild/testing.py +++ b/python/mozbuild/mozbuild/testing.py @@ -68,11 +68,13 @@ REFTEST_FLAVORS = ('crashtest', 'reftest') # Web platform tests have their own manifest format and are processed separately. WEB_PLATFORM_TESTS_FLAVORS = ('web-platform-tests',) + def all_test_flavors(): return ([v[0] for v in TEST_MANIFESTS.values()] + list(REFTEST_FLAVORS) + list(WEB_PLATFORM_TESTS_FLAVORS)) + class TestInstallInfo(object): def __init__(self): self.seen = set() @@ -88,6 +90,7 @@ class TestInstallInfo(object): self.deferred_installs |= other.deferred_installs return self + class SupportFilesConverter(object): """Processes a "support-files" entry from a test object, either from a parsed object from a test manifests or its representation in @@ -97,6 +100,7 @@ class SupportFilesConverter(object): effect, and the structure of the parsed objects from manifests will have a lot of repeated entries, so this class takes care of memoizing. """ + def __init__(self): self._fields = (('head', set()), ('support-files', set()), @@ -175,6 +179,7 @@ class SupportFilesConverter(object): info.installs.append((full, mozpath.normpath(dest_path))) return info + def _resolve_installs(paths, topobjdir, manifest): """Using the given paths as keys, find any unresolved installs noted by the build backend corresponding to those keys, and add them @@ -188,9 +193,9 @@ def _resolve_installs(paths, topobjdir, manifest): path = path[2:] if path not in resolved_installs: raise Exception('A cross-directory support file path noted in a ' - 'test manifest does not appear in any other manifest.\n "%s" ' - 'must appear in another test manifest to specify an install ' - 'for "!/%s".' % (path, path)) + 'test manifest does not appear in any other manifest.\n "%s" ' + 'must appear in another test manifest to specify an install ' + 'for "!/%s".' % (path, path)) installs = resolved_installs[path] for install_info in installs: try: @@ -290,6 +295,7 @@ def read_manifestparser_manifest(context, manifest_path): finder=context._finder, handle_defaults=False) + def read_reftest_manifest(context, manifest_path): import reftest path = manifest_path.full_path @@ -297,6 +303,7 @@ def read_reftest_manifest(context, manifest_path): manifest.load(path) return manifest + def read_wpt_manifest(context, paths): manifest_path, tests_root = paths full_path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path)) diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py index d60b5729465c..87d9ecf34c96 100644 --- a/python/mozbuild/mozbuild/util.py +++ b/python/mozbuild/mozbuild/util.py @@ -85,12 +85,14 @@ class EmptyValue(unicode): :py:class:`mozbuild.frontend.reader.EmptyConfig`. It should likely not be used elsewhere. """ + def __init__(self): super(EmptyValue, self).__init__() class ReadOnlyNamespace(object): """A class for objects with immutable attributes set at initialization.""" + def __init__(self, **kwargs): for k, v in kwargs.iteritems(): super(ReadOnlyNamespace, self).__setattr__(k, v) @@ -114,6 +116,7 @@ class ReadOnlyNamespace(object): class ReadOnlyDict(dict): """A read-only dictionary.""" + def __init__(self, *args, **kwargs): dict.__init__(self, *args, **kwargs) @@ -136,6 +139,7 @@ undefined = undefined_default() class ReadOnlyDefaultDict(ReadOnlyDict): """A read-only dictionary that supports default values on retrieval.""" + def __init__(self, default_factory, *args, **kwargs): ReadOnlyDict.__init__(self, *args, **kwargs) self._default_factory = default_factory @@ -215,6 +219,7 @@ class FileAvoidWrite(BytesIO): out, but reports whether the file was existing and would have been updated still occur, as well as diff capture if requested. """ + def __init__(self, filename, capture_diff=False, dry_run=False, mode='rU'): BytesIO.__init__(self) self.name = filename @@ -292,6 +297,7 @@ class FileAvoidWrite(BytesIO): def __enter__(self): return self + def __exit__(self, type, value, traceback): if not self.closed: self.close() @@ -428,7 +434,7 @@ class UnsortedError(Exception): s.write('An attempt was made to add an unsorted sequence to a list. ') s.write('The incoming list is unsorted starting at element %d. ' % - self.i) + self.i) s.write('We expected "%s" but got "%s"' % ( self.sorted[self.i], self.original[self.i])) @@ -467,7 +473,7 @@ class StrictOrderingOnAppendListMixin(object): StrictOrderingOnAppendListMixin.ensure_sorted(sequence) return super(StrictOrderingOnAppendListMixin, self).__setslice__(i, j, - sequence) + sequence) def __add__(self, other): StrictOrderingOnAppendListMixin.ensure_sorted(other) @@ -481,16 +487,18 @@ class StrictOrderingOnAppendListMixin(object): class StrictOrderingOnAppendList(ListMixin, StrictOrderingOnAppendListMixin, - list): + list): """A list specialized for moz.build environments. We overload the assignment and append operations to require that incoming elements be ordered. This enforces cleaner style in moz.build files. """ + class ImmutableStrictOrderingOnAppendList(StrictOrderingOnAppendList): """Like StrictOrderingOnAppendList, but not allowing mutations of the value. """ + def append(self, elt): raise Exception("cannot use append on this type") @@ -509,6 +517,7 @@ class ImmutableStrictOrderingOnAppendList(StrictOrderingOnAppendList): class ListWithActionMixin(object): """Mixin to create lists with pre-processing. See ListWithAction.""" + def __init__(self, iterable=None, action=None): if iterable is None: iterable = [] @@ -532,8 +541,9 @@ class ListWithActionMixin(object): other = [self._action(i) for i in other] return super(ListWithActionMixin, self).__iadd__(other) + class StrictOrderingOnAppendListWithAction(StrictOrderingOnAppendListMixin, - ListMixin, ListWithActionMixin, list): + ListMixin, ListWithActionMixin, list): """An ordered list that accepts a callable to be applied to each item. A callable (action) passed to the constructor is run on each item of input. @@ -542,6 +552,7 @@ class StrictOrderingOnAppendListWithAction(StrictOrderingOnAppendListMixin, Note that the order of superclasses is therefore significant. """ + class ListWithAction(ListMixin, ListWithActionMixin, list): """A list that accepts a callable to be applied to each item. @@ -550,6 +561,7 @@ class ListWithAction(ListMixin, ListWithActionMixin, list): stored in place of the original input. """ + class MozbuildDeletionError(Exception): pass @@ -839,7 +851,7 @@ class LockFile(object): raise -def lock_file(lockfile, max_wait = 600): +def lock_file(lockfile, max_wait=600): """Create and hold a lockfile of the given name, with the given timeout. To release the lock, delete the returned object. @@ -867,8 +879,8 @@ def lock_file(lockfile, max_wait = 600): s = os.stat(lockfile) except EnvironmentError as e: if e.errno == errno.ENOENT or e.errno == errno.EACCES: - # We didn't create the lockfile, so it did exist, but it's - # gone now. Just try again + # We didn't create the lockfile, so it did exist, but it's + # gone now. Just try again continue raise Exception('{0} exists but stat() failed: {1}'.format( @@ -880,7 +892,7 @@ def lock_file(lockfile, max_wait = 600): if now - s[stat.ST_MTIME] > max_wait: pid = f.readline().rstrip() raise Exception('{0} has been locked for more than ' - '{1} seconds (PID {2})'.format(lockfile, max_wait, pid)) + '{1} seconds (PID {2})'.format(lockfile, max_wait, pid)) # It's not been locked too long, wait a while and retry f.close() @@ -897,6 +909,7 @@ def lock_file(lockfile, max_wait = 600): class OrderedDefaultDict(OrderedDict): '''A combination of OrderedDict and defaultdict.''' + def __init__(self, default_factory, *args, **kwargs): OrderedDict.__init__(self, *args, **kwargs) self._default_factory = default_factory @@ -909,6 +922,7 @@ class OrderedDefaultDict(OrderedDict): class KeyedDefaultDict(dict): '''Like a defaultdict, but the default_factory function takes the key as argument''' + def __init__(self, default_factory, *args, **kwargs): dict.__init__(self, *args, **kwargs) self._default_factory = default_factory @@ -929,6 +943,7 @@ class memoize(dict): Both functions and instance methods are handled, although in the instance method case, the results are cache in the instance itself. ''' + def __init__(self, func): self.func = func functools.update_wrapper(self, func) @@ -956,6 +971,7 @@ class memoized_property(object): '''A specialized version of the memoize decorator that works for class instance properties. ''' + def __init__(self, func): self.func = func @@ -1007,7 +1023,7 @@ def TypedNamedTuple(name, fields): if not isinstance(value, ftype): raise TypeError('field in tuple not of proper type: %s; ' 'got %s, expected %s' % (fname, - type(value), ftype)) + type(value), ftype)) super(TypedTuple, self).__init__(*args, **kwargs) @@ -1041,7 +1057,7 @@ class TypedListMixin(object): sequence = self._ensure_type(sequence) return super(TypedListMixin, self).__setslice__(i, j, - sequence) + sequence) def __add__(self, other): other = self._ensure_type(other) @@ -1078,6 +1094,7 @@ def TypedList(type, base_class=List): return _TypedList + def group_unified_files(files, unified_prefix, unified_suffix, files_per_unified_file): """Return an iterator of (unified_filename, source_filenames) tuples. @@ -1101,6 +1118,7 @@ def group_unified_files(files, unified_prefix, unified_suffix, # don't want the fill value inserted by izip_longest to be an # issue. So we do a little dance to filter it out ourselves. dummy_fill_value = ("dummy",) + def filter_out_dummy(iterable): return itertools.ifilter(lambda x: x != dummy_fill_value, iterable) @@ -1152,6 +1170,7 @@ def expand_variables(s, variables): class DefinesAction(argparse.Action): '''An ArgumentParser action to handle -Dvar[=value] type of arguments.''' + def __call__(self, parser, namespace, values, option_string): defines = getattr(namespace, self.dest) if defines is None: @@ -1180,6 +1199,7 @@ class EnumString(unicode): subclasses. ''' POSSIBLE_VALUES = () + def __init__(self, value): if value not in self.POSSIBLE_VALUES: raise ValueError("'%s' is not a valid value for %s" @@ -1209,6 +1229,7 @@ def _escape_char(c): return "\\'" return unicode(c.encode('unicode_escape')) + # Mapping table between raw characters below \x80 and their escaped # counterpart, when they differ _INDENTED_REPR_TABLE = { @@ -1229,6 +1250,7 @@ def indented_repr(o, indent=4): assumes `from __future__ import unicode_literals`. ''' one_indent = ' ' * indent + def recurse_indented_repr(o, level): if isinstance(o, dict): yield '{\n' @@ -1329,6 +1351,7 @@ def patch_main(): import os import sys orig_find_module = imp.find_module + def my_find_module(name, dirs): if name == main_module_name: path = os.path.join(dirs[0], main_file_name) @@ -1338,6 +1361,7 @@ def patch_main(): # Don't allow writing bytecode file for the main module. orig_load_module = imp.load_module + def my_load_module(name, file, path, description): # multiprocess.forking invokes imp.load_module manually and # hard-codes the name __parents_main__ as the module name. @@ -1353,7 +1377,8 @@ def patch_main(): imp.find_module = my_find_module imp.load_module = my_load_module - from multiprocessing.forking import main; main() + from multiprocessing.forking import main + main() def my_get_command_line(): fork_code, lineno = inspect.getsourcelines(fork_interpose) diff --git a/python/mozbuild/mozbuild/vendor_aom.py b/python/mozbuild/mozbuild/vendor_aom.py index da0040266a5c..62498b35608a 100644 --- a/python/mozbuild/mozbuild/vendor_aom.py +++ b/python/mozbuild/mozbuild/vendor_aom.py @@ -19,6 +19,7 @@ import sys import tarfile from urlparse import urlparse + class VendorAOM(MozbuildObject): def upstream_snapshot(self, revision): '''Construct a url for a tarball snapshot of the given revision.''' @@ -137,7 +138,7 @@ Please set a repository url with --repo on either googlesource or github.''' % h mozfile.remove(mozpath.join(target, '.gitattributes')) mozfile.remove(mozpath.join(target, '.gitignore')) mozfile.remove(mozpath.join(target, 'build', '.gitattributes')) - mozfile.remove(mozpath.join(target, 'build' ,'.gitignore')) + mozfile.remove(mozpath.join(target, 'build', '.gitignore')) def generate_sources(self, target): ''' diff --git a/python/mozbuild/mozbuild/vendor_dav1d.py b/python/mozbuild/mozbuild/vendor_dav1d.py index 27b98c980658..5518497c130f 100644 --- a/python/mozbuild/mozbuild/vendor_dav1d.py +++ b/python/mozbuild/mozbuild/vendor_dav1d.py @@ -19,6 +19,7 @@ import sys import tarfile from urlparse import urlparse + class VendorDav1d(MozbuildObject): def upstream_snapshot(self, revision): '''Construct a url for a tarball snapshot of the given revision.''' @@ -96,8 +97,8 @@ Please set a repository url with --repo on either googlesource or github.''' % h prefix = ' release: commit' if prefix in yaml: new_yaml = re.sub(prefix + ' [v\.a-f0-9]+.*$', - prefix + ' %s (%s).' % (revision, timestamp), - yaml, flags=re.MULTILINE) + prefix + ' %s (%s).' % (revision, timestamp), + yaml, flags=re.MULTILINE) else: new_yaml = '%s\n\n%s %s.' % (yaml, prefix, revision) @@ -114,13 +115,12 @@ Please set a repository url with --repo on either googlesource or github.''' % h with open(dst_filename, 'w') as f: f.write(vcs_version) - def clean_upstream(self, target): '''Remove files we don't want to import.''' mozfile.remove(mozpath.join(target, '.gitattributes')) mozfile.remove(mozpath.join(target, '.gitignore')) mozfile.remove(mozpath.join(target, 'build', '.gitattributes')) - mozfile.remove(mozpath.join(target, 'build' ,'.gitignore')) + mozfile.remove(mozpath.join(target, 'build', '.gitignore')) def check_modified_files(self): ''' diff --git a/python/mozbuild/mozbuild/vendor_python.py b/python/mozbuild/mozbuild/vendor_python.py index c5e8d00fb3dd..71bc0f221bf0 100644 --- a/python/mozbuild/mozbuild/vendor_python.py +++ b/python/mozbuild/mozbuild/vendor_python.py @@ -31,7 +31,8 @@ class VendorPython(MozbuildObject): self._activate_virtualenv() pip_compile = os.path.join(self.virtualenv_manager.bin_path, 'pip-compile') if not os.path.exists(pip_compile): - path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party', 'python', 'pip-tools')) + path = os.path.normpath(os.path.join( + self.topsrcdir, 'third_party', 'python', 'pip-tools')) self.virtualenv_manager.install_pip_package(path, vendored=True) spec = os.path.join(vendor_dir, 'requirements.in') requirements = os.path.join(vendor_dir, 'requirements.txt') diff --git a/python/mozbuild/mozbuild/vendor_rust.py b/python/mozbuild/mozbuild/vendor_rust.py index 24689c96b937..e4ee1674cc2b 100644 --- a/python/mozbuild/mozbuild/vendor_rust.py +++ b/python/mozbuild/mozbuild/vendor_rust.py @@ -18,6 +18,7 @@ import re import subprocess import sys + class VendorRust(MozbuildObject): def get_cargo_path(self): try: @@ -62,7 +63,8 @@ class VendorRust(MozbuildObject): on the user. Allow changes to Cargo.{toml,lock} since that's likely to be a common use case. ''' - modified = [f for f in self.repository.get_changed_files('M') if os.path.basename(f) not in ('Cargo.toml', 'Cargo.lock')] + modified = [f for f in self.repository.get_changed_files( + 'M') if os.path.basename(f) not in ('Cargo.toml', 'Cargo.lock')] if modified: self.log(logging.ERROR, 'modified_files', {}, '''You have uncommitted changes to the following files: @@ -108,23 +110,28 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- ''' cargo = self.get_cargo_path() if not self.check_cargo_version(cargo): - self.log(logging.ERROR, 'cargo_version', {}, 'Cargo >= 0.13 required (install Rust 1.12 or newer)') + self.log(logging.ERROR, 'cargo_version', {}, + 'Cargo >= 0.13 required (install Rust 1.12 or newer)') return None else: self.log(logging.DEBUG, 'cargo_version', {}, 'cargo is new enough') - have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output([cargo, '--list']).splitlines()) + have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output( + [cargo, '--list']).splitlines()) if not have_vendor: - self.log(logging.INFO, 'installing', {}, 'Installing cargo-vendor (this may take a few minutes)...') + self.log(logging.INFO, 'installing', {}, + 'Installing cargo-vendor (this may take a few minutes)...') env = self.check_openssl() self.run_process(args=[cargo, 'install', 'cargo-vendor'], append_env=env) elif not self.check_cargo_vendor_version(cargo): - self.log(logging.INFO, 'cargo_vendor', {}, 'cargo-vendor >= 0.1.23 required; force-reinstalling (this may take a few minutes)...') + self.log(logging.INFO, 'cargo_vendor', { + }, 'cargo-vendor >= 0.1.23 required; force-reinstalling (this may take a few minutes)...') env = self.check_openssl() self.run_process(args=[cargo, 'install', '--force', 'cargo-vendor'], append_env=env) else: - self.log(logging.DEBUG, 'cargo_vendor', {}, 'sufficiently new cargo-vendor is already installed') + self.log(logging.DEBUG, 'cargo_vendor', {}, + 'sufficiently new cargo-vendor is already installed') return cargo @@ -230,7 +237,7 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- if not self.runtime_license(package, license): if license not in self.BUILDTIME_LICENSE_WHITELIST: self.log(logging.ERROR, 'package_license_error', {}, - '''Package {} has a non-approved license: {}. + '''Package {} has a non-approved license: {}. Please request license review on the package's license. If the package's license is approved, please add it to the whitelist of suitable licenses. @@ -238,7 +245,7 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- return False elif package not in self.BUILDTIME_LICENSE_WHITELIST[license]: self.log(logging.ERROR, 'package_license_error', {}, - '''Package {} has a license that is approved for build-time dependencies: {} + '''Package {} has a license that is approved for build-time dependencies: {} but the package itself is not whitelisted as being a build-time only package. If your package is build-time only, please add it to the whitelist of build-time @@ -258,8 +265,10 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- # to scanning individual lines. with open(toml_file, 'r') as f: license_lines = [l for l in f if l.strip().startswith(b'license')] - license_matches = list(filter(lambda x: x, [LICENSE_LINE_RE.match(l) for l in license_lines])) - license_file_matches = list(filter(lambda x: x, [LICENSE_FILE_LINE_RE.match(l) for l in license_lines])) + license_matches = list( + filter(lambda x: x, [LICENSE_LINE_RE.match(l) for l in license_lines])) + license_file_matches = list( + filter(lambda x: x, [LICENSE_FILE_LINE_RE.match(l) for l in license_lines])) # License information is optional for crates to provide, but # we require it. @@ -295,7 +304,8 @@ to the whitelist of packages whose licenses are suitable. return False approved_hash = self.RUNTIME_LICENSE_FILE_PACKAGE_WHITELIST[package] - license_contents = open(os.path.join(vendor_dir, package, license_file), 'r').read() + license_contents = open(os.path.join( + vendor_dir, package, license_file), 'r').read() current_hash = hashlib.sha256(license_contents).hexdigest() if current_hash != approved_hash: self.log(logging.ERROR, 'package_license_file_mismatch', {}, @@ -311,7 +321,8 @@ license file's hash. # Force all of the packages to be checked for license information # before reducing via `all`, so all license issues are found in a # single `mach vendor rust` invocation. - results = [check_package(p) for p in os.listdir(vendor_dir) if os.path.isdir(os.path.join(vendor_dir, p))] + results = [check_package(p) for p in os.listdir(vendor_dir) + if os.path.isdir(os.path.join(vendor_dir, p))] return all(results) def vendor(self, ignore_modified=False, @@ -332,7 +343,8 @@ license file's hash. # We do an |update -p| here to regenerate the Cargo.lock file with minimal changes. See bug 1324462 subprocess.check_call([cargo, 'update', '-p', 'gkrust'], cwd=self.topsrcdir) - subprocess.check_call([cargo, 'vendor', '--quiet', '--sync', 'Cargo.lock'] + [vendor_dir], cwd=self.topsrcdir) + subprocess.check_call([cargo, 'vendor', '--quiet', '--sync', + 'Cargo.lock'] + [vendor_dir], cwd=self.topsrcdir) if not self._check_licenses(vendor_dir): self.log(logging.ERROR, 'license_check_failed', {}, diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py index 17c7268de7aa..8973893a6d41 100644 --- a/python/mozbuild/mozbuild/virtualenv.py +++ b/python/mozbuild/mozbuild/virtualenv.py @@ -47,13 +47,14 @@ class VirtualenvManager(object): """Contains logic for managing virtualenvs for building the tree.""" def __init__(self, topsrcdir, topobjdir, virtualenv_path, log_handle, - manifest_path): + manifest_path): """Create a new manager. Each manager is associated with a source directory, a path where you want the virtualenv to be created, and a handle to write output to. """ - assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path) + assert os.path.isabs( + manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path) self.topsrcdir = topsrcdir self.topobjdir = topobjdir self.virtualenv_root = virtualenv_path @@ -71,7 +72,7 @@ class VirtualenvManager(object): def virtualenv_script_path(self): """Path to virtualenv's own populator script.""" return os.path.join(self.topsrcdir, 'third_party', 'python', - 'virtualenv', 'virtualenv.py') + 'virtualenv', 'virtualenv.py') @property def bin_path(self): @@ -127,7 +128,7 @@ class VirtualenvManager(object): # check if virtualenv exists if not os.path.exists(self.virtualenv_root) or \ - not os.path.exists(self.activate_path): + not os.path.exists(self.activate_path): return False @@ -198,12 +199,12 @@ class VirtualenvManager(object): env.pop('PYTHONDONTWRITEBYTECODE', None) args = [python, self.virtualenv_script_path, - # Without this, virtualenv.py may attempt to contact the outside - # world and search for or download a newer version of pip, - # setuptools, or wheel. This is bad for security, reproducibility, - # and speed. - '--no-download', - self.virtualenv_root] + # Without this, virtualenv.py may attempt to contact the outside + # world and search for or download a newer version of pip, + # setuptools, or wheel. This is bad for security, reproducibility, + # and speed. + '--no-download', + self.virtualenv_root] result = self._log_process_output(args, env=env) @@ -279,7 +280,7 @@ class VirtualenvManager(object): assert len(package) >= 2 self.call_setup(os.path.join(self.topsrcdir, package[1]), - package[2:]) + package[2:]) return True @@ -330,9 +331,9 @@ class VirtualenvManager(object): handle_package(package[1:]) return True except: - print('Error processing command. Ignoring', \ - 'because optional. (%s)' % ':'.join(package), - file=self.log_handle) + print('Error processing command. Ignoring', + 'because optional. (%s)' % ':'.join(package), + file=self.log_handle) return False if package[0] in ('windows', '!windows'): @@ -376,7 +377,7 @@ class VirtualenvManager(object): # Python is smart enough to find a proper compiler and to use the # proper compiler flags. If it isn't your Python is likely broken. IGNORE_ENV_VARIABLES = ('CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', - 'PYTHONDONTWRITEBYTECODE') + 'PYTHONDONTWRITEBYTECODE') try: old_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None) @@ -405,13 +406,13 @@ class VirtualenvManager(object): # explained at # http://stackoverflow.com/questions/3047542/building-lxml-for-python-2-7-on-windows/5122521#5122521. if sys.platform in ('win32', 'cygwin') and \ - 'VS90COMNTOOLS' not in os.environ: + 'VS90COMNTOOLS' not in os.environ: warnings.warn('Hacking environment to allow binary Python ' - 'extensions to build. You can make this warning go away ' - 'by installing Visual Studio 2008. You can download the ' - 'Express Edition installer from ' - 'http://go.microsoft.com/?linkid=7729279') + 'extensions to build. You can make this warning go away ' + 'by installing Visual Studio 2008. You can download the ' + 'Express Edition installer from ' + 'http://go.microsoft.com/?linkid=7729279') # We list in order from oldest to newest to prefer the closest # to 2008 so differences are minimized. @@ -476,7 +477,7 @@ class VirtualenvManager(object): # the virtualenv for paths to be proper. args = [self.python_path, __file__, 'populate', self.topsrcdir, - self.topobjdir, self.virtualenv_root, self.manifest_path] + self.topobjdir, self.virtualenv_root, self.manifest_path] result = self._log_process_output(args, cwd=self.topsrcdir) @@ -672,7 +673,7 @@ if __name__ == '__main__': topsrcdir, topobjdir, virtualenv_path, manifest_path = sys.argv[2:] manager = VirtualenvManager(topsrcdir, topobjdir, virtualenv_path, - sys.stdout, manifest_path) + sys.stdout, manifest_path) if populate: manager.populate()