Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-24 21:31:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1320194 - Generate all-tests.pkl and related files when resolving tests r=mshal

Browse Source 
This replaces the 'run-tests-deps' make target with a python function that will directly
read moz.build files, emit them with TestManifestEmitter, then consume them with
TestManifestBackend. Because the TestResolver is the only place that actually reads the
test metadata files, we can remove this logic from the CommonBackend as well.

MozReview-Commit-ID: DXgMoeH5dKf



MozReview-Commit-ID: HstZ57qkqf2

--HG--
extra : rebase_source : f377fa6863ef66d3adb86ed64f844e346686862f
This commit is contained in:
Andrew Halberstadt 2017-02-01 09:56:09 -05:00
parent eb228a736f
commit ea89582f61
8 changed files with 73 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile.in
View File

@ -186,11 +186,6 @@ $(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): instal
.PHONY: install-tests
install-tests: install-test-files
# We no longer run "make install-tests" directly before running tests, but we still
# want to depend on things like config.status, hence this target.
.PHONY: run-tests-deps
run-tests-deps: $(install_manifest_depends)
# Force --no-remove, because $objdir/_tests is handled by multiple manifests.
.PHONY: install-test-files
install-test-files:

32
build/gen_test_backend.py Normal file
View File

@ -0,0 +1,32 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import sys
from mozbuild.backend.test_manifest import TestManifestBackend
from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
from mozbuild.frontend.emitter import TreeMetadataEmitter
from mozbuild.frontend.reader import BuildReader, EmptyConfig
def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
print("No build detected, test metadata may be incomplete.")
config = EmptyConfig(build_obj.topsrcdir)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
if __name__ == '__main__':
sys.exit(gen_test_backend())

8
python/moz.build
View File

@ -23,7 +23,11 @@ SPHINX_TREES['mach'] = 'mach/docs'
PYTHON_UNITTEST_MANIFESTS += [
'mach/mach/test/python.ini',
'mozbuild/dumbmake/test/python.ini',
'mozbuild/mozbuild/test/python.ini',
'mozbuild/mozpack/test/python.ini',
'mozlint/test/python.ini',
]
if CONFIG['MOZ_BUILD_APP']:
PYTHON_UNITTEST_MANIFESTS += [
'mozbuild/mozbuild/test/python.ini',
'mozbuild/mozpack/test/python.ini',
]

4
python/mozbuild/mozbuild/backend/base.py
View File

@ -26,6 +26,7 @@ from ..util import (
simple_diff,
)
from ..frontend.data import ContextDerived
from ..frontend.reader import EmptyConfig
from .configenvironment import ConfigEnvironment
from mozbuild.base import ExecutionSummary
@ -41,8 +42,7 @@ class BuildBackend(LoggingMixin):
__metaclass__ = ABCMeta
def __init__(self, environment):
assert isinstance(environment, ConfigEnvironment)
assert isinstance(environment, (ConfigEnvironment, EmptyConfig))
self.populate_logger()
self.environment = environment

16
python/mozbuild/mozbuild/backend/common.py
View File

@ -11,7 +11,6 @@ import os
import mozpack.path as mozpath
from mozbuild.backend.base import BuildBackend
from mozbuild.backend.test_manifest import TestManifestBackend
from mozbuild.frontend.context import (
Context,
@ -32,7 +31,6 @@ from mozbuild.frontend.data import (
PreprocessedTestWebIDLFile,
PreprocessedWebIDLFile,
SharedLibrary,
TestManifest,
TestWebIDLFile,
UnifiedSources,
XPIDLFile,
@ -45,8 +43,6 @@ from mozbuild.jar import (
from mozbuild.preprocessor import Preprocessor
from mozpack.chrome.manifest import parse_manifest_line
from collections import defaultdict
from mozbuild.util import group_unified_files
class XPIDLManager(object):
@ -184,17 +180,10 @@ class CommonBackend(BuildBackend):
self._configs = set()
self._ipdl_sources = set()
# Temporarily compose a partial TestManifestBackend, soon test manifest
# processing will no longer be part of the build and this will be removed.
self._test_backend = TestManifestBackend(self.environment)
def consume_object(self, obj):
self._configs.add(obj.config)
if isinstance(obj, TestManifest):
self._test_backend.consume_object(obj)
elif isinstance(obj, XPIDLFile):
if isinstance(obj, XPIDLFile):
# TODO bug 1240134 tracks not processing XPIDL files during
# artifact builds.
self._idl_manager.register_idl(obj)
@ -327,9 +316,6 @@ class CommonBackend(BuildBackend):
for config in self._configs:
self.backend_input_files.add(config.source)
# Write out a machine-readable file describing every test.
self._test_backend.consume_finished()
# Write out a machine-readable file describing binaries.
topobjdir = self.environment.topobjdir
with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:

8
python/mozbuild/mozbuild/backend/recursivemake.py
View File

@ -443,15 +443,12 @@ class RecursiveMakeBackend(CommonBackend):
consumed = CommonBackend.consume_object(self, obj)
# CommonBackend handles XPIDLFile and TestManifest, but we want to do
# CommonBackend handles XPIDLFile, but we want to do
# some extra things for them.
if isinstance(obj, XPIDLFile):
backend_file.xpt_name = '%s.xpt' % obj.module
self._idl_dirs.add(obj.relobjdir)
elif isinstance(obj, TestManifest):
self._process_test_manifest(obj, backend_file)
# If CommonBackend acknowledged the object, we're done with it.
if consumed:
return True
@ -657,6 +654,9 @@ class RecursiveMakeBackend(CommonBackend):
elif isinstance(obj, ChromeManifestEntry):
self._process_chrome_manifest_entry(obj, backend_file)
elif isinstance(obj, TestManifest):
self._process_test_manifest(obj, backend_file)
else:
return False

35
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
View File

@ -18,6 +18,7 @@ from mozbuild.backend.recursivemake import (
RecursiveMakeBackend,
RecursiveMakeTraversal,
)
from mozbuild.backend.test_manifest import TestManifestBackend
from mozbuild.frontend.emitter import TreeMetadataEmitter
from mozbuild.frontend.reader import BuildReader
@ -560,21 +561,29 @@ class TestRecursiveMakeBackend(BackendTester):
def test_test_manifest_deffered_installs_written(self):
"""Shared support files are written to their own data file by the backend."""
env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
# First, read the generated for ini manifest contents.
test_files_manifest = mozpath.join(env.topobjdir,
'_build_manifests',
'install',
'_test_files')
m = InstallManifest(path=test_files_manifest)
# Then, synthesize one from the test-installs.pkl file. This should
# allow us to re-create a subset of the above.
env = self._consume('test-manifest-shared-support', TestManifestBackend)
test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl')
with open(test_installs_path, 'r') as fh:
test_installs = pickle.load(fh)
test_files_manifest = mozpath.join(env.topobjdir,
'_build_manifests',
'install',
'_test_files')
self.assertEqual(set(test_installs.keys()),
set(['child/test_sub.js',
'child/data/**',
'child/another-file.sjs']))
for key in test_installs.keys():
self.assertIn(key, test_installs)
# First, read the generated for ini manifest contents.
m = InstallManifest(path=test_files_manifest)
# Then, synthesize one from the test-installs.pkl file. This should
# allow us to re-create a subset of the above.
synthesized_manifest = InstallManifest()
for item, installs in test_installs.items():
for install_info in installs:
@ -941,14 +950,6 @@ class TestRecursiveMakeBackend(BackendTester):
"""Ensure test suites honor package_tests=False."""
env = self._consume('test-manifests-package-tests', RecursiveMakeBackend)
all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
self.assertTrue(os.path.exists(all_tests_path))
with open(all_tests_path, 'rb') as fh:
o = pickle.load(fh)
self.assertIn('mochitest.js', o)
self.assertIn('not_packaged.java', o)
man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
self.assertTrue(os.path.isdir(man_dir))

12
python/mozbuild/mozbuild/testing.py
View File

@ -185,8 +185,16 @@ class TestResolver(MozbuildObject):
# If installing tests is going to result in re-generating the build
# backend, we need to do this here, so that the updated contents of
# all-tests.pkl make it to the set of tests to run.
self._run_make(target='run-tests-deps', pass_thru=True,
print_directory=False)
self._run_make(
target='backend.TestManifestBackend', pass_thru=True, print_directory=False,
filename=mozpath.join(self.topsrcdir, 'build', 'rebuild-backend.mk'),
append_env={
b'PYTHON': self.virtualenv_manager.python_path,
b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend',
b'BACKEND_GENERATION_SCRIPT': mozpath.join(
self.topsrcdir, 'build', 'gen_test_backend.py'),
},
)
self._tests = TestMetadata(os.path.join(self.topobjdir,
'all-tests.pkl'),