diff --git a/build/moz.configure/node.configure b/build/moz.configure/node.configure index 1c8d55fa070f..ba2003d197ad 100644 --- a/build/moz.configure/node.configure +++ b/build/moz.configure/node.configure @@ -8,13 +8,20 @@ option("--disable-nodejs", help="Require Node.js to build") option(env="NODEJS", nargs=1, help="Path to nodejs") -@depends("--enable-nodejs", "NODEJS") +@depends("--enable-nodejs", "NODEJS", bootstrap_search_path("node")) @checking( "for nodejs", callback=lambda x: "%s (%s)" % (x.path, x.str_version) if x else "no" ) @imports(_from="mozbuild.nodeutil", _import="find_node_executable") @imports(_from="mozbuild.nodeutil", _import="NODE_MIN_VERSION") -def nodejs(require, env_node): +def nodejs(require, env_node, search_path): + # We don't use the dependency directly, but having it ensures the + # auto-upgrade code in bootstrap_search_path is triggered, while + # find_node_executable will use more or less the same search path. + # We do however need to use the variable for the configure lint + # not to fail. + search_path + node_exe = env_node[0] if env_node else None nodejs, version = find_node_executable(node_exe) diff --git a/moz.configure b/moz.configure index 29952a66ade2..c7071cb60018 100755 --- a/moz.configure +++ b/moz.configure @@ -239,18 +239,116 @@ def original_path(): return environ["PATH"].split(os.pathsep) +option( + "--enable-bootstrap", + when=developer_options, + help="Automatically update/bootstrap some toolchains when they are present but out of date", +) + + +@depends("--enable-bootstrap", when=developer_options) +def bootstrap(value): + if value: + return True + + +@depends(host, bootstrap_search_path_order, when=bootstrap) +@imports("os") +@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions") +@imports(_from="__builtin__", _import="Exception") +def bootstrap_toolchain_tasks(host, order): + prefix = { + ("x86_64", "GNU", "Linux"): "linux64", + ("x86_64", "OSX", "Darwin"): "macosx64", + ("x86_64", "WINNT", "WINNT"): "win64", + }.get((host.cpu, host.os, host.kernel)) + if prefix: + try: + return namespace(prefix=prefix, tasks=toolchain_task_definitions()) + except Exception: + return None + + @template def bootstrap_search_path(*path_parts, **kwargs): when = kwargs.pop("when", None) if kwargs: configure_error("bootstrap_search_path only takes `when` as keyword argument") - @depends(bootstrap_search_path_order, original_path, toolchains_base_dir, when=when) - def bootstrap_search_path(order, original_path, toolchains_base_dir): - path = [os.path.join(toolchains_base_dir, *path_parts)] + @depends( + bootstrap, + bootstrap_search_path_order, + original_path, + toolchains_base_dir, + bootstrap_toolchain_tasks, + shell, + check_build_environment, + when=when, + ) + @imports("os") + @imports("subprocess") + @imports(_from="mozbuild.util", _import="ensureParentDir") + @imports(_from="__builtin__", _import="open") + @imports(_from="__builtin__", _import="Exception") + def bootstrap_search_path( + bootstrap, order, original_path, toolchains_base_dir, tasks, shell, build_env + ): + def try_bootstrap(): + label = "toolchain-{}-{}".format( + tasks.prefix, path_parts[0].replace("_", "-") + ) + task = tasks.tasks.get(label) + if not task: + return + task_index = task.optimization.get("index-search") + if not task_index: + return + task_index = task_index[0].split(".")[-1] + artifact = task.attributes["toolchain-artifact"] + # `mach artifact toolchain` doesn't support authentication for + # private artifacts. + if not artifact.startswith("public/"): + return + index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0]) + try: + with open(index_file) as fh: + index = fh.read().strip() + except Exception: + index = None + if index == task_index: + return + log.info( + "Updating bootstrapped toolchain in %s", + os.path.join(toolchains_base_dir, path_parts[0]), + ) + subprocess.run( + [ + shell, + os.path.join(build_env.topsrcdir, "mach"), + "--log-no-times", + "artifact", + "toolchain", + "--from-build", + label, + ], + cwd=toolchains_base_dir, + check=True, + ) + ensureParentDir(index_file) + with open(index_file, "w") as fh: + fh.write(task_index) + + path = os.path.join(toolchains_base_dir, *path_parts) + # Only bootstrap toolchains that have been bootstrapped at least once. + if bootstrap and os.path.exists(path): + try: + try_bootstrap() + except Exception as e: + log.error("%s", e) + die("If you can't fix the above, retry with --disable-bootstrap.") if order == "prepend": - return path + original_path - return original_path + path + return [path] + original_path + return original_path + [path] return bootstrap_search_path diff --git a/python/mozbuild/mozbuild/artifact_commands.py b/python/mozbuild/mozbuild/artifact_commands.py index cae9c45a423f..852fb1907445 100644 --- a/python/mozbuild/mozbuild/artifact_commands.py +++ b/python/mozbuild/mozbuild/artifact_commands.py @@ -25,7 +25,6 @@ from mozbuild.base import ( MachCommandConditions as conditions, ) from mozbuild.util import ensureParentDir -import mozpack.path as mozpath import mozversioncontrol @@ -397,18 +396,9 @@ class PackageFrontend(MachCommandBase): ) return 1 from taskgraph.optimize.strategies import IndexSearch - from taskgraph.generator import load_tasks_for_kind + from mozbuild.toolchains import toolchain_task_definitions - params = {"level": six.ensure_text(os.environ.get("MOZ_SCM_LEVEL", "3"))} - - root_dir = mozpath.join(self.topsrcdir, "taskcluster/ci") - toolchains = load_tasks_for_kind(params, "toolchain", root_dir=root_dir) - - aliases = {} - for t in toolchains.values(): - alias = t.attributes.get("toolchain-alias") - if alias: - aliases["toolchain-{}".format(alias)] = t.task["metadata"]["name"] + tasks = toolchain_task_definitions() for b in from_build: user_value = b @@ -416,7 +406,7 @@ class PackageFrontend(MachCommandBase): if not b.startswith("toolchain-"): b = "toolchain-{}".format(b) - task = toolchains.get(aliases.get(b, b)) + task = tasks.get(b) if not task: self.log( logging.ERROR, diff --git a/python/mozbuild/mozbuild/toolchains.py b/python/mozbuild/mozbuild/toolchains.py new file mode 100644 index 000000000000..2e51374389a6 --- /dev/null +++ b/python/mozbuild/mozbuild/toolchains.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import +import os + + +def toolchain_task_definitions(): + from taskgraph.generator import load_tasks_for_kind + + # Don't import globally to allow this module being imported without + # the taskgraph module being available (e.g. standalone js) + params = {"level": os.environ.get("MOZ_SCM_LEVEL", "3")} + root_dir = os.path.join( + os.path.dirname(__file__), "..", "..", "..", "taskcluster", "ci" + ) + toolchains = load_tasks_for_kind(params, "toolchain", root_dir=root_dir) + aliased = {} + for t in toolchains.values(): + alias = t.attributes.get("toolchain-alias") + if alias: + aliased["toolchain-{}".format(alias)] = t + toolchains.update(aliased) + + return toolchains