Bug 1553230 - Allow to opt-in to automatically update some bootstrapped toolchains. r=firefox-build-system-reviewers,nalexander,mhentges

This adds a --enable-bootstrap build flag that will automatically update
cbindgen, node, clang, sccache, nasm, wine, lucetc, dump_syms, pdbstr,
and winchecksec if they are already installed in ~/.mozbuild.

Eventually, we'll want to allow to install toolchains that weren't
already install, but one step at a time.

This explicitly doesn't cover rustc, which is its own can of worms, or
android-{ndk,sdk}, which are not installed via toolchain artifacts
currently.

Differential Revision: https://phabricator.services.mozilla.com/D101723

build/moz.configure/node.configure

@@ -8,13 +8,20 @@ option("--disable-nodejs", help="Require Node.js to build")
 option(env="NODEJS", nargs=1, help="Path to nodejs")
 
 
-@depends("--enable-nodejs", "NODEJS")
+@depends("--enable-nodejs", "NODEJS", bootstrap_search_path("node"))
 @checking(
     "for nodejs", callback=lambda x: "%s (%s)" % (x.path, x.str_version) if x else "no"
 )
 @imports(_from="mozbuild.nodeutil", _import="find_node_executable")
 @imports(_from="mozbuild.nodeutil", _import="NODE_MIN_VERSION")
-def nodejs(require, env_node):
+def nodejs(require, env_node, search_path):
+    # We don't use the dependency directly, but having it ensures the
+    # auto-upgrade code in bootstrap_search_path is triggered, while
+    # find_node_executable will use more or less the same search path.
+    # We do however need to use the variable for the configure lint
+    # not to fail.
+    search_path
+
     node_exe = env_node[0] if env_node else None
 
     nodejs, version = find_node_executable(node_exe)

moz.configure

@@ -239,18 +239,116 @@ def original_path():
     return environ["PATH"].split(os.pathsep)
 
 
+option(
+    "--enable-bootstrap",
+    when=developer_options,
+    help="Automatically update/bootstrap some toolchains when they are present but out of date",
+)
+
+
+@depends("--enable-bootstrap", when=developer_options)
+def bootstrap(value):
+    if value:
+        return True
+
+
+@depends(host, bootstrap_search_path_order, when=bootstrap)
+@imports("os")
+@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions")
+@imports(_from="__builtin__", _import="Exception")
+def bootstrap_toolchain_tasks(host, order):
+    prefix = {
+        ("x86_64", "GNU", "Linux"): "linux64",
+        ("x86_64", "OSX", "Darwin"): "macosx64",
+        ("x86_64", "WINNT", "WINNT"): "win64",
+    }.get((host.cpu, host.os, host.kernel))
+    if prefix:
+        try:
+            return namespace(prefix=prefix, tasks=toolchain_task_definitions())
+        except Exception:
+            return None
+
+
 @template
 def bootstrap_search_path(*path_parts, **kwargs):
     when = kwargs.pop("when", None)
     if kwargs:
         configure_error("bootstrap_search_path only takes `when` as keyword argument")
 
-    @depends(bootstrap_search_path_order, original_path, toolchains_base_dir, when=when)
-    def bootstrap_search_path(order, original_path, toolchains_base_dir):
-        path = [os.path.join(toolchains_base_dir, *path_parts)]
+    @depends(
+        bootstrap,
+        bootstrap_search_path_order,
+        original_path,
+        toolchains_base_dir,
+        bootstrap_toolchain_tasks,
+        shell,
+        check_build_environment,
+        when=when,
+    )
+    @imports("os")
+    @imports("subprocess")
+    @imports(_from="mozbuild.util", _import="ensureParentDir")
+    @imports(_from="__builtin__", _import="open")
+    @imports(_from="__builtin__", _import="Exception")
+    def bootstrap_search_path(
+        bootstrap, order, original_path, toolchains_base_dir, tasks, shell, build_env
+    ):
+        def try_bootstrap():
+            label = "toolchain-{}-{}".format(
+                tasks.prefix, path_parts[0].replace("_", "-")
+            )
+            task = tasks.tasks.get(label)
+            if not task:
+                return
+            task_index = task.optimization.get("index-search")
+            if not task_index:
+                return
+            task_index = task_index[0].split(".")[-1]
+            artifact = task.attributes["toolchain-artifact"]
+            # `mach artifact toolchain` doesn't support authentication for
+            # private artifacts.
+            if not artifact.startswith("public/"):
+                return
+            index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0])
+            try:
+                with open(index_file) as fh:
+                    index = fh.read().strip()
+            except Exception:
+                index = None
+            if index == task_index:
+                return
+            log.info(
+                "Updating bootstrapped toolchain in %s",
+                os.path.join(toolchains_base_dir, path_parts[0]),
+            )
+            subprocess.run(
+                [
+                    shell,
+                    os.path.join(build_env.topsrcdir, "mach"),
+                    "--log-no-times",
+                    "artifact",
+                    "toolchain",
+                    "--from-build",
+                    label,
+                ],
+                cwd=toolchains_base_dir,
+                check=True,
+            )
+            ensureParentDir(index_file)
+            with open(index_file, "w") as fh:
+                fh.write(task_index)
+
+        path = os.path.join(toolchains_base_dir, *path_parts)
+        # Only bootstrap toolchains that have been bootstrapped at least once.
+        if bootstrap and os.path.exists(path):
+            try:
+                try_bootstrap()
+            except Exception as e:
+                log.error("%s", e)
+                die("If you can't fix the above, retry with --disable-bootstrap.")
         if order == "prepend":
-            return path + original_path
-        return original_path + path
+            return [path] + original_path
+        return original_path + [path]
 
     return bootstrap_search_path
 

python/mozbuild/mozbuild/artifact_commands.py

@@ -25,7 +25,6 @@ from mozbuild.base import (
     MachCommandConditions as conditions,
 )
 from mozbuild.util import ensureParentDir
-import mozpack.path as mozpath
 import mozversioncontrol
 
 
@@ -397,18 +396,9 @@ class PackageFrontend(MachCommandBase):
                 )
                 return 1
             from taskgraph.optimize.strategies import IndexSearch
-            from taskgraph.generator import load_tasks_for_kind
+            from mozbuild.toolchains import toolchain_task_definitions
 
-            params = {"level": six.ensure_text(os.environ.get("MOZ_SCM_LEVEL", "3"))}
-
-            root_dir = mozpath.join(self.topsrcdir, "taskcluster/ci")
-            toolchains = load_tasks_for_kind(params, "toolchain", root_dir=root_dir)
-
-            aliases = {}
-            for t in toolchains.values():
-                alias = t.attributes.get("toolchain-alias")
-                if alias:
-                    aliases["toolchain-{}".format(alias)] = t.task["metadata"]["name"]
+            tasks = toolchain_task_definitions()
 
             for b in from_build:
                 user_value = b
@@ -416,7 +406,7 @@ class PackageFrontend(MachCommandBase):
                 if not b.startswith("toolchain-"):
                     b = "toolchain-{}".format(b)
 
-                task = toolchains.get(aliases.get(b, b))
+                task = tasks.get(b)
                 if not task:
                     self.log(
                         logging.ERROR,

python/mozbuild/mozbuild/toolchains.py

@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import os
+
+
+def toolchain_task_definitions():
+    from taskgraph.generator import load_tasks_for_kind
+
+    # Don't import globally to allow this module being imported without
+    # the taskgraph module being available (e.g. standalone js)
+    params = {"level": os.environ.get("MOZ_SCM_LEVEL", "3")}
+    root_dir = os.path.join(
+        os.path.dirname(__file__), "..", "..", "..", "taskcluster", "ci"
+    )
+    toolchains = load_tasks_for_kind(params, "toolchain", root_dir=root_dir)
+    aliased = {}
+    for t in toolchains.values():
+        alias = t.attributes.get("toolchain-alias")
+        if alias:
+            aliased["toolchain-{}".format(alias)] = t
+    toolchains.update(aliased)
+
+    return toolchains