Bug 1772036 - Add a mach cargo vet command. r=firefox-build-system-reviewers,ahochheiden

The cargo-vet toolchain is auto-bootstrapped and setup for things to
work properly. We modify `mach vendor rust` to invoke `mach cargo vet`
instead of doing its own setup, but in a underhanded way to work around
bug 1772453.

Differential Revision: https://phabricator.services.mozilla.com/D148218

python/mozbuild/mozbuild/bootstrap.py

@@ -39,5 +39,6 @@ def _bootstrap_sandbox():
 def bootstrap_toolchain(toolchain_job):
     # Expand the `bootstrap_path` template for the given toolchain_job, and execute the
     # expanded function via `_value_for`, which will trigger autobootstrap.
+    # Returns the path to the toolchain.
     sandbox = _bootstrap_sandbox()
-    sandbox._value_for(sandbox["bootstrap_path"](toolchain_job))
+    return sandbox._value_for(sandbox["bootstrap_path"](toolchain_job))

python/mozbuild/mozbuild/mach_commands.py

@@ -22,6 +22,7 @@ import errno
 import mozbuild.settings  # noqa need @SettingsProvider hook to execute
 import mozpack.path as mozpath
 
+from pathlib import Path
 from mach.decorators import (
     CommandArgument,
     CommandArgumentGroup,
@@ -178,6 +179,69 @@ def check(command_context, all_crates=None, crates=None, jobs=0, verbose=False):
     return 0
 
 
+@SubCommand(
+    "cargo",
+    "vet",
+    description="Run `cargo vet`.",
+)
+@CommandArgument("arguments", nargs=argparse.REMAINDER)
+def cargo_vet(command_context, arguments, stdout=None, env=os.environ):
+    from mozbuild.bootstrap import bootstrap_toolchain
+
+    # Logging of commands enables logging from `bootstrap_toolchain` that we
+    # don't want to expose. Disable them temporarily.
+    logger = logging.getLogger("gecko_taskgraph.generator")
+    level = logger.getEffectiveLevel()
+    logger.setLevel(logging.ERROR)
+
+    env = env.copy()
+    cargo_vet = bootstrap_toolchain("cargo-vet")
+    if cargo_vet:
+        env["PATH"] = os.pathsep.join([cargo_vet, env["PATH"]])
+    logger.setLevel(level)
+    try:
+        cargo = command_context.substs["CARGO"]
+    except (BuildEnvironmentNotFoundException, KeyError):
+        # Default if this tree isn't configured.
+        from mozfile import which
+
+        cargo = which("cargo", path=env["PATH"])
+        if not cargo:
+            raise OSError(
+                errno.ENOENT,
+                (
+                    "Could not find 'cargo' on your $PATH. "
+                    "Hint: have you run `mach build` or `mach configure`?"
+                ),
+            )
+
+    locked = "--locked" in arguments
+    if locked:
+        # The use of --locked requires .cargo/config to exist, but other things,
+        # like cargo update, don't want it there, so remove it once we're done.
+        topsrcdir = Path(command_context.topsrcdir)
+        shutil.copyfile(
+            topsrcdir / ".cargo" / "config.in", topsrcdir / ".cargo" / "config"
+        )
+
+    try:
+        res = subprocess.run(
+            [cargo, "vet"] + arguments,
+            cwd=command_context.topsrcdir,
+            stdout=stdout,
+            env=env,
+        )
+    finally:
+        if locked:
+            (topsrcdir / ".cargo" / "config").unlink()
+
+    # When the function is invoked without stdout set (the default when running
+    # as a mach subcommand), exit with the returncode from cargo vet.
+    # When the function is invoked with stdout (direct function call), return
+    # the full result from subprocess.run.
+    return res if stdout else res.returncode
+
+
 @Command(
     "doctor",
     category="devenv",

python/mozbuild/mozbuild/vendor/vendor_rust.py

@@ -11,12 +11,12 @@ import json
 import logging
 import os
 import re
-import shutil
 import subprocess
 from collections import defaultdict, OrderedDict
 from distutils.version import LooseVersion
 from itertools import dropwhile
 from mozboot.util import MINIMUM_RUST_VERSION
+from mozbuild.mach_commands import cargo_vet
 from pathlib import Path
 
 import pytoml
@@ -714,36 +714,26 @@ license file's hash.
             env = os.environ.copy()
             env["PATH"] = os.pathsep.join(
                 (
-                    os.path.join(os.environ["MOZ_FETCHES_DIR"], "cargo-vet"),
                     os.path.join(os.environ["MOZ_FETCHES_DIR"], "rustc", "bin"),
                     os.environ["PATH"],
                 )
             )
-            # The use of --locked requires .cargo/config to exist, but other things,
-            # like cargo update, don't want it there, so remove it after cargo vet.
-            topsrcdir = Path(self.topsrcdir)
-            shutil.copyfile(
-                topsrcdir / ".cargo" / "config.in", topsrcdir / ".cargo" / "config"
+            res = cargo_vet(
+                self,
+                ["--output-format=json", "--locked"],
+                stdout=subprocess.PIPE,
+                env=env,
             )
-            try:
-                res = subprocess.run(
-                    [cargo, "vet", "--output-format=json", "--locked"],
-                    cwd=self.topsrcdir,
-                    stdout=subprocess.PIPE,
-                    env=env,
-                )
-                if res.returncode:
-                    vet = json.loads(res.stdout)
-                    for failure in vet.get("failures", []):
-                        failure["crate"] = failure.pop("name")
-                        self.log(
-                            logging.WARNING,
-                            "cargo_vet_failed",
-                            failure,
-                            "Vetting missing for {crate}:{version} {missing_criteria}",
-                        )
-            finally:
-                os.unlink(topsrcdir / ".cargo" / "config")
+            if res.returncode:
+                vet = json.loads(res.stdout)
+                for failure in vet.get("failures", []):
+                    failure["crate"] = failure.pop("name")
+                    self.log(
+                        logging.WARNING,
+                        "cargo_vet_failed",
+                        failure,
+                        "Vetting missing for {crate}:{version} {missing_criteria}",
+                    )
 
         if failed:
             return False

taskcluster/ci/source-test/vendor.yml

@@ -24,6 +24,9 @@ rust:
     description: Rust vendoring validation
     treeherder:
         symbol: rust(vendor)
+    worker:
+        env:
+            MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system
     run:
         cwd: '{checkout}'
         command: >-