gecko-dev/build/moz.configure/bootstrap.configure

# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

option(
    env="MOZ_FETCHES_DIR",
    nargs=1,
    when="MOZ_AUTOMATION",
    help="Directory containing fetched artifacts",
)


@depends("MOZ_FETCHES_DIR", when="MOZ_AUTOMATION")
def moz_fetches_dir(value):
    if value:
        return value[0]


@depends(vcs_checkout_type, milestone.is_nightly, "MOZ_AUTOMATION")
def bootstrap_default(vcs_checkout_type, is_nightly, automation):
    if automation:
        return False
    # We only enable if building off a VCS checkout of central.
    if is_nightly and vcs_checkout_type:
        return True


option(
    "--enable-bootstrap",
    nargs="*",
    default=bootstrap_default,
    help="{Automatically bootstrap or update some toolchains|Disable bootstrap or update of toolchains}",
)


@depends_if("--enable-bootstrap")
def want_bootstrap(bootstrap):
    include = set()
    exclude = set()
    for item in bootstrap:
        if item == "no-update":
            continue
        if item.startswith("-"):
            exclude.add(item.lstrip("-"))
        else:
            include.add(item)

    def match(name):
        if name in exclude:
            return False
        if include and name in include:
            return True
        return not bool(include)

    return match


toolchains_base_dir = moz_fetches_dir | mozbuild_state_path


@dependable
@imports("os")
@imports(_from="os", _import="environ")
def original_path():
    return environ["PATH"].split(os.pathsep)


@depends(host, when="--enable-bootstrap")
@imports("os")
@imports("traceback")
@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions")
@imports(_from="__builtin__", _import="Exception")
def bootstrap_toolchain_tasks(host):
    prefix = {
        ("x86_64", "GNU", "Linux"): "linux64",
        ("x86_64", "OSX", "Darwin"): "macosx64",
        ("aarch64", "OSX", "Darwin"): "macosx64-aarch64",
        ("x86_64", "WINNT", "WINNT"): "win64",
        ("aarch64", "WINNT", "WINNT"): "win64-aarch64",
    }.get((host.cpu, host.os, host.kernel))
    try:
        tasks = toolchain_task_definitions()
    except Exception as e:
        message = traceback.format_exc()
        log.warning(str(e))
        log.debug(message)
        return None

    def task_data(t):
        result = {
            "index": t.optimization["index-search"],
            "artifact": t.attributes["toolchain-artifact"],
        }
        command = t.attributes.get("toolchain-command")
        if command:
            result["command"] = command
        return result

    # We only want to use toolchains annotated with "local-toolchain". We also limit the
    # amount of data to what we use, so that trace logs can be more useful.
    tasks = {
        k: task_data(t)
        for k, t in tasks.items()
        if t.attributes.get("local-toolchain") and "index-search" in t.optimization
    }

    return namespace(prefix=prefix, tasks=tasks)


@template
def bootstrap_path(path, **kwargs):
    when = kwargs.pop("when", None)
    allow_failure = kwargs.pop("allow_failure", None)
    if kwargs:
        configure_error(
            "bootstrap_path only takes `when` and `allow_failure` as a keyword argument"
        )

    @depends(
        "--enable-bootstrap",
        want_bootstrap,
        toolchains_base_dir,
        moz_fetches_dir,
        bootstrap_toolchain_tasks,
        build_environment,
        dependable(path),
        dependable(allow_failure),
        when=when,
    )
    @imports("os")
    @imports("subprocess")
    @imports("sys")
    @imports(_from="mozbuild.dirutils", _import="ensureParentDir")
    @imports(_from="importlib", _import="import_module")
    @imports(_from="shutil", _import="rmtree")
    @imports(_from="__builtin__", _import="open")
    @imports(_from="__builtin__", _import="Exception")
    def bootstrap_path(
        enable_bootstrap,
        want_bootstrap,
        toolchains_base_dir,
        moz_fetches_dir,
        tasks,
        build_env,
        path,
        allow_failure,
    ):
        if not path:
            return
        path_parts = path.split("/")
        path_prefix = ""
        # Small hack until clang-tidy stops being a separate toolchain in a
        # weird location.
        if path_parts[0] == "clang-tools":
            path_prefix = path_parts.pop(0)

        def try_bootstrap(exists):
            if not tasks:
                return False
            prefixes = [""]
            if tasks.prefix:
                prefixes.insert(0, "{}-".format(tasks.prefix))
            for prefix in prefixes:
                label = "toolchain-{}{}".format(prefix, path_parts[0])
                task = tasks.tasks.get(label)
                if task:
                    break
            log.debug("Trying to bootstrap %s", label)
            if not task:
                return False
            task_index = task["index"]
            log.debug("Resolved %s to %s", label, task_index[0])
            task_index = task_index[0].split(".")[-1]
            artifact = task["artifact"]
            # `mach artifact toolchain` doesn't support authentication for
            # private artifacts. Some toolchains may provide a command that can be
            # used for local production of the artifact.
            command = None
            if not artifact.startswith("public/"):
                command = task.get("command")
                if not command:
                    log.debug("Cannot bootstrap %s: not a public artifact", label)
                    return False
            index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0])
            try:
                with open(index_file) as fh:
                    index = fh.read().strip()
            except Exception:
                # On automation, if there's an artifact in MOZ_FETCHES_DIR, we assume it's
                # up-to-date.
                index = task_index if moz_fetches_dir else None
            if index == task_index and exists:
                log.debug("%s is up-to-date", label)
                return True
            # Manually import with import_module so that we can gracefully disable bootstrap
            # when e.g. building from a js standalone tarball, that doesn't contain the
            # taskgraph code. In those cases, `mach artifact toolchain --from-build` would
            # also fail.
            task_id = None
            if not command:
                try:
                    IndexSearch = import_module(
                        "taskgraph.optimize.strategies"
                    ).IndexSearch
                except Exception:
                    log.debug("Cannot bootstrap %s: missing taskgraph module", label)
                    return False
                task_id = IndexSearch().should_replace_task(
                    task, {}, None, task["index"]
                )
            if task_id:
                # If we found the task in the index, use the `mach artifact toolchain`
                # fast path.
                command = [
                    "artifact",
                    "toolchain",
                    "--from-task",
                    f"{task_id}:{artifact}",
                ]
            elif command:
                # For private local toolchains, run the associated command.
                command = (
                    [
                        "python",
                        "--virtualenv",
                        "build",
                        os.path.join(
                            build_env.topsrcdir,
                            "taskcluster/scripts/misc",
                            command["script"],
                        ),
                    ]
                    + command["arguments"]
                    + [path_parts[0]]
                )
                # Clean up anything that was bootstrapped previously before going
                # forward. In other cases, that's taken care of by mach artifact toolchain.
                rmtree(
                    os.path.join(toolchains_base_dir, path_prefix, path_parts[0]),
                    ignore_errors=True,
                )
            else:
                # Otherwise, use the slower path, which will print a better error than
                # we would be able to.
                command = ["artifact", "toolchain", "--from-build", label]

            log.info(
                "%s bootstrapped toolchain in %s",
                "Updating" if exists else "Installing",
                os.path.join(toolchains_base_dir, path_prefix, path_parts[0]),
            )
            os.makedirs(os.path.join(toolchains_base_dir, path_prefix), exist_ok=True)
            proc = subprocess.run(
                [
                    sys.executable,
                    os.path.join(build_env.topsrcdir, "mach"),
                    "--log-no-times",
                ]
                + command,
                cwd=os.path.join(toolchains_base_dir, path_prefix),
                check=not allow_failure,
            )
            if proc.returncode != 0 and allow_failure:
                return False
            ensureParentDir(index_file)
            with open(index_file, "w") as fh:
                fh.write(task_index)
            return True

        path = os.path.join(toolchains_base_dir, path_prefix, *path_parts)
        if enable_bootstrap and want_bootstrap(path_parts[0]):
            exists = os.path.exists(path)
            try:
                # With --enable-bootstrap=no-update, we don't `try_bootstrap`, except
                # when the toolchain can't be found.
                if (
                    "no-update" not in enable_bootstrap or not exists
                ) and not try_bootstrap(exists):
                    # If there aren't toolchain artifacts to use for this build,
                    # don't return a path.
                    return None
            except Exception as e:
                log.error("%s", e)
                die("If you can't fix the above, retry with --disable-bootstrap.")
        if enable_bootstrap or enable_bootstrap.origin == "default":
            # We re-test whether the path exists because it may have been created by
            # try_bootstrap. Automation will not have gone through the bootstrap
            # process, but we want to return the path if it exists.
            if os.path.exists(path):
                return path

    return bootstrap_path


@template
def bootstrap_search_path(path, paths=original_path, **kwargs):
    @depends(
        bootstrap_path(path, **kwargs),
        paths,
        original_path,
    )
    def bootstrap_search_path(path, paths, original_path):
        if paths is None:
            paths = original_path
        if not path:
            return paths
        return [path] + paths

    return bootstrap_search_path