mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-12-14 10:43:24 +00:00
5c93c1892b
Rather than defining a distinct `Parameter` class for Gecko, this now uses the `extend_parameters_schema` utility function (which mobile repos are already using). As a consequence, shared parameters are now defined in standalone taskgraph. And only Gecko-specific parameters are listed in `gecko_taskgraph/parameters.py` The only exception is `project` which gets redefined so we can override the standalone taskgraph default (since it derives `project` from the repo name, which doesn't work for Gecko). Differential Revision: https://phabricator.services.mozilla.com/D134515
452 lines
13 KiB
Python
452 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from functools import partial
|
|
|
|
from mach.decorators import (
|
|
Command,
|
|
CommandArgument,
|
|
SettingsProvider,
|
|
SubCommand,
|
|
)
|
|
|
|
import gecko_taskgraph.main
|
|
from gecko_taskgraph.main import commands as taskgraph_commands
|
|
|
|
logger = logging.getLogger("taskcluster")
|
|
|
|
|
|
@SettingsProvider
|
|
class TaskgraphConfig(object):
|
|
@classmethod
|
|
def config_settings(cls):
|
|
return [
|
|
(
|
|
"taskgraph.diffcmd",
|
|
"string",
|
|
"The command to run with `./mach taskgraph --diff`",
|
|
"diff --report-identical-files "
|
|
"--label={attr}@{base} --label={attr}@{cur} -U20",
|
|
{},
|
|
)
|
|
]
|
|
|
|
|
|
def strtobool(value):
|
|
"""Convert string to boolean.
|
|
|
|
Wraps "distutils.util.strtobool", deferring the import of the package
|
|
in case it's not installed. Otherwise, we have a "chicken and egg problem" where
|
|
|mach bootstrap| would install the required package to enable "distutils.util", but
|
|
it can't because mach fails to interpret this file.
|
|
"""
|
|
from distutils.util import strtobool
|
|
|
|
return bool(strtobool(value))
|
|
|
|
|
|
def get_taskgraph_command_parser(name):
|
|
"""Given a command name, obtain its argument parser.
|
|
|
|
Args:
|
|
name (str): Name of the command.
|
|
|
|
Returns:
|
|
ArgumentParser: An ArgumentParser instance.
|
|
"""
|
|
command = taskgraph_commands[name]
|
|
parser = argparse.ArgumentParser()
|
|
for arg in command.func.args:
|
|
parser.add_argument(*arg[0], **arg[1])
|
|
|
|
parser.set_defaults(func=command.func, **command.defaults)
|
|
return parser
|
|
|
|
|
|
def get_taskgraph_decision_parser():
|
|
parser = get_taskgraph_command_parser("decision")
|
|
|
|
extra_args = [
|
|
(
|
|
["--optimize-target-tasks"],
|
|
{
|
|
"type": lambda flag: strtobool(flag),
|
|
"nargs": "?",
|
|
"const": "true",
|
|
"help": "If specified, this indicates whether the target "
|
|
"tasks are eligible for optimization. Otherwise, the default "
|
|
"for the project is used.",
|
|
},
|
|
),
|
|
(
|
|
["--include-push-tasks"],
|
|
{
|
|
"action": "store_true",
|
|
"help": "Whether tasks from the on-push graph should be re-used "
|
|
"in this graph. This allows cron graphs to avoid rebuilding "
|
|
"jobs that were built on-push.",
|
|
},
|
|
),
|
|
(
|
|
["--rebuild-kind"],
|
|
{
|
|
"dest": "rebuild_kinds",
|
|
"action": "append",
|
|
"default": argparse.SUPPRESS,
|
|
"help": "Kinds that should not be re-used from the on-push graph.",
|
|
},
|
|
),
|
|
(
|
|
["--comm-base-repository"],
|
|
{
|
|
"required": False,
|
|
"help": "URL for 'base' comm-* repository to clone",
|
|
},
|
|
),
|
|
(
|
|
["--comm-head-repository"],
|
|
{
|
|
"required": False,
|
|
"help": "URL for 'head' comm-* repository to fetch revision from",
|
|
},
|
|
),
|
|
(
|
|
["--comm-head-ref"],
|
|
{
|
|
"required": False,
|
|
"help": "comm-* Reference (this is same as rev usually for hg)",
|
|
},
|
|
),
|
|
(
|
|
["--comm-head-rev"],
|
|
{
|
|
"required": False,
|
|
"help": "Commit revision to use from head comm-* repository",
|
|
},
|
|
),
|
|
]
|
|
for arg in extra_args:
|
|
parser.add_argument(*arg[0], **arg[1])
|
|
|
|
return parser
|
|
|
|
|
|
@Command(
|
|
"taskgraph",
|
|
category="ci",
|
|
description="Manipulate TaskCluster task graphs defined in-tree",
|
|
)
|
|
def taskgraph_command(command_context):
|
|
"""The taskgraph subcommands all relate to the generation of task graphs
|
|
for Gecko continuous integration. A task graph is a set of tasks linked
|
|
by dependencies: for example, a binary must be built before it is tested,
|
|
and that build may further depend on various toolchains, libraries, etc.
|
|
"""
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"tasks",
|
|
description="Show all tasks in the taskgraph",
|
|
parser=partial(get_taskgraph_command_parser, "tasks"),
|
|
)
|
|
def taskgraph_tasks(command_context, **options):
|
|
return run_show_taskgraph(command_context, **options)
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"full",
|
|
description="Show the full taskgraph",
|
|
parser=partial(get_taskgraph_command_parser, "full"),
|
|
)
|
|
def taskgraph_full(command_context, **options):
|
|
return run_show_taskgraph(command_context, **options)
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"target",
|
|
description="Show the target task set",
|
|
parser=partial(get_taskgraph_command_parser, "target"),
|
|
)
|
|
def taskgraph_target(command_context, **options):
|
|
return run_show_taskgraph(command_context, **options)
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"target-graph",
|
|
description="Show the target taskgraph",
|
|
parser=partial(get_taskgraph_command_parser, "target-graph"),
|
|
)
|
|
def taskgraph_target_graph(command_context, **options):
|
|
return run_show_taskgraph(command_context, **options)
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"optimized",
|
|
description="Show the optimized taskgraph",
|
|
parser=partial(get_taskgraph_command_parser, "optimized"),
|
|
)
|
|
def taskgraph_optimized(command_context, **options):
|
|
return run_show_taskgraph(command_context, **options)
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"morphed",
|
|
description="Show the morphed taskgraph",
|
|
parser=partial(get_taskgraph_command_parser, "morphed"),
|
|
)
|
|
def taskgraph_morphed(command_context, **options):
|
|
return run_show_taskgraph(command_context, **options)
|
|
|
|
|
|
def run_show_taskgraph(command_context, **options):
|
|
# There are cases where we don't want to set up mach logging (e.g logs
|
|
# are being redirected to disk). By monkeypatching the 'setup_logging'
|
|
# function we can let 'taskgraph.main' decide whether or not to log to
|
|
# the terminal.
|
|
gecko_taskgraph.main.setup_logging = partial(
|
|
setup_logging,
|
|
command_context,
|
|
quiet=options["quiet"],
|
|
verbose=options["verbose"],
|
|
)
|
|
show_taskgraph = options.pop("func")
|
|
return show_taskgraph(options)
|
|
|
|
|
|
@SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
|
|
@CommandArgument(
|
|
"--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
|
|
)
|
|
@CommandArgument(
|
|
"--quiet", "-q", action="store_true", help="suppress all logging output"
|
|
)
|
|
@CommandArgument(
|
|
"--verbose",
|
|
"-v",
|
|
action="store_true",
|
|
help="include debug-level logging output",
|
|
)
|
|
@CommandArgument(
|
|
"--parameters",
|
|
"-p",
|
|
default="project=mozilla-central",
|
|
help="parameters file (.yml or .json; see `taskcluster/docs/parameters.rst`)`",
|
|
)
|
|
def taskgraph_actions(command_context, **options):
|
|
return show_actions(command_context, options)
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"decision",
|
|
description="Run the decision task",
|
|
parser=get_taskgraph_decision_parser,
|
|
)
|
|
def taskgraph_decision(command_context, **options):
|
|
"""Run the decision task: generate a task graph and submit to
|
|
TaskCluster. This is only meant to be called within decision tasks,
|
|
and requires a great many arguments. Commands like `mach taskgraph
|
|
optimized` are better suited to use on the command line, and can take
|
|
the parameters file generated by a decision task."""
|
|
try:
|
|
setup_logging(command_context)
|
|
start = time.monotonic()
|
|
ret = taskgraph_commands["decision"].func(options)
|
|
end = time.monotonic()
|
|
if os.environ.get("MOZ_AUTOMATION") == "1":
|
|
perfherder_data = {
|
|
"framework": {"name": "build_metrics"},
|
|
"suites": [
|
|
{
|
|
"name": "decision",
|
|
"value": end - start,
|
|
"lowerIsBetter": True,
|
|
"shouldAlert": True,
|
|
"subtests": [],
|
|
}
|
|
],
|
|
}
|
|
print(
|
|
"PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
|
|
file=sys.stderr,
|
|
)
|
|
return ret
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"cron",
|
|
description="Provide a pointer to the new `.cron.yml` handler.",
|
|
)
|
|
def taskgraph_cron(command_context, **options):
|
|
print(
|
|
'Handling of ".cron.yml" files has move to '
|
|
"https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
|
|
)
|
|
sys.exit(1)
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"action-callback",
|
|
description="Run action callback used by action tasks",
|
|
parser=partial(get_taskgraph_command_parser, "action-callback"),
|
|
)
|
|
def action_callback(command_context, **options):
|
|
setup_logging(command_context)
|
|
taskgraph_commands["action-callback"].func(options)
|
|
|
|
|
|
@SubCommand(
|
|
"taskgraph",
|
|
"test-action-callback",
|
|
description="Run an action callback in a testing mode",
|
|
parser=partial(get_taskgraph_command_parser, "test-action-callback"),
|
|
)
|
|
def test_action_callback(command_context, **options):
|
|
setup_logging(command_context)
|
|
|
|
if not options["parameters"]:
|
|
options["parameters"] = "project=mozilla-central"
|
|
|
|
taskgraph_commands["test-action-callback"].func(options)
|
|
|
|
|
|
def setup_logging(command_context, quiet=False, verbose=True):
|
|
"""
|
|
Set up Python logging for all loggers, sending results to stderr (so
|
|
that command output can be redirected easily) and adding the typical
|
|
mach timestamp.
|
|
"""
|
|
# remove the old terminal handler
|
|
old = command_context.log_manager.replace_terminal_handler(None)
|
|
|
|
# re-add it, with level and fh set appropriately
|
|
if not quiet:
|
|
level = logging.DEBUG if verbose else logging.INFO
|
|
command_context.log_manager.add_terminal_logging(
|
|
fh=sys.stderr,
|
|
level=level,
|
|
write_interval=old.formatter.write_interval,
|
|
write_times=old.formatter.write_times,
|
|
)
|
|
|
|
# all of the taskgraph logging is unstructured logging
|
|
command_context.log_manager.enable_unstructured()
|
|
|
|
|
|
def show_actions(command_context, options):
|
|
import gecko_taskgraph
|
|
import gecko_taskgraph.actions
|
|
import gecko_taskgraph.generator
|
|
from taskgraph.parameters import parameters_loader
|
|
|
|
try:
|
|
setup_logging(
|
|
command_context, quiet=options["quiet"], verbose=options["verbose"]
|
|
)
|
|
parameters = parameters_loader(options["parameters"])
|
|
|
|
tgg = gecko_taskgraph.generator.TaskGraphGenerator(
|
|
root_dir=options.get("root"),
|
|
parameters=parameters,
|
|
)
|
|
|
|
actions = gecko_taskgraph.actions.render_actions_json(
|
|
tgg.parameters,
|
|
tgg.graph_config,
|
|
decision_task_id="DECISION-TASK",
|
|
)
|
|
print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
@Command(
|
|
"taskcluster-load-image",
|
|
category="ci",
|
|
description="Load a pre-built Docker image. Note that you need to "
|
|
"have docker installed and running for this to work.",
|
|
parser=partial(get_taskgraph_command_parser, "load-image"),
|
|
)
|
|
def load_image(command_context, **kwargs):
|
|
taskgraph_commands["load-image"].func(kwargs)
|
|
|
|
|
|
@Command(
|
|
"taskcluster-build-image",
|
|
category="ci",
|
|
description="Build a Docker image",
|
|
parser=partial(get_taskgraph_command_parser, "build-image"),
|
|
)
|
|
def build_image(command_context, **kwargs):
|
|
try:
|
|
taskgraph_commands["build-image"].func(kwargs)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
@Command(
|
|
"taskcluster-image-digest",
|
|
category="ci",
|
|
description="Print the digest of the image of this name based on the "
|
|
"current contents of the tree.",
|
|
parser=partial(get_taskgraph_command_parser, "build-image"),
|
|
)
|
|
def image_digest(command_context, **kwargs):
|
|
taskgraph_commands["image-digest"].func(kwargs)
|
|
|
|
|
|
@Command(
|
|
"release-history",
|
|
category="ci",
|
|
description="Query balrog for release history used by enable partials generation",
|
|
)
|
|
@CommandArgument(
|
|
"-b",
|
|
"--branch",
|
|
help="The gecko project branch used in balrog, such as "
|
|
"mozilla-central, release, maple",
|
|
)
|
|
@CommandArgument(
|
|
"--product", default="Firefox", help="The product identifier, such as 'Firefox'"
|
|
)
|
|
def generate_partials_builds(command_context, product, branch):
|
|
from gecko_taskgraph.util.partials import populate_release_history
|
|
|
|
try:
|
|
import yaml
|
|
|
|
release_history = {"release_history": populate_release_history(product, branch)}
|
|
print(
|
|
yaml.safe_dump(
|
|
release_history, allow_unicode=True, default_flow_style=False
|
|
)
|
|
)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|