From 6d48558d79a48c978ac298d8b6386a759f04cf96 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 27 Mar 2019 19:34:53 +0000 Subject: [PATCH] Bug 1488313: [taskgraph] Add options to reuse on-push tasks in cron graphs; r=dustin,aki Differential Revision: https://phabricator.services.mozilla.com/D13151 --HG-- extra : moz-landing-system : lando --- taskcluster/mach_commands.py | 11 ++++++++++ taskcluster/taskgraph/cron/decision.py | 5 +++++ taskcluster/taskgraph/cron/schema.py | 11 +++++++++- taskcluster/taskgraph/decision.py | 29 +++++++++++++++++++++++--- 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/taskcluster/mach_commands.py b/taskcluster/mach_commands.py index 61e3d7e51d26..b2213a67e486 100644 --- a/taskcluster/mach_commands.py +++ b/taskcluster/mach_commands.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, print_function, unicode_literals +import argparse import json import logging import os @@ -179,6 +180,16 @@ class MachCommands(MachCommandBase): help='path to try task configuration file') @CommandArgument('--tasks-for', help='the tasks_for value used to generate this task') + @CommandArgument('--include-push-tasks', + action='store_true', + help='Whether tasks from the on-push graph should be re-used ' + 'in this graph. This allows cron graphs to avoid rebuilding ' + 'jobs that were built on-push.') + @CommandArgument('--rebuild-kind', + dest='rebuild_kinds', + action='append', + default=argparse.SUPPRESS, + help='Kinds that should not be re-used from the on-push graph.') def taskgraph_decision(self, **options): """Run the decision task: generate a task graph and submit to TaskCluster. This is only meant to be called within decision tasks, diff --git a/taskcluster/taskgraph/cron/decision.py b/taskcluster/taskgraph/cron/decision.py index 33a80859e780..49b1b222a64f 100644 --- a/taskcluster/taskgraph/cron/decision.py +++ b/taskcluster/taskgraph/cron/decision.py @@ -25,6 +25,11 @@ def run_decision_task(job, params, root): arguments.append('--optimize-target-tasks={}'.format( str(job['optimize-target-tasks']).lower(), )) + if 'include-push-tasks' in job: + arguments.append('--include-push-tasks') + if 'rebuild-kinds' in job: + for kind in job['rebuild-kinds']: + arguments.append('--rebuild-kind={}'.format(kind)) return [ make_decision_task( params, diff --git a/taskcluster/taskgraph/cron/schema.py b/taskcluster/taskgraph/cron/schema.py index f8e3263259d2..aed697aae0db 100644 --- a/taskcluster/taskgraph/cron/schema.py +++ b/taskcluster/taskgraph/cron/schema.py @@ -35,7 +35,7 @@ cron_yml_schema = Schema({ Required('treeherder-symbol'): basestring, # --target-tasks-method './mach taskgraph decision' argument - 'target-tasks-method': basestring, + Required('target-tasks-method'): basestring, Optional( 'optimize-target-tasks', @@ -43,6 +43,15 @@ cron_yml_schema = Schema({ 'tasks are eligible for optimization. Otherwise, ' 'the default for the project is used.', ): bool, + Optional( + 'include-push-tasks', + description='Whether tasks from the on-push graph should be re-used ' + 'in the cron graph.', + ): bool, + Optional( + 'rebuild-kinds', + description='Kinds that should not be re-used from the on-push graph.', + ): [basestring], }, # when to run it diff --git a/taskcluster/taskgraph/decision.py b/taskcluster/taskgraph/decision.py index a9e5d3d4c0eb..cc7a0b417399 100644 --- a/taskcluster/taskgraph/decision.py +++ b/taskcluster/taskgraph/decision.py @@ -19,10 +19,12 @@ from .generator import TaskGraphGenerator from .parameters import Parameters, get_version, get_app_version from .taskgraph import TaskGraph from .try_option_syntax import parse_message +from .util.hg import get_hg_revision_branch, get_hg_commit_message +from .util.partials import populate_release_history from .util.schema import validate_schema, Schema -from taskgraph.util.hg import get_hg_revision_branch, get_hg_commit_message -from taskgraph.util.partials import populate_release_history -from taskgraph.util.yaml import load_yaml +from .util.taskcluster import get_artifact +from .util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds +from .util.yaml import load_yaml from voluptuous import Required, Optional @@ -283,6 +285,9 @@ def get_decision_parameters(config, options): if 'DONTBUILD' in commit_message and options['tasks_for'] == 'hg-push': parameters['target_tasks_method'] = 'nothing' + if options.get('include_push_tasks'): + get_existing_tasks(options.get('rebuild_kinds', []), parameters, config) + # If the target method is nightly, we should build partials. This means # knowing what has been released previously. # An empty release_history is fine, it just means no partials will be built @@ -308,6 +313,24 @@ def get_decision_parameters(config, options): return result +def get_existing_tasks(rebuild_kinds, parameters, graph_config): + """ + Find the decision task corresponding to the on-push graph, and return + a mapping of labels to task-ids from it. This will skip the kinds specificed + by `rebuild_kinds`. + """ + try: + decision_task = find_decision_task(parameters, graph_config) + task_graph = get_artifact(decision_task, "public/full-task-graph.json") + except Exception: + logger.exception("Didn't find existing push task.") + return + _, task_graph = TaskGraph.from_json(task_graph) + parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds( + task_graph, [decision_task], rebuild_kinds + ) + + def set_try_config(parameters, task_config_file): if os.path.isfile(task_config_file): logger.info("using try tasks from {}".format(task_config_file))