Bug 1488313: [taskgraph] Add options to reuse on-push tasks in cron graphs; r=dustin,aki

Differential Revision: https://phabricator.services.mozilla.com/D13151

--HG--
extra : moz-landing-system : lando

taskcluster/mach_commands.py

@@ -7,6 +7,7 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import argparse
 import json
 import logging
 import os
@@ -179,6 +180,16 @@ class MachCommands(MachCommandBase):
                      help='path to try task configuration file')
     @CommandArgument('--tasks-for',
                      help='the tasks_for value used to generate this task')
+    @CommandArgument('--include-push-tasks',
+                     action='store_true',
+                     help='Whether tasks from the on-push graph should be re-used '
+                          'in this graph. This allows cron graphs to avoid rebuilding '
+                          'jobs that were built on-push.')
+    @CommandArgument('--rebuild-kind',
+                     dest='rebuild_kinds',
+                     action='append',
+                     default=argparse.SUPPRESS,
+                     help='Kinds that should not be re-used from the on-push graph.')
     def taskgraph_decision(self, **options):
         """Run the decision task: generate a task graph and submit to
         TaskCluster.  This is only meant to be called within decision tasks,

taskcluster/taskgraph/cron/decision.py

@@ -25,6 +25,11 @@ def run_decision_task(job, params, root):
         arguments.append('--optimize-target-tasks={}'.format(
             str(job['optimize-target-tasks']).lower(),
         ))
+    if 'include-push-tasks' in job:
+        arguments.append('--include-push-tasks')
+    if 'rebuild-kinds' in job:
+        for kind in job['rebuild-kinds']:
+            arguments.append('--rebuild-kind={}'.format(kind))
     return [
         make_decision_task(
             params,

taskcluster/taskgraph/cron/schema.py

@@ -35,7 +35,7 @@ cron_yml_schema = Schema({
             Required('treeherder-symbol'): basestring,
 
             # --target-tasks-method './mach taskgraph decision' argument
-            'target-tasks-method': basestring,
+            Required('target-tasks-method'): basestring,
 
             Optional(
                 'optimize-target-tasks',
@@ -43,6 +43,15 @@ cron_yml_schema = Schema({
                             'tasks are eligible for optimization. Otherwise, '
                             'the default for the project is used.',
             ): bool,
+            Optional(
+                'include-push-tasks',
+                description='Whether tasks from the on-push graph should be re-used '
+                            'in the cron graph.',
+            ): bool,
+            Optional(
+                'rebuild-kinds',
+                description='Kinds that should not be re-used from the on-push graph.',
+            ): [basestring],
         },
 
         # when to run it

taskcluster/taskgraph/decision.py

@@ -19,10 +19,12 @@ from .generator import TaskGraphGenerator
 from .parameters import Parameters, get_version, get_app_version
 from .taskgraph import TaskGraph
 from .try_option_syntax import parse_message
+from .util.hg import get_hg_revision_branch, get_hg_commit_message
+from .util.partials import populate_release_history
 from .util.schema import validate_schema, Schema
-from taskgraph.util.hg import get_hg_revision_branch, get_hg_commit_message
+from .util.taskcluster import get_artifact
-from taskgraph.util.partials import populate_release_history
+from .util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds
-from taskgraph.util.yaml import load_yaml
+from .util.yaml import load_yaml
 from voluptuous import Required, Optional
 
 
@@ -283,6 +285,9 @@ def get_decision_parameters(config, options):
     if 'DONTBUILD' in commit_message and options['tasks_for'] == 'hg-push':
         parameters['target_tasks_method'] = 'nothing'
 
+    if options.get('include_push_tasks'):
+        get_existing_tasks(options.get('rebuild_kinds', []), parameters, config)
+
     # If the target method is nightly, we should build partials. This means
     # knowing what has been released previously.
     # An empty release_history is fine, it just means no partials will be built
@@ -308,6 +313,24 @@ def get_decision_parameters(config, options):
     return result
 
 
+def get_existing_tasks(rebuild_kinds, parameters, graph_config):
+    """
+    Find the decision task corresponding to the on-push graph, and return
+    a mapping of labels to task-ids from it. This will skip the kinds specificed
+    by `rebuild_kinds`.
+    """
+    try:
+        decision_task = find_decision_task(parameters, graph_config)
+        task_graph = get_artifact(decision_task, "public/full-task-graph.json")
+    except Exception:
+        logger.exception("Didn't find existing push task.")
+        return
+    _, task_graph = TaskGraph.from_json(task_graph)
+    parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
+        task_graph, [decision_task], rebuild_kinds
+    )
+
+
 def set_try_config(parameters, task_config_file):
     if os.path.isfile(task_config_file):
         logger.info("using try tasks from {}".format(task_config_file))