Bug 1333255: introduce graph morphs, use them to make index tasks; r=jonasfj

Graph morphs modify the graph after optimization, without changing its meaning.
In this case, that means adding index tasks that will insert paths into the
index beyond the relatively limited number afforded in task.routes.

MozReview-Commit-ID: AJy4exX7q2v

--HG--
extra : rebase_source : d61e7462defd41e7112739fb057edb493f495430
extra : source : c580568ed47c1ed2af40d98b47fbb0d136e63060

taskcluster/docs/taskgraph.rst

@@ -99,7 +99,11 @@ Graph generation, as run via ``mach taskgraph decision``, proceeds as follows:
 #. Optimize the target task graph using task-specific optimization methods.
    The result is the "optimized task graph" with fewer nodes than the target
    task graph.  See :ref:`optimization`.
-#. Create tasks for all tasks in the optimized task graph.
+#. Morph the graph. Morphs are like syntactic sugar: they keep the same meaning,
+   but express it in a lower-level way. These generally work around limitations
+   in the TaskCluster platform, such as number of dependencies or routes in
+   a task.
+#. Create tasks for all tasks in the morphed task graph.
 
 Transitive Closure
 ..................

taskcluster/mach_commands.py

@@ -106,6 +106,11 @@ class MachCommands(MachCommandBase):
     def taskgraph_optimized(self, **options):
         return self.show_taskgraph('optimized_task_graph', options)
 
+    @ShowTaskGraphSubCommand('taskgraph', 'morphed',
+                             description="Show the morphed taskgraph")
+    def taskgraph_morphed(self, **options):
+        return self.show_taskgraph('morphed_task_graph', options)
+
     @SubCommand('taskgraph', 'decision',
                 description="Run the decision task")
     @CommandArgument('--root', '-r',

taskcluster/taskgraph/decision.py

@@ -126,11 +126,11 @@ def taskgraph_decision(options):
 
     # write out the optimized task graph to describe what will actually happen,
     # and the map of labels to taskids
-    write_artifact('task-graph.json', tgg.optimized_task_graph.to_json())
+    write_artifact('task-graph.json', tgg.morphed_task_graph.to_json())
     write_artifact('label-to-taskid.json', tgg.label_to_taskid)
 
     # actually create the graph
-    create_tasks(tgg.optimized_task_graph, tgg.label_to_taskid, parameters)
+    create_tasks(tgg.morphed_task_graph, tgg.label_to_taskid, parameters)
 
 
 def get_decision_parameters(options):

taskcluster/taskgraph/generator.py

@@ -13,6 +13,7 @@ from .graph import Graph
 from .taskgraph import TaskGraph
 from .task import Task
 from .optimize import optimize_task_graph
+from .morph import morph
 from .util.python_path import find_object
 from .transforms.base import TransformSequence, TransformConfig
 from .util.verify import (
@@ -167,6 +168,17 @@ class TaskGraphGenerator(object):
         """
         return self._run_until('label_to_taskid')
 
+    @property
+    def morphed_task_graph(self):
+        """
+        The optimized task graph, with any subsequent morphs applied. This graph
+        will have the same meaning as the optimized task graph, but be in a form
+        more palatable to TaskCluster.
+
+        @type: TaskGraph
+        """
+        return self._run_until('morphed_task_graph')
+
     def _load_kinds(self):
         for path in os.listdir(self.root_dir):
             path = os.path.join(self.root_dir, path)
@@ -260,9 +272,14 @@ class TaskGraphGenerator(object):
         optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
                                                                     self.parameters,
                                                                     do_not_optimize)
-        yield 'label_to_taskid', label_to_taskid
+
         yield 'optimized_task_graph', optimized_task_graph
 
+        morphed_task_graph, label_to_taskid = morph(optimized_task_graph, label_to_taskid)
+
+        yield 'label_to_taskid', label_to_taskid
+        yield 'morphed_task_graph', morphed_task_graph
+
     def _run_until(self, name):
         while name not in self._run_results:
             try:

taskcluster/taskgraph/morph.py

@@ -0,0 +1,136 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Graph morphs are modifications to task-graphs that take place *after* the
+optimization phase.
+
+These graph morphs are largely invisible to developers running `./mach`
+locally, so they should be limited to changes that do not modify the meaning of
+the graph.
+"""
+
+# Note that the translation of `{'task-reference': '..'}` is handled in the
+# optimization phase (since optimization involves dealing with taskIds
+# directly).  Similarly, `{'relative-datestamp': '..'}` is handled at the last
+# possible moment during task creation.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+from slugid import nice as slugid
+from .task import Task
+from .graph import Graph
+from .taskgraph import TaskGraph
+
+logger = logging.getLogger(__name__)
+MAX_ROUTES = 10
+
+
+def amend_taskgraph(taskgraph, label_to_taskid, to_add):
+    """Add the given tasks to the taskgraph, returning a new taskgraph"""
+    new_tasks = taskgraph.tasks.copy()
+    new_edges = taskgraph.graph.edges.copy()
+    for task in to_add:
+        new_tasks[task.task_id] = task
+        assert task.label not in label_to_taskid
+        label_to_taskid[task.label] = task.task_id
+        for depname, dep in task.dependencies.iteritems():
+            new_edges.add((task.task_id, dep, depname))
+
+    taskgraph = TaskGraph(new_tasks, Graph(set(new_tasks), new_edges))
+    return taskgraph, label_to_taskid
+
+
+def derive_misc_task(task, purpose, image, label_to_taskid):
+    """Create the shell of a task that depends on `task` and on the given docker
+    image."""
+    label = '{}-{}'.format(purpose, task.label)
+
+    # this is why all docker image tasks are included in the target task graph: we
+    # need to find them in label_to_taskid, if if nothing else required them
+    image_taskid = label_to_taskid['build-docker-image-' + image]
+
+    task_def = {
+        'provisionerId': 'aws-provisioner-v1',
+        'workerType': 'gecko-misc',
+        'dependencies': [task.task_id, image_taskid],
+        'created': {'relative-timestamp': '0 seconds'},
+        'deadline': task.task['deadline'],
+        # no point existing past the parent task's deadline
+        'expires': task.task['deadline'],
+        'metadata': {
+            'name': label,
+            'description': '{} for {}'.format(purpose, task.task['metadata']['description']),
+            'owner': task.task['metadata']['owner'],
+            'source': task.task['metadata']['source'],
+        },
+        'scopes': [],
+        'payload': {
+            'image': {
+                'path': 'public/image.tar.zst',
+                'taskId': image_taskid,
+                'type': 'task-image',
+            },
+            'features': {
+                'taskclusterProxy': True,
+            },
+            'maxRunTime': 600,
+        }
+    }
+    dependencies = {
+        'parent': task.task_id,
+        'docker-image': image_taskid,
+    }
+    task = Task(kind='misc', label=label, attributes={}, task=task_def,
+                dependencies=dependencies)
+    task.task_id = slugid()
+    return task
+
+
+def make_index_task(parent_task, label_to_taskid):
+    index_paths = [r.split('.', 1)[1] for r in parent_task.task['routes']
+                   if r.startswith('index.')]
+    parent_task.task['routes'] = [r for r in parent_task.task['routes']
+                                  if not r.startswith('index.')]
+
+    task = derive_misc_task(parent_task, 'index-task',
+                            'index-task', label_to_taskid)
+    task.task['scopes'] = [
+        'index:insert-task:{}'.format(path) for path in index_paths]
+    task.task['payload']['command'] = ['insert-indexes.js'] + index_paths
+    task.task['payload']['env'] = {
+        "TARGET_TASKID": parent_task.task_id,
+    }
+    return task
+
+
+def add_index_tasks(taskgraph, label_to_taskid):
+    """
+    The TaskCluster queue only allows 10 routes on a task, but we have tasks
+    with many more routes, for purposes of indexing. This graph morph adds
+    "index tasks" that depend on such tasks and do the index insertions
+    directly, avoiding the limits on task.routes.
+    """
+    logger.debug('Morphing: adding index tasks')
+
+    added = []
+    for label, task in taskgraph.tasks.iteritems():
+        if len(task.task.get('routes', [])) <= MAX_ROUTES:
+            continue
+        added.append(make_index_task(task, label_to_taskid))
+
+    if added:
+        taskgraph, label_to_taskid = amend_taskgraph(
+            taskgraph, label_to_taskid, added)
+        logger.info('Added {} index tasks'.format(len(added)))
+
+    return taskgraph, label_to_taskid
+
+
+def morph(taskgraph, label_to_taskid):
+    """Apply all morphs"""
+    taskgraph, label_to_taskid = add_index_tasks(taskgraph, label_to_taskid)
+    return taskgraph, label_to_taskid