Bug 1527895 - Add soft-dependencies to taskgraph, r=ahal,marco,tomprince,dustin

Differential Revision: https://phabricator.services.mozilla.com/D19791

--HG--
extra : moz-landing-system : lando

taskcluster/docs/optimization-process.rst

@@ -65,6 +65,9 @@ simultaneously rewrites all dependencies to refer to taskIds instead of labels.
 To do so, it assigns a taskId to each retained task and uses the replacement
 taskId for all replaced tasks.
 
+The `soft-dependencies` are then solved for each task, by adding all the
+remaining tasks in the subgraph from that list to its `dependencies`.
+
 The result is an optimized taskgraph with tasks named by taskId instead of
 label. At this phase, the edges in the task graph diverge from the
 ``task.dependencies`` attributes, as the latter may contain dependencies

taskcluster/docs/taskgraph.rst

@@ -65,6 +65,18 @@ For example, a test task must depend on the build task creating the artifact it
 tests, and this dependency edge is named 'build'.  The task graph generation
 process later resolves these dependencies to specific taskIds.
 
+Dependencies are typically used to ensure that prerequisites to a task, such as
+creation of binary artifacts, are completed before that task runs. But
+dependencies can also be used to schedule follow-up work such as summarizing
+test results. In the latter case, the summarization task will "pull in" all of
+the tasks it depends on, even if those tasks might otherwise be optimized away.
+The fix for this situation is "soft dependencies".
+To add a task depending only on tasks remaining after the optimization process
+completed, you can use `soft-dependencies`, as a list of optimized tasks labels.
+This is useful for tasks that should not pull other tasks into the graph, but do
+need to run after them, if they are in the graph (signing task after an optional
+build or reporting on tasks outputs).
+
 Decision Task
 -------------
 

taskcluster/taskgraph/generator.py

@@ -71,6 +71,7 @@ class Kind(object):
                       task=task_dict['task'],
                       optimization=task_dict.get('optimization'),
                       dependencies=task_dict.get('dependencies'),
+                      soft_dependencies=task_dict.get('soft-dependencies'),
                       release_artifacts=task_dict.get('release-artifacts'),
                       )
                  for task_dict in transforms(trans_config, inputs)]

taskcluster/taskgraph/optimize.py

@@ -214,6 +214,15 @@ def get_subgraph(target_task_graph, removed_tasks, replaced_tasks, label_to_task
         named_task_dependencies = {
             name: label_to_taskid[label]
             for name, label in named_links_dict.get(label, {}).iteritems()}
+
+        # Add remaining soft dependencies
+        if task.soft_dependencies:
+            named_task_dependencies.update({
+                label: label_to_taskid[label]
+                for label in task.soft_dependencies
+                if label in label_to_taskid and label not in omit
+            })
+
         task.task = resolve_task_references(task.label, task.task, named_task_dependencies)
         deps = task.task.setdefault('dependencies', [])
         deps.extend(sorted(named_task_dependencies.itervalues()))

taskcluster/taskgraph/task.py

@@ -19,6 +19,8 @@ class Task(object):
     - optimization: optimization to apply to the task (see taskgraph.optimize)
     - dependencies: tasks this one depends on, in the form {name: label}, for example
       {'build': 'build-linux64/opt', 'docker-image': 'build-docker-image-desktop-test'}
+    - soft_dependencies: tasks this one may depend on if they are available post
+      optimisation. They are set as a list of tasks label.
 
     And later, as the task-graph processing proceeds:
 
@@ -35,6 +37,7 @@ class Task(object):
     task_id = attr.ib(default=None, init=False)
     optimization = attr.ib(default=None)
     dependencies = attr.ib(factory=dict)
+    soft_dependencies = attr.ib(factory=list)
     release_artifacts = attr.ib(
         converter=attr.converters.optional(frozenset),
         default=None,
@@ -49,6 +52,7 @@ class Task(object):
             'label': self.label,
             'attributes': self.attributes,
             'dependencies': self.dependencies,
+            'soft_dependencies': self.soft_dependencies,
             'optimization': self.optimization,
             'task': self.task,
         }
@@ -72,6 +76,7 @@ class Task(object):
             task=task_dict['task'],
             optimization=task_dict['optimization'],
             dependencies=task_dict.get('dependencies'),
+            soft_dependencies=task_dict.get('soft_dependencies'),
             release_artifacts=task_dict.get('release-artifacts'),
         )
         if 'task_id' in task_dict:

taskcluster/taskgraph/test/test_taskgraph.py

@@ -41,6 +41,7 @@ class TestTaskGraph(unittest.TestCase):
                 'attributes': {'attr': 'a-task', 'kind': 'test'},
                 'task': {'taskdef': True},
                 'dependencies': {'edgelabel': 'b'},
+                'soft_dependencies': [],
                 'optimization': None,
             },
             'b': {
@@ -49,6 +50,7 @@ class TestTaskGraph(unittest.TestCase):
                 'attributes': {'kind': 'test'},
                 'task': {'task': 'def'},
                 'dependencies': {},
+                'soft_dependencies': [],
                 'optimization': {'seta': None},
             }
         })

taskcluster/taskgraph/transforms/job/__init__.py

@@ -48,6 +48,7 @@ job_description_schema = Schema({
     Optional('attributes'): task_description_schema['attributes'],
     Optional('job-from'): task_description_schema['job-from'],
     Optional('dependencies'): task_description_schema['dependencies'],
+    Optional('soft-dependencies'): task_description_schema['soft-dependencies'],
     Optional('expires-after'): task_description_schema['expires-after'],
     Optional('routes'): task_description_schema['routes'],
     Optional('scopes'): task_description_schema['scopes'],
@@ -247,6 +248,7 @@ def make_task_description(config, jobs):
         # fill in some empty defaults to make run implementations easier
         taskdesc.setdefault('attributes', {})
         taskdesc.setdefault('dependencies', {})
+        taskdesc.setdefault('soft-dependencies', [])
         taskdesc.setdefault('routes', [])
         taskdesc.setdefault('scopes', [])
         taskdesc.setdefault('extra', {})

taskcluster/taskgraph/transforms/task.py

@@ -68,6 +68,9 @@ task_description_schema = Schema({
     # method.
     Optional('dependencies'): {basestring: object},
 
+    # Soft dependencies of this task, as a list of tasks labels
+    Optional('soft-dependencies'): [basestring],
+
     Optional('requires'): Any('all-completed', 'all-resolved'),
 
     # expiration and deadline times, relative to task creation, with units
@@ -1771,6 +1774,7 @@ def build_task(config, tasks):
             'label': task['label'],
             'task': task_def,
             'dependencies': task.get('dependencies', {}),
+            'soft-dependencies': task.get('soft-dependencies', []),
             'attributes': attributes,
             'optimization': task.get('optimization', None),
             'release-artifacts': task.get('release-artifacts', []),