Bug 1468812 - [taskgraph] Support artifacts from any dependency via fetches r=gps

Fetches no longer need to be artifacts exposed via a 'fetch' task, they can
also be artifacts from a task's dependencies. The new format is:

fetches:
    fetch:
        - fetch-artifact-1.zip
        - fetch-artifact-2.zip
    build:
        - build-artifact-1.zip
        ...

Specifying 'build' artifacts to fetch will error out if the task doesn't have
any build dependencies.

The 'fetch' key works the same as before, but it is now a special case. Unlike
'build' (or other dependencies), adding a fetch task's artifact here will
implicitly make our task depend on the corresponding fetch task. It will not
be an error.

Depends on D2028.

Differential Revision: https://phabricator.services.mozilla.com/D2102

--HG--
extra : moz-landing-system : lando

taskcluster/ci/build/android-stuff.yml

@@ -80,7 +80,8 @@ android-test-ccov/opt:
         - android-gradle-dependencies
         - android-sdk-linux
     fetches:
-        - grcov-linux-x86_64
+        fetch:
+            - grcov-linux-x86_64
 
 android-lint/opt:
     description: "Android lint"

taskcluster/ci/toolchain/linux.yml

@@ -142,13 +142,14 @@ linux64-gcc-4.9:
             - 'build/unix/build-gcc/build-gcc.sh'
         toolchain-artifact: public/build/gcc.tar.xz
     fetches:
-        - binutils-2.25.1
-        - cloog-0.18.1
-        - gcc-4.9.4
-        - gmp-5.1.3
-        - isl-0.12.2
-        - mpc-0.8.2
-        - mpfr-3.1.5
+        fetch:
+            - binutils-2.25.1
+            - cloog-0.18.1
+            - gcc-4.9.4
+            - gmp-5.1.3
+            - isl-0.12.2
+            - mpc-0.8.2
+            - mpfr-3.1.5
 
 linux64-gcc-6:
     description: "GCC 6 toolchain build"
@@ -168,12 +169,13 @@ linux64-gcc-6:
         toolchain-alias: linux64-gcc
         toolchain-artifact: public/build/gcc.tar.xz
     fetches:
-        - binutils-2.28.1
-        - gcc-6.4.0
-        - gmp-5.1.3
-        - isl-0.15
-        - mpc-0.8.2
-        - mpfr-3.1.5
+        fetch:
+            - binutils-2.28.1
+            - gcc-6.4.0
+            - gmp-5.1.3
+            - isl-0.15
+            - mpc-0.8.2
+            - mpfr-3.1.5
 
 linux64-gcc-7:
     description: "GCC 7 toolchain build"
@@ -192,12 +194,13 @@ linux64-gcc-7:
             - 'build/unix/build-gcc/build-gcc.sh'
         toolchain-artifact: public/build/gcc.tar.xz
     fetches:
-        - binutils-2.28.1
-        - gcc-7.3.0
-        - gmp-6.1.0
-        - isl-0.16.1
-        - mpc-1.0.3
-        - mpfr-3.1.4
+        fetch:
+            - binutils-2.28.1
+            - gcc-7.3.0
+            - gmp-6.1.0
+            - isl-0.16.1
+            - mpc-1.0.3
+            - mpfr-3.1.4
 
 linux64-gcc-sixgill:
     description: "sixgill GCC plugin build"
@@ -219,12 +222,13 @@ linux64-gcc-sixgill:
     toolchains:
         - linux64-gcc-6
     fetches:
-        - binutils-2.28.1
-        - isl-0.15
-        - gcc-6.4.0
-        - gmp-5.1.3
-        - mpc-0.8.2
-        - mpfr-3.1.5
+        fetch:
+            - binutils-2.28.1
+            - isl-0.15
+            - gcc-6.4.0
+            - gmp-5.1.3
+            - mpc-0.8.2
+            - mpfr-3.1.5
 
 linux64-llvm-dsymutil:
     description: "llvm-dsymutil toolchain build"
@@ -648,12 +652,13 @@ linux64-mingw32-gcc:
             - 'build/unix/build-gcc/build-gcc.sh'
         toolchain-artifact: public/build/mingw32.tar.xz
     fetches:
-        - binutils-2.27
-        - gcc-6.4.0
-        - gmp-5.1.3
-        - isl-0.15
-        - mpc-0.8.2
-        - mpfr-3.1.5
+        fetch:
+            - binutils-2.27
+            - gcc-6.4.0
+            - gmp-5.1.3
+            - isl-0.15
+            - mpc-0.8.2
+            - mpfr-3.1.5
 
 linux64-mingw32-nsis:
     description: "NSIS build for MinGW32 Cross Compile"

taskcluster/taskgraph/transforms/job/__init__.py

@@ -20,6 +20,7 @@ from taskgraph.util.schema import (
     validate_schema,
     Schema,
 )
+from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import (
@@ -78,7 +79,9 @@ job_description_schema = Schema({
     }),
 
     # A list of artifacts to install from 'fetch' tasks.
-    Optional('fetches'): [basestring],
+    Optional('fetches'): {
+        basestring: [basestring],
+    },
 
     # A description of how to run this job.
     'run': {
@@ -141,41 +144,57 @@ def get_attribute(dict, key, attributes, attribute_name):
 
 @transforms.add
 def use_fetches(config, jobs):
-    artifacts = {}
+    all_fetches = {}
 
     for task in config.kind_dependencies_tasks:
         if task.kind != 'fetch':
             continue
 
         name = task.label.replace('%s-' % task.kind, '')
-        get_attribute(artifacts, name, task.attributes, 'fetch-artifact')
+        get_attribute(all_fetches, name, task.attributes, 'fetch-artifact')
 
     for job in jobs:
-        fetches = job.pop('fetches', [])
+        fetches = job.pop('fetches', None)
+        if not fetches:
+            yield job
+            continue
 
         # Hack added for `mach artifact toolchain` to support reading toolchain
         # kinds in isolation.
-        if config.params.get('ignore_fetches'):
-            fetches[:] = []
+        if 'fetch' in fetches and config.params.get('ignore_fetches'):
+            fetches['fetch'][:] = []
 
-        for fetch in fetches:
-            if fetch not in artifacts:
-                raise Exception('Missing fetch job for %s-%s: %s' % (
-                    config.kind, job['name'], fetch))
+        job_fetches = []
+        name = job.get('name', job.get('label'))
+        dependencies = job.setdefault('dependencies', {})
+        prefix = get_artifact_prefix(job)
+        for kind, artifacts in fetches.items():
+            if kind == 'fetch':
+                for fetch in artifacts:
+                    if fetch not in all_fetches:
+                        raise Exception('Missing fetch job for {kind}-{name}: {fetch}'.format(
+                            kind=config.kind, name=name, fetch=fetch))
 
-            if not artifacts[fetch].startswith('public/'):
-                raise Exception('non-public artifacts not supported')
+                    path = all_fetches[fetch]
+                    if not path.startswith('public/'):
+                        raise Exception('Non-public artifacts not supported for {kind}-{name}: '
+                                        '{fetch}'.format(kind=config.kind, name=name, fetch=fetch))
 
-        if fetches:
-            job.setdefault('dependencies', {}).update(
-                ('fetch-%s' % f, 'fetch-%s' % f)
-                for f in fetches)
+                    dep = 'fetch-{}'.format(fetch)
+                    dependencies[dep] = dep
+                    job_fetches.append('{path}@<{dep}>'.format(path=path, dep=dep))
 
-            env = job.setdefault('worker', {}).setdefault('env', {})
-            env['MOZ_FETCHES'] = {'task-reference': ' '.join(
-                '%s@<fetch-%s>' % (artifacts[f], f)
-                for f in fetches)}
+            else:
+                if kind not in dependencies:
+                    raise Exception("{name} can't fetch {kind} artifacts because "
+                                    "it has no {kind} dependencies!".format(name=name, kind=kind))
 
+                for path in artifacts:
+                    job_fetches.append('{prefix}/{path}@<{dep}>'.format(
+                        prefix=prefix, path=path, dep=kind))
+
+        env = job.setdefault('worker', {}).setdefault('env', {})
+        env['MOZ_FETCHES'] = {'task-reference': ' '.join(job_fetches)}
         yield job
 
 

taskcluster/taskgraph/transforms/tests.py

@@ -740,12 +740,16 @@ def enable_code_coverage(config, tests):
             test['optimization'] = None
 
             # Add a fetch task for the grcov binary.
+            if any(p in test['build-platform'] for p in ('linux', 'osx', 'win')):
+                test.setdefault('fetches', {})
+                test['fetches'].setdefault('fetch', [])
+
             if 'linux' in test['build-platform']:
-                test['fetches'] = ['grcov-linux-x86_64']
+                test['fetches']['fetch'].append('grcov-linux-x86_64')
             elif 'osx' in test['build-platform']:
-                test['fetches'] = ['grcov-osx-x86_64']
+                test['fetches']['fetch'].append('grcov-osx-x86_64')
             elif 'win' in test['build-platform']:
-                test['fetches'] = ['grcov-win-x86_64']
+                test['fetches']['fetch'].append('grcov-win-x86_64')
 
             if 'talos' in test['test-name']:
                 test['max-run-time'] = 7200