Bug 1335651 - Setup an index path in the gecko.cache namespace for toolchain builds. r=dustin

This allows to find them and optimize them out during the taskgraph
optimization phase, and will allow to get toolchain artifacts through a
mach command for developers.

The index path is generated similarly to git trees or mercurial
manifests, and allows to find the right task corresponding to the the
contents of the files in the task `extra.resources` along the toolchain
scripts.

`when.files-changed` is not used when a task has index paths because we
need tasks to happen independently of whether there were changes to
those files when the index or artifacts expire.

--HG--
extra : rebase_source : e9995cee0ee39d7b64090a243e380aeae336a69f

taskcluster/ci/toolchain/linux.yml

@@ -9,17 +9,16 @@ linux64-clang/opt:
         platform: toolchains/opt
         symbol: TL(clang)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-clang-linux.sh
-        tooltool-downloads: public
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
-    when:
-        files-changed:
+    run:
+        using: toolchain-script
+        script: build-clang-linux.sh
+        tooltool-downloads: public
+        resources:
             - 'build/build-clang/**'
 
 linux64-clang-tidy/opt:
@@ -32,17 +31,16 @@ linux64-clang-tidy/opt:
         platform: toolchains/opt
         symbol: TL(clang-tidy)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-clang-tidy-linux.sh
-        tooltool-downloads: public
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
-    when:
-        files-changed:
+    run:
+        using: toolchain-script
+        script: build-clang-tidy-linux.sh
+        tooltool-downloads: public
+        resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
 
@@ -53,16 +51,15 @@ linux64-gcc/opt:
         platform: toolchains/opt
         symbol: TL(gcc)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-gcc-linux.sh
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
-    when:
-        files-changed:
+    run:
+        using: toolchain-script
+        script: build-gcc-linux.sh
+        resources:
             - 'build/unix/build-gcc/**'
 
 linux64-binutils/opt:
@@ -72,16 +69,15 @@ linux64-binutils/opt:
         platform: toolchains/opt
         symbol: TL(binutil)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-binutils-linux.sh
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
-    when:
-        files-changed:
+    run:
+        using: toolchain-script
+        script: build-binutils-linux.sh
+        resources:
             - 'build/unix/build-binutils/**'
 
 linux64-cctools-port/opt:
@@ -91,14 +87,14 @@ linux64-cctools-port/opt:
         platform: toolchains/opt
         symbol: TL(cctools)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-cctools-port.sh
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
+    run:
+        using: toolchain-script
+        script: build-cctools-port.sh
 
 linux64-hfsplus/opt:
     description: "hfsplus toolchain build"
@@ -107,17 +103,16 @@ linux64-hfsplus/opt:
         platform: toolchains/opt
         symbol: TL(hfs+)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-hfsplus-linux.sh
-        tooltool-downloads: public
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
-    when:
-        files-changed:
+    run:
+        using: toolchain-script
+        script: build-hfsplus-linux.sh
+        tooltool-downloads: public
+        resources:
             - 'build/unix/build-hfsplus/**'
 
 linux64-libdmg/opt:
@@ -127,11 +122,11 @@ linux64-libdmg/opt:
         platform: toolchains/opt
         symbol: TL(libdmg-hfs+)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-libdmg-hfsplus.sh
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
+    run:
+        using: toolchain-script
+        script: build-libdmg-hfsplus.sh

taskcluster/ci/toolchain/macosx.yml

@@ -9,17 +9,16 @@ macosx64-clang/opt:
         platform: toolchains/opt
         symbol: TM(clang)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-clang-macosx.sh
-        tooltool-downloads: internal
     worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
-    when:
-        files-changed:
+    run:
+        using: toolchain-script
+        script: build-clang-macosx.sh
+        tooltool-downloads: internal
+        resources:
             - 'build/build-clang/**'
 
 macosx64-clang-tidy/opt:
@@ -32,17 +31,16 @@ macosx64-clang-tidy/opt:
         platform: toolchains/opt
         symbol: TM(clang-tidy)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-clang-tidy-macosx.sh
-        tooltool-downloads: internal
     worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
-    when:
-        files-changed:
+    run:
+        using: toolchain-script
+        script: build-clang-tidy-macosx.sh
+        tooltool-downloads: internal
+        resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
 
@@ -53,12 +51,12 @@ macosx64-cctools-port/opt:
         platform: toolchains/opt
         symbol: TM(cctools)
         tier: 1
-    run:
-        using: toolchain-script
-        script: build-cctools-port-macosx.sh
-        tooltool-downloads: internal
     worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
     worker:
         implementation: docker-worker
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
+    run:
+        using: toolchain-script
+        script: build-cctools-port-macosx.sh
+        tooltool-downloads: internal

taskcluster/ci/toolchain/windows.yml

@@ -16,8 +16,7 @@ win32-clang-cl/opt:
     run:
         using: toolchain-script
         script: build-clang32-windows.sh
-    when:
-        files-changed:
+        resources:
             - 'build/build-clang/**'
 
 win64-clang-cl/opt:
@@ -34,8 +33,7 @@ win64-clang-cl/opt:
     run:
         using: toolchain-script
         script: build-clang64-windows.sh
-    when:
-        files-changed:
+        resources:
             - 'build/build-clang/**'
 
 win32-clang-tidy/opt:
@@ -55,8 +53,7 @@ win32-clang-tidy/opt:
     run:
         using: toolchain-script
         script: build-clang-tidy32-windows.sh
-    when:
-        files-changed:
+        resources:
             - 'build/build-clang/**'
 
 win64-clang-tidy/opt:
@@ -76,6 +73,5 @@ win64-clang-tidy/opt:
     run:
         using: toolchain-script
         script: build-clang-tidy64-windows.sh
-    when:
-        files-changed:
+        resources:
             - 'build/build-clang/**'

taskcluster/taskgraph/task/transform.py

@@ -80,13 +80,19 @@ class TransformTask(base.Task):
         self.dependencies = task['dependencies']
         self.when = task['when']
         super(TransformTask, self).__init__(kind, task['label'],
-                                            task['attributes'], task['task'])
+                                            task['attributes'], task['task'],
+                                            index_paths=task.get('index-paths'))
 
     def get_dependencies(self, taskgraph):
         return [(label, name) for name, label in self.dependencies.items()]
 
     def optimize(self, params):
-        if 'files-changed' in self.when:
+        if self.index_paths:
+            optimized, taskId = super(TransformTask, self).optimize(params)
+            if optimized:
+                return optimized, taskId
+
+        elif 'files-changed' in self.when:
             changed = files_changed.check(
                 params, self.when['files-changed'])
             if not changed:

taskcluster/taskgraph/transforms/job/toolchain.py

@@ -7,7 +7,9 @@ Support for running toolchain-building jobs via dedicated scripts
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-from voluptuous import Schema, Required, Any
+import os
+
+from voluptuous import Schema, Optional, Required, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
@@ -15,6 +17,11 @@ from taskgraph.transforms.job.common import (
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_support_vcs_checkout,
 )
+from taskgraph.util.hash import hash_paths
+
+
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..', '..'))
+TOOLCHAIN_INDEX = 'gecko.cache.level-{level}.toolchains.v1.{name}.{digest}'
 
 toolchain_run_schema = Schema({
     Required('using'): 'toolchain-script',
@@ -29,16 +36,38 @@ toolchain_run_schema = Schema({
         'public',
         'internal',
     ),
+
+    # Paths/patterns pointing to files that influence the outcome of a
+    # toolchain build.
+    Optional('resources'): [basestring],
 })
 
 
-def add_files_changed(run, taskdesc):
-    files = taskdesc.setdefault('when', {}).setdefault('files-changed', [])
+def add_index_paths(config, run, taskdesc):
+    files = list(run.get('resources', []))
     # This file
     files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
     # The script
     files.append('taskcluster/scripts/misc/{}'.format(run['script']))
 
+    label = taskdesc['label']
+    subs = {
+        'name': label.replace('toolchain-', '').split('/')[0],
+        'digest': hash_paths(GECKO, files),
+    }
+
+    index_paths = taskdesc.setdefault('index-paths', [])
+
+    # We'll try to find a cached version of the toolchain at levels above
+    # and including the current level, starting at the highest level.
+    for level in reversed(range(int(config.params['level']), 4)):
+        subs['level'] = level
+        index_paths.append(TOOLCHAIN_INDEX.format(**subs))
+
+    # ... and cache at the lowest level.
+    taskdesc.setdefault('routes', []).append(
+        'index.{}'.format(TOOLCHAIN_INDEX.format(**subs)))
+
 
 @run_job_using("docker-worker", "toolchain-script", schema=toolchain_run_schema)
 def docker_worker_toolchain(config, job, taskdesc):
@@ -97,7 +126,7 @@ def docker_worker_toolchain(config, job, taskdesc):
             run['script'])
     ]
 
-    add_files_changed(run, taskdesc)
+    add_index_paths(config, run, taskdesc)
 
 
 @run_job_using("generic-worker", "toolchain-script", schema=toolchain_run_schema)
@@ -146,4 +175,4 @@ def windows_toolchain(config, job, taskdesc):
         r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
     ]
 
-    add_files_changed(run, taskdesc)
+    add_index_paths(config, run, taskdesc)

taskcluster/taskgraph/transforms/task.py

@@ -51,6 +51,10 @@ task_description_schema = Schema({
     # automatically
     Optional('routes'): [basestring],
 
+    # The index paths where this task may be cached. Transforms are expected to
+    # fill these automatically when wanted.
+    Optional('index-paths'): [basestring],
+
     # custom scopes for this task; any scopes required for the worker will be
     # added automatically
     Optional('scopes'): [basestring],
@@ -855,6 +859,7 @@ def build_task(config, tasks):
             'task': task_def,
             'dependencies': task.get('dependencies', {}),
             'attributes': attributes,
+            'index-paths': task.get('index-paths'),
             'when': task.get('when', {}),
         }
 

taskcluster/taskgraph/util/hash.py

@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+from mozbuild.util import memoize
+from mozpack.files import FileFinder
+import mozpack.path as mozpath
+import hashlib
+
+
+@memoize
+def _hash_path(path):
+    with open(path) as fh:
+        return (hashlib.sha256(fh.read()).hexdigest(),
+                mozpath.normsep(path))
+
+
+def hash_paths(base_path, patterns):
+    """
+    Give a list of path patterns, return a digest of the contents of all
+    the corresponding files, similarly to git tree objects or mercurial
+    manifests.
+
+    Each file is hashed. The list of all hashes and file paths is then
+    itself hashed to produce the result.
+    """
+    finder = FileFinder(base_path)
+    h = hashlib.sha256()
+    files = {}
+    for pattern in patterns:
+        files.update(finder.find(pattern))
+    for path in sorted(files.keys()):
+        h.update('{} {}\n'.format(
+            _hash_path(mozpath.abspath(mozpath.join(base_path, path))),
+            mozpath.normsep(path)
+        ))
+    return h.hexdigest()