Bug 1336559 - Add ability to depend on build artifacts to 'run_task' based tasks, r=dustin

Currently 'run_task' tasks have no easy way to depend on a build task. For example, some
python unittests need a Firefox binary for their tests, like the mozrunner tests and future
test harness selftests (like mochitest tests).

This patch allows kinds to add a new key to the kind config which maps test platforms to
build-labels. Then 'run_task' jobs can add a 'requires-build': true field to get a build
dependency automatically added. The build artifact url will also be stored in the
$GECKO_INSTALLER_URL environment variable on the test host.

MozReview-Commit-ID: Jqyhbj7nC6z

--HG--
extra : rebase_source : 2f44b6c94f35a0d2e11464cf773e821ae6fe8538

taskcluster/ci/source-test/kind.yml

@@ -14,3 +14,8 @@ jobs-from:
     - mozlint.yml
     - doc.yml
     - webidl.yml
+
+# This is used by run-task based tasks to lookup which build task it
+# should depend on based on its own platform.
+dependent-build-platforms:
+    linux64.*: build-linux64/opt

taskcluster/taskgraph/transforms/job/__init__.py

@@ -133,11 +133,6 @@ def make_task_description(config, jobs):
             job['label'] = '{}-{}'.format(config.kind, job['name'])
         if job['name']:
             del job['name']
-        if 'platform' in job:
-            if 'treeherder' in job:
-                job['treeherder']['platform'] = job['platform']
-            del job['platform']
-
         taskdesc = copy.deepcopy(job)
 
         # fill in some empty defaults to make run implementations easier
@@ -152,6 +147,11 @@ def make_task_description(config, jobs):
         configure_taskdesc_for_run(config, job, taskdesc)
         del taskdesc['run']
 
+        if 'platform' in taskdesc:
+            if 'treeherder' in taskdesc:
+                taskdesc['treeherder']['platform'] = taskdesc['platform']
+            del taskdesc['platform']
+
         # yield only the task description, discarding the job description
         yield taskdesc
 

taskcluster/taskgraph/transforms/job/common.py

@@ -9,6 +9,10 @@ consistency.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+from taskgraph.util.attributes import keymatch
+
+
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
 
 
@@ -57,6 +61,35 @@ def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc):
     })
 
 
+def docker_worker_add_build_dependency(config, job, taskdesc):
+    """Add build dependency to the task description and installer_url to env."""
+    key = job['platform']
+    build_labels = config.config.get('dependent-build-platforms', {})
+    matches = keymatch(build_labels, key)
+    if not matches:
+        raise Exception("No build platform found for '{}'. "
+                        "Define 'dependent-build-platforms' in the kind config.".format(key))
+
+    if len(matches) > 1:
+        raise Exception("More than one build platform found for '{}'.".format(key))
+
+    label = matches[0]
+    deps = taskdesc.setdefault('dependencies', {})
+    deps.update({'build': label})
+
+    if 'macosx' in label:
+        target = 'target.dmg'
+    elif 'android' in label:
+        target = 'target.apk'
+    else:
+        target = 'target.tar.bz2'
+    build_artifact = 'public/build/{}'.format(target)
+    installer_url = ARTIFACT_URL.format('<build>', build_artifact)
+
+    env = taskdesc['worker'].setdefault('env', {})
+    env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
+
+
 def docker_worker_support_vcs_checkout(config, job, taskdesc):
     """Update a job/task with parameters to enable a VCS checkout.
 

taskcluster/taskgraph/transforms/job/mach.py

@@ -16,6 +16,12 @@ mach_schema = Schema({
 
     # The mach command (omitting `./mach`) to run
     Required('mach'): basestring,
+
+    # Whether the job requires a build artifact or not. If True, the task
+    # will depend on a build task and run-task will download and set up the
+    # installer. Build labels are determined by the `dependent-build-platforms`
+    # config in kind.yml.
+    Required('requires-build', default=False): bool,
 })
 
 

taskcluster/taskgraph/transforms/job/run_task.py

@@ -11,6 +11,7 @@ import copy
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
+    docker_worker_add_build_dependency,
     docker_worker_support_vcs_checkout,
 )
 from voluptuous import Schema, Required, Any
@@ -29,6 +30,12 @@ run_task_schema = Schema({
     # checkout arguments.  If a list, it will be passed directly; otherwise
     # it will be included in a single argument to `bash -cx`.
     Required('command'): Any([basestring], basestring),
+
+    # Whether the job requires a build artifact or not. If True, the task
+    # will depend on a build task and run-task will download and set up the
+    # installer. Build labels are determined by the `dependent-build-platforms`
+    # config in kind.yml.
+    Required('requires-build', default=False): bool,
 })
 
 
@@ -41,6 +48,9 @@ def docker_worker_run_task(config, job, taskdesc):
     if run['checkout']:
         docker_worker_support_vcs_checkout(config, job, taskdesc)
 
+    if run['requires-build']:
+        docker_worker_add_build_dependency(config, job, taskdesc)
+
     if run.get('cache-dotcache') and int(config.params['level']) > 1:
         worker['caches'].append({
             'type': 'persistent',