From 2d35842ffcd0a0b31842de6d3d4e61f75b7ef73b Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Fri, 24 Feb 2017 09:04:22 -0500 Subject: [PATCH] Bug 1336559 - Add ability to depend on build artifacts to 'run_task' based tasks, r=dustin Currently 'run_task' tasks have no easy way to depend on a build task. For example, some python unittests need a Firefox binary for their tests, like the mozrunner tests and future test harness selftests (like mochitest tests). This patch allows kinds to add a new key to the kind config which maps test platforms to build-labels. Then 'run_task' jobs can add a 'requires-build': true field to get a build dependency automatically added. The build artifact url will also be stored in the $GECKO_INSTALLER_URL environment variable on the test host. MozReview-Commit-ID: Jqyhbj7nC6z --HG-- extra : rebase_source : 2f44b6c94f35a0d2e11464cf773e821ae6fe8538 --- taskcluster/ci/source-test/kind.yml | 5 +++ .../taskgraph/transforms/job/__init__.py | 10 +++--- .../taskgraph/transforms/job/common.py | 33 +++++++++++++++++++ taskcluster/taskgraph/transforms/job/mach.py | 6 ++++ .../taskgraph/transforms/job/run_task.py | 10 ++++++ 5 files changed, 59 insertions(+), 5 deletions(-) diff --git a/taskcluster/ci/source-test/kind.yml b/taskcluster/ci/source-test/kind.yml index c3d9fe96f14f..04bc09b472da 100644 --- a/taskcluster/ci/source-test/kind.yml +++ b/taskcluster/ci/source-test/kind.yml @@ -14,3 +14,8 @@ jobs-from: - mozlint.yml - doc.yml - webidl.yml + +# This is used by run-task based tasks to lookup which build task it +# should depend on based on its own platform. +dependent-build-platforms: + linux64.*: build-linux64/opt diff --git a/taskcluster/taskgraph/transforms/job/__init__.py b/taskcluster/taskgraph/transforms/job/__init__.py index dd9366ed08a2..7c6d98f81507 100644 --- a/taskcluster/taskgraph/transforms/job/__init__.py +++ b/taskcluster/taskgraph/transforms/job/__init__.py @@ -133,11 +133,6 @@ def make_task_description(config, jobs): job['label'] = '{}-{}'.format(config.kind, job['name']) if job['name']: del job['name'] - if 'platform' in job: - if 'treeherder' in job: - job['treeherder']['platform'] = job['platform'] - del job['platform'] - taskdesc = copy.deepcopy(job) # fill in some empty defaults to make run implementations easier @@ -152,6 +147,11 @@ def make_task_description(config, jobs): configure_taskdesc_for_run(config, job, taskdesc) del taskdesc['run'] + if 'platform' in taskdesc: + if 'treeherder' in taskdesc: + taskdesc['treeherder']['platform'] = taskdesc['platform'] + del taskdesc['platform'] + # yield only the task description, discarding the job description yield taskdesc diff --git a/taskcluster/taskgraph/transforms/job/common.py b/taskcluster/taskgraph/transforms/job/common.py index 59a51d75ad4a..b585a8d73fe7 100644 --- a/taskcluster/taskgraph/transforms/job/common.py +++ b/taskcluster/taskgraph/transforms/job/common.py @@ -9,6 +9,10 @@ consistency. from __future__ import absolute_import, print_function, unicode_literals +from taskgraph.util.attributes import keymatch + + +ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}' @@ -57,6 +61,35 @@ def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc): }) +def docker_worker_add_build_dependency(config, job, taskdesc): + """Add build dependency to the task description and installer_url to env.""" + key = job['platform'] + build_labels = config.config.get('dependent-build-platforms', {}) + matches = keymatch(build_labels, key) + if not matches: + raise Exception("No build platform found for '{}'. " + "Define 'dependent-build-platforms' in the kind config.".format(key)) + + if len(matches) > 1: + raise Exception("More than one build platform found for '{}'.".format(key)) + + label = matches[0] + deps = taskdesc.setdefault('dependencies', {}) + deps.update({'build': label}) + + if 'macosx' in label: + target = 'target.dmg' + elif 'android' in label: + target = 'target.apk' + else: + target = 'target.tar.bz2' + build_artifact = 'public/build/{}'.format(target) + installer_url = ARTIFACT_URL.format('', build_artifact) + + env = taskdesc['worker'].setdefault('env', {}) + env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}}) + + def docker_worker_support_vcs_checkout(config, job, taskdesc): """Update a job/task with parameters to enable a VCS checkout. diff --git a/taskcluster/taskgraph/transforms/job/mach.py b/taskcluster/taskgraph/transforms/job/mach.py index 8df202dbcce8..c7faf2b3bdcd 100644 --- a/taskcluster/taskgraph/transforms/job/mach.py +++ b/taskcluster/taskgraph/transforms/job/mach.py @@ -16,6 +16,12 @@ mach_schema = Schema({ # The mach command (omitting `./mach`) to run Required('mach'): basestring, + + # Whether the job requires a build artifact or not. If True, the task + # will depend on a build task and run-task will download and set up the + # installer. Build labels are determined by the `dependent-build-platforms` + # config in kind.yml. + Required('requires-build', default=False): bool, }) diff --git a/taskcluster/taskgraph/transforms/job/run_task.py b/taskcluster/taskgraph/transforms/job/run_task.py index 296fe43eee27..5af4361d4015 100644 --- a/taskcluster/taskgraph/transforms/job/run_task.py +++ b/taskcluster/taskgraph/transforms/job/run_task.py @@ -11,6 +11,7 @@ import copy from taskgraph.transforms.job import run_job_using from taskgraph.transforms.job.common import ( + docker_worker_add_build_dependency, docker_worker_support_vcs_checkout, ) from voluptuous import Schema, Required, Any @@ -29,6 +30,12 @@ run_task_schema = Schema({ # checkout arguments. If a list, it will be passed directly; otherwise # it will be included in a single argument to `bash -cx`. Required('command'): Any([basestring], basestring), + + # Whether the job requires a build artifact or not. If True, the task + # will depend on a build task and run-task will download and set up the + # installer. Build labels are determined by the `dependent-build-platforms` + # config in kind.yml. + Required('requires-build', default=False): bool, }) @@ -41,6 +48,9 @@ def docker_worker_run_task(config, job, taskdesc): if run['checkout']: docker_worker_support_vcs_checkout(config, job, taskdesc) + if run['requires-build']: + docker_worker_add_build_dependency(config, job, taskdesc) + if run.get('cache-dotcache') and int(config.params['level']) > 1: worker['caches'].append({ 'type': 'persistent',