mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-16 14:55:47 +00:00
Bug 1469436 - Create a multi_dep loader and assign a first consumer in l10n. r=tomprince r=aki
Differential Revision: https://phabricator.services.mozilla.com/D1696 --HG-- extra : rebase_source : d6a9c1b12bbf53dba3e78ecdaaf30c5964e43d2e
This commit is contained in:
Justin Wood
parent
a2773825ad
commit
31b8682d56
@ -2,8 +2,9 @@
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
loader: taskgraph.loader.single_dep:loader
|
||||
loader: taskgraph.loader.multi_dep:loader
|
||||
|
||||
group-by: platform
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.l10n:transforms
|
||||
@ -13,6 +14,9 @@ transforms:
|
||||
|
||||
kind-dependencies:
|
||||
- build
|
||||
- build-signing
|
||||
- repackage
|
||||
- repackage-signing
|
||||
- toolchain
|
||||
|
||||
only-for-build-platforms:
|
||||
@ -99,27 +103,27 @@ job-template:
|
||||
linux.*: # linux64 and 32 get same treatment here
|
||||
EN_US_PACKAGE_NAME: target.tar.bz2
|
||||
EN_US_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<signed-build>/artifacts/{artifact_prefix}
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/{artifact_prefix}
|
||||
MAR_TOOLS_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
|
||||
macosx64:
|
||||
EN_US_PACKAGE_NAME: target.dmg
|
||||
EN_US_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<repackage>/artifacts/{artifact_prefix}
|
||||
MAR_TOOLS_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
|
||||
win.*:
|
||||
EN_US_PACKAGE_NAME: target.zip
|
||||
EN_US_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<signed-build>/artifacts/{artifact_prefix}
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/{artifact_prefix}
|
||||
EN_US_INSTALLER_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<repackage-signed>/artifacts/{artifact_prefix}
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<repackage-signing>/artifacts/{artifact_prefix}
|
||||
MAR_TOOLS_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
|
||||
android-api-16:
|
||||
EN_US_PACKAGE_NAME: target.apk
|
||||
EN_US_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}
|
||||
mozharness:
|
||||
config:
|
||||
by-build-platform:
|
||||
|
||||
@ -2,7 +2,9 @@
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
loader: taskgraph.loader.single_dep:loader
|
||||
loader: taskgraph.loader.multi_dep:loader
|
||||
|
||||
group-by: platform
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.l10n:transforms
|
||||
@ -12,6 +14,9 @@ transforms:
|
||||
|
||||
kind-dependencies:
|
||||
- build
|
||||
- build-signing
|
||||
- repackage
|
||||
- repackage-signing
|
||||
- toolchain
|
||||
|
||||
only-for-build-platforms:
|
||||
@ -123,27 +128,27 @@ job-template:
|
||||
linux.*: # linux64 and 32 get same treatment here
|
||||
EN_US_PACKAGE_NAME: target.tar.bz2
|
||||
EN_US_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<signed-build>/artifacts/{artifact_prefix}
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/{artifact_prefix}
|
||||
MAR_TOOLS_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
|
||||
macosx64.*:
|
||||
EN_US_PACKAGE_NAME: target.dmg
|
||||
EN_US_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<repackage>/artifacts/{artifact_prefix}
|
||||
MAR_TOOLS_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
|
||||
win.*:
|
||||
EN_US_PACKAGE_NAME: target.zip
|
||||
EN_US_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<signed-build>/artifacts/{artifact_prefix}
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/{artifact_prefix}
|
||||
EN_US_INSTALLER_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<repackage-signed>/artifacts/{artifact_prefix}
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<repackage-signing>/artifacts/{artifact_prefix}
|
||||
MAR_TOOLS_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
|
||||
android-api-16-nightly:
|
||||
EN_US_PACKAGE_NAME: target.apk
|
||||
EN_US_BINARY_URL:
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/en-US
|
||||
task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/en-US
|
||||
mozharness:
|
||||
config:
|
||||
by-build-platform:
|
||||
|
||||
99
taskcluster/taskgraph/loader/multi_dep.py
Normal file
99
taskcluster/taskgraph/loader/multi_dep.py
Normal file
@ -0,0 +1,99 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import copy
|
||||
|
||||
|
||||
# Define a collection of group_by functions
|
||||
GROUP_BY_MAP = {}
|
||||
|
||||
|
||||
def group_by(name):
|
||||
def wrapper(func):
|
||||
GROUP_BY_MAP[name] = func
|
||||
return func
|
||||
return wrapper
|
||||
|
||||
|
||||
def loader(kind, path, config, params, loaded_tasks):
|
||||
"""
|
||||
Load tasks based on the jobs dependant kinds, designed for use as
|
||||
multiple-dependent needs.
|
||||
|
||||
Required ``group-by-fn`` is used to define how we coalesce the
|
||||
multiple deps together to pass to transforms, e.g. all kinds specified get
|
||||
collapsed by platform with `platform`
|
||||
|
||||
The `only-for-build-platforms` kind configuration, if specified, will limit
|
||||
the build platforms for which a job will be created. Alternatively there is
|
||||
'not-for-build-platforms' kind configuration which will be consulted only after
|
||||
'only-for-build-platforms' is checked (if present), and omit any jobs where the
|
||||
build platform matches.
|
||||
|
||||
Optional ``job-template`` kind configuration value, if specified, will be used to
|
||||
pass configuration down to the specified transforms used.
|
||||
"""
|
||||
job_template = config.get('job-template')
|
||||
|
||||
for dep_tasks in group_tasks(config, loaded_tasks):
|
||||
job = {'dependent-tasks': dep_tasks}
|
||||
if job_template:
|
||||
job.update(copy.deepcopy(job_template))
|
||||
# copy shipping_product from upstream
|
||||
product = dep_tasks[dep_tasks.keys()[0]].attributes.get(
|
||||
'shipping_product',
|
||||
dep_tasks[dep_tasks.keys()[0]].task.get('shipping-product')
|
||||
)
|
||||
if product:
|
||||
job.setdefault('shipping-product', product)
|
||||
|
||||
yield job
|
||||
|
||||
|
||||
def group_tasks(config, tasks):
|
||||
group_by_fn = GROUP_BY_MAP[config['group-by']]
|
||||
|
||||
groups = group_by_fn(config, tasks)
|
||||
|
||||
for combinations in groups.itervalues():
|
||||
kinds = [f.kind for f in combinations]
|
||||
assert_unique_members(kinds, error_msg=(
|
||||
"Multi_dep.py should have filtered down to one task per kind"))
|
||||
dependencies = {t.kind: copy.deepcopy(t) for t in combinations}
|
||||
yield dependencies
|
||||
|
||||
|
||||
@group_by('platform')
|
||||
def platform_grouping(config, tasks):
|
||||
only_platforms = config.get('only-for-build-platforms')
|
||||
not_platforms = config.get('not-for-build-platforms')
|
||||
|
||||
groups = {}
|
||||
for task in tasks:
|
||||
if task.kind not in config.get('kind-dependencies', []):
|
||||
continue
|
||||
platform = task.attributes.get('build_platform')
|
||||
build_type = task.attributes.get('build_type')
|
||||
product = task.attributes.get('shipping_product',
|
||||
task.task.get('shipping-product'))
|
||||
|
||||
# Skip only_ and not_ platforms that don't match
|
||||
if only_platforms or not_platforms:
|
||||
if not platform or not build_type:
|
||||
continue
|
||||
combined_platform = "{}/{}".format(platform, build_type)
|
||||
if only_platforms and combined_platform not in only_platforms:
|
||||
continue
|
||||
elif not_platforms and combined_platform in not_platforms:
|
||||
continue
|
||||
|
||||
groups.setdefault((platform, build_type, product), []).append(task)
|
||||
return groups
|
||||
|
||||
|
||||
def assert_unique_members(kinds, error_msg=None):
|
||||
if len(kinds) != len(set(kinds)):
|
||||
raise Exception(error_msg)
|
||||
@ -97,8 +97,8 @@ l10n_description_schema = Schema({
|
||||
|
||||
Optional('run-on-projects'): job_description_schema['run-on-projects'],
|
||||
|
||||
# task object of the dependent task
|
||||
Required('dependent-task'): object,
|
||||
# dictionary of dependent task objects, keyed by kind.
|
||||
Required('dependent-tasks'): {basestring: object},
|
||||
|
||||
# worker-type to utilize
|
||||
Required('worker-type'): _by_platform(basestring),
|
||||
@ -197,7 +197,7 @@ def _remove_locales(locales, to_remove=None):
|
||||
@transforms.add
|
||||
def setup_name(config, jobs):
|
||||
for job in jobs:
|
||||
dep = job['dependent-task']
|
||||
dep = job['dependent-tasks']['build']
|
||||
# Set the name to the same as the dep task, without kind name.
|
||||
# Label will get set automatically with this kinds name.
|
||||
job['name'] = job.get('name',
|
||||
@ -208,7 +208,7 @@ def setup_name(config, jobs):
|
||||
@transforms.add
|
||||
def copy_in_useful_magic(config, jobs):
|
||||
for job in jobs:
|
||||
dep = job['dependent-task']
|
||||
dep = job['dependent-tasks']['build']
|
||||
attributes = copy_attributes_from_dependent_job(dep)
|
||||
attributes.update(job.get('attributes', {}))
|
||||
# build-platform is needed on `job` for by-build-platform
|
||||
@ -229,20 +229,19 @@ def validate_early(config, jobs):
|
||||
def setup_nightly_dependency(config, jobs):
|
||||
""" Sets up a task dependency to the signing job this relates to """
|
||||
for job in jobs:
|
||||
job['dependencies'] = {'unsigned-build': job['dependent-task'].label}
|
||||
job['dependencies'] = {'build': job['dependent-tasks']['build'].label}
|
||||
if job['attributes']['build_platform'].startswith('win') or \
|
||||
job['attributes']['build_platform'].startswith('linux'):
|
||||
# Weave these in and just assume they will be there in the resulting graph
|
||||
job['dependencies'].update({
|
||||
'signed-build': 'build-signing-{}'.format(job['name']),
|
||||
'build-signing': job['dependent-tasks']['build-signing'].label,
|
||||
})
|
||||
if job['attributes']['build_platform'].startswith('macosx'):
|
||||
job['dependencies'].update({
|
||||
'repackage': 'repackage-{}'.format(job['name'])
|
||||
'repackage': job['dependent-tasks']['repackage'].label
|
||||
})
|
||||
if job['attributes']['build_platform'].startswith('win'):
|
||||
job['dependencies'].update({
|
||||
'repackage-signed': 'repackage-signing-{}'.format(job['name'])
|
||||
'repackage-signing': job['dependent-tasks']['repackage-signing'].label
|
||||
})
|
||||
yield job
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user