mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-09 19:35:51 +00:00
Bug 1286075: rename taskgraph.transforms.make_task; r=wcosta
Rename to taskgraph.transforms.task. This also adds some Required and Optional declarations to the schema to be explicit, and adjusts the transform to handle treeherder being optional. MozReview-Commit-ID: FuKYayvlwB9 --HG-- rename : taskcluster/taskgraph/transforms/make_task.py => taskcluster/taskgraph/transforms/task.py extra : rebase_source : 0913aa8cdf153cd086a7786de957535e9b3a4ee8
This commit is contained in:
Dustin J. Mitchell
parent
09fd15c2e1
commit
e7fecbd4ae
@ -9,4 +9,4 @@ transforms:
|
||||
- taskgraph.transforms.tests.all_kinds:transforms
|
||||
- taskgraph.transforms.tests.test_description:validate
|
||||
- taskgraph.transforms.tests.make_task_description:transforms
|
||||
- taskgraph.transforms.make_task:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
@ -9,4 +9,4 @@ transforms:
|
||||
- taskgraph.transforms.tests.all_kinds:transforms
|
||||
- taskgraph.transforms.tests.test_description:validate
|
||||
- taskgraph.transforms.tests.make_task_description:transforms
|
||||
- taskgraph.transforms.make_task:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
@ -81,10 +81,10 @@ common. They all run on one of a small set of worker implementations, each
|
||||
with their own idiosyncracies. And they all report to TreeHerder in a similar
|
||||
way.
|
||||
|
||||
The transforms in ``taskcluster/taskgraph/transforms/make_task.py`` implement
|
||||
The transforms in ``taskcluster/taskgraph/transforms/task.py`` implement
|
||||
this common functionality. They expect a "task description", and produce a
|
||||
task definition. The schema for a task description is defined at the top of
|
||||
``make_task.py``, with copious comments. The parts of the task description
|
||||
``task.py``, with copious comments. The parts of the task description
|
||||
that are specific to a worker implementation are isolated in a ``worker``
|
||||
object which has an ``implementation`` property naming the worker
|
||||
implementation. Thus the transforms that produce a task description must be
|
||||
@ -130,7 +130,7 @@ configuration in ``kind.yml``:
|
||||
embodies the specifics of how test runs work: invoking mozharness, various
|
||||
worker options, and so on.
|
||||
|
||||
* Finally, the ``taskgraph.transforms.make_task:transforms``, described above
|
||||
* Finally, the ``taskgraph.transforms.task:transforms``, described above
|
||||
under "Task-Generation Transforms", are applied.
|
||||
|
||||
Test dependencies are produced in the form of a dictionary mapping dependency
|
||||
|
||||
@ -8,6 +8,8 @@ transformations is generic to any kind of task, but abstracts away some of the
|
||||
complexities of worker implementations, scopes, and treeherder annotations.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.util.treeherder import split_symbol
|
||||
from taskgraph.transforms.base import (
|
||||
validate_schema,
|
||||
@ -23,38 +25,40 @@ taskref_or_string = Any(
|
||||
# A task description is a general description of a TaskCluster task
|
||||
task_description_schema = Schema({
|
||||
# the label for this task
|
||||
'label': basestring,
|
||||
Required('label'): basestring,
|
||||
|
||||
# description of the task (for metadata)
|
||||
'description': basestring,
|
||||
Required('description'): basestring,
|
||||
|
||||
# attributes for this task
|
||||
'attributes': {basestring: object},
|
||||
Optional('attributes'): {basestring: object},
|
||||
|
||||
# dependencies of this task, keyed by name; these are passed through
|
||||
# verbatim and subject to the interpretation of the Task's get_dependencies
|
||||
# method.
|
||||
'dependencies': {basestring: object},
|
||||
Optional('dependencies'): {basestring: object},
|
||||
|
||||
# expiration and deadline times, relative to task creation, with units
|
||||
# (e.g., "14 days")
|
||||
'expires-after': basestring,
|
||||
'deadline-after': basestring,
|
||||
# (e.g., "14 days"). Defaults are set based on the project.
|
||||
Optional('expires-after'): basestring,
|
||||
Optional('deadline-after'): basestring,
|
||||
|
||||
# custom routes for this task; the default treeherder routes will be added
|
||||
# automatically
|
||||
'routes': [basestring],
|
||||
Optional('routes'): [basestring],
|
||||
|
||||
# custom scopes for this task; any scopes required for the worker will be
|
||||
# added automatically
|
||||
'scopes': [basestring],
|
||||
Optional('scopes'): [basestring],
|
||||
|
||||
# custom "task.extra" content
|
||||
'extra': {basestring: object},
|
||||
Optional('extra'): {basestring: object},
|
||||
|
||||
# treeherder-related information; see
|
||||
# https://schemas.taskcluster.net/taskcluster-treeherder/v1/task-treeherder-config.json
|
||||
'treeherder': {
|
||||
# If not specified, no treeherder extra information or routes will be
|
||||
# added to the task
|
||||
Optional('treeherder'): {
|
||||
# either a bare symbol, or "grp(sym)".
|
||||
'symbol': basestring,
|
||||
|
||||
@ -78,7 +82,7 @@ task_description_schema = Schema({
|
||||
|
||||
# information specific to the worker implementation that will run this task
|
||||
'worker': Any({
|
||||
'implementation': Any('docker-worker', 'docker-engine'),
|
||||
Required('implementation'): Any('docker-worker', 'docker-engine'),
|
||||
|
||||
# the docker image (in docker's `host/repo/image:tag` format) in which
|
||||
# to run the task; if omitted, this will be a reference to the image
|
||||
@ -88,12 +92,14 @@ task_description_schema = Schema({
|
||||
|
||||
# worker features that should be enabled
|
||||
Required('relengapi-proxy', default=False): bool,
|
||||
Required('taskcluster-proxy', default=False): bool,
|
||||
Required('allow-ptrace', default=False): bool,
|
||||
Required('loopback-video', default=False): bool,
|
||||
Required('loopback-audio', default=False): bool,
|
||||
Optional('superseder-url'): basestring,
|
||||
|
||||
# caches to set up for the task
|
||||
'caches': [{
|
||||
Optional('caches'): [{
|
||||
# only one type is supported by any of the workers right now
|
||||
'type': 'persistent',
|
||||
|
||||
@ -106,7 +112,7 @@ task_description_schema = Schema({
|
||||
}],
|
||||
|
||||
# artifacts to extract from the task image after completion
|
||||
'artifacts': [{
|
||||
Optional('artifacts'): [{
|
||||
# type of artifact -- simple file, or recursive directory
|
||||
'type': Any('file', 'directory'),
|
||||
|
||||
@ -119,7 +125,7 @@ task_description_schema = Schema({
|
||||
}],
|
||||
|
||||
# environment variables
|
||||
'env': {basestring: taskref_or_string},
|
||||
Required('env', default={}): {basestring: taskref_or_string},
|
||||
|
||||
# the command to run
|
||||
'command': [taskref_or_string],
|
||||
@ -127,14 +133,14 @@ task_description_schema = Schema({
|
||||
# the maximum time to run, in seconds
|
||||
'max-run-time': int,
|
||||
}, {
|
||||
'implementation': 'generic-worker',
|
||||
Required('implementation'): 'generic-worker',
|
||||
|
||||
# command is a list of commands to run, sequentially
|
||||
'command': [basestring],
|
||||
|
||||
# artifacts to extract from the task image after completion; note that artifacts
|
||||
# for the generic worker cannot have names
|
||||
'artifacts': [{
|
||||
Optional('artifacts'): [{
|
||||
# type of artifact -- simple file, or recursive directory
|
||||
'type': Any('file', 'directory'),
|
||||
|
||||
@ -143,14 +149,15 @@ task_description_schema = Schema({
|
||||
}],
|
||||
|
||||
# environment variables
|
||||
'env': {basestring: taskref_or_string},
|
||||
Required('env', default={}): {basestring: taskref_or_string},
|
||||
|
||||
# the maximum time to run, in seconds
|
||||
'max-run-time': int,
|
||||
}, {
|
||||
'implementation': 'buildbot-bridge',
|
||||
Required('implementation'): 'buildbot-bridge',
|
||||
|
||||
# see https://github.com/mozilla/buildbot-bridge/blob/master/bbb/schemas/payload.yml
|
||||
# see
|
||||
# https://github.com/mozilla/buildbot-bridge/blob/master/bbb/schemas/payload.yml
|
||||
'buildername': basestring,
|
||||
'sourcestamp': {
|
||||
'branch': basestring,
|
||||
@ -298,37 +305,37 @@ def validate(config, tasks):
|
||||
def build_task(config, tasks):
|
||||
for task in tasks:
|
||||
provisioner_id, worker_type = task['worker-type'].split('/', 1)
|
||||
routes = task['routes']
|
||||
scopes = task['scopes']
|
||||
routes = task.get('routes', [])
|
||||
scopes = task.get('scopes', [])
|
||||
|
||||
# set up extra
|
||||
extra = task['extra']
|
||||
extra['treeherderEnv'] = task['treeherder']['environments']
|
||||
extra = task.get('extra', {})
|
||||
task_th = task.get('treeherder')
|
||||
if task_th:
|
||||
extra['treeherderEnv'] = task_th['environments']
|
||||
|
||||
task_th = task['treeherder']
|
||||
treeherder = extra.setdefault('treeherder', {})
|
||||
treeherder = extra.setdefault('treeherder', {})
|
||||
|
||||
machine_platform, collection = task_th['platform'].split('/', 1)
|
||||
treeherder['machine'] = {'platform': machine_platform}
|
||||
treeherder['collection'] = {collection: True}
|
||||
machine_platform, collection = task_th['platform'].split('/', 1)
|
||||
treeherder['machine'] = {'platform': machine_platform}
|
||||
treeherder['collection'] = {collection: True}
|
||||
|
||||
groupSymbol, symbol = split_symbol(task_th['symbol'])
|
||||
if groupSymbol != '?':
|
||||
groupSymbol, symbol = split_symbol(task_th['symbol'])
|
||||
treeherder['groupSymbol'] = groupSymbol
|
||||
if groupSymbol not in GROUP_NAMES:
|
||||
raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
|
||||
treeherder['groupName'] = GROUP_NAMES[groupSymbol]
|
||||
treeherder['symbol'] = symbol
|
||||
treeherder['jobKind'] = task_th['kind']
|
||||
treeherder['tier'] = task_th['tier']
|
||||
treeherder['symbol'] = symbol
|
||||
treeherder['jobKind'] = task_th['kind']
|
||||
treeherder['tier'] = task_th['tier']
|
||||
|
||||
routes.extend([
|
||||
'{}.v2.{}.{}.{}'.format(root,
|
||||
config.params['project'],
|
||||
config.params['head_rev'],
|
||||
config.params['pushlog_id'])
|
||||
for root in 'tc-treeherder', 'tc-treeherder-stage'
|
||||
])
|
||||
routes.extend([
|
||||
'{}.v2.{}.{}.{}'.format(root,
|
||||
config.params['project'],
|
||||
config.params['head_rev'],
|
||||
config.params['pushlog_id'])
|
||||
for root in 'tc-treeherder', 'tc-treeherder-stage'
|
||||
])
|
||||
|
||||
task_def = {
|
||||
'provisionerId': provisioner_id,
|
||||
@ -37,8 +37,8 @@ transforms = TransformSequence()
|
||||
|
||||
@transforms.add
|
||||
def make_task_description(config, tests):
|
||||
"""Convert *test* descriptions to *task* descriptions, suitable for input
|
||||
to make_task"""
|
||||
"""Convert *test* descriptions to *task* descriptions (input to
|
||||
taskgraph.transforms.task)"""
|
||||
|
||||
for test in tests:
|
||||
label = '{}-{}-{}'.format(config.kind, test['test-platform'], test['test-name'])
|
||||
|
||||
Loading…
Reference in New Issue
Block a user