Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-18 15:55:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1507898: [taskgraph] Factor out schema validation in transformations; r=dustin

Browse Source 
Most jobs include at least one transform that verifies the input of all the
tasks against a schema. This code is duplicated in each transform. Refactor it,
so that we only need one copy of the logic.

Differential Revision: https://phabricator.services.mozilla.com/D12165

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Tom Prince 2018-11-20 23:44:12 +00:00
parent 355e9a39df
commit becc9482f9
33 changed files with 87 additions and 364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
taskcluster/taskgraph/transforms/balrog_submit.py
View File

@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
get_balrog_server_scope, get_worker_type_for_scope
)
@ -22,8 +21,6 @@ from voluptuous import Any, Required, Optional
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
@ -44,15 +41,8 @@ balrog_description_schema = schema.extend({
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
balrog_description_schema, job,
"In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(balrog_description_schema)
@transforms.add

26
taskcluster/taskgraph/transforms/base.py
View File

@ -6,8 +6,9 @@ from __future__ import absolute_import, print_function, unicode_literals
import attr
from ..parameters import Parameters
from ..config import GraphConfig
from ..parameters import Parameters
from ..util.schema import Schema, validate_schema
@attr.s(frozen=True)
@ -61,3 +62,26 @@ class TransformSequence(object):
def add(self, func):
self._transforms.append(func)
return func
def add_validate(self, schema):
self.add(ValidateSchema(schema))
@attr.s
class ValidateSchema(object):
schema = attr.ib(type=Schema)
def __call__(self, config, tasks):
for task in tasks:
if 'name' in task:
error = "In {kind} kind task {name!r}:".format(
kind=config.kind, name=task['name'])
elif 'label' in task:
error = "In job {label!r}:".format(label=task['label'])
elif 'primary-dependency' in task:
error = "In {kind} kind task for {dependency!r}:".format(
kind=config.kind, dependency=task['primary-dependency'].label)
else:
error = "In unknown task:"
validate_schema(self.schema, task, error)
yield task

10
taskcluster/taskgraph/transforms/beetmover.py
View File

@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
@ -142,14 +141,7 @@ beetmover_description_schema = schema.extend({
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_description_schema, job,
"In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms.add_validate(beetmover_description_schema)
@transforms.add

14
taskcluster/taskgraph/transforms/beetmover_checksums.py
View File

@ -11,7 +11,6 @@ from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
@ -22,8 +21,6 @@ from voluptuous import Any, Required, Optional
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
@ -37,15 +34,8 @@ beetmover_checksums_description_schema = schema.extend({
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_checksums_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(beetmover_checksums_description_schema)
@transforms.add

13
taskcluster/taskgraph/transforms/beetmover_emefree_checksums.py
View File

@ -11,7 +11,6 @@ from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
@ -19,8 +18,6 @@ from voluptuous import Any, Required, Optional
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
@ -34,14 +31,8 @@ beetmover_checksums_description_schema = schema.extend({
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_checksums_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(beetmover_checksums_description_schema)
@transforms.add

15
taskcluster/taskgraph/transforms/beetmover_geckoview.py
View File

@ -12,7 +12,7 @@ from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import \
craft_release_properties as beetmover_craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, resolve_keyed_by, optionally_keyed_by
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.util.scriptworker import get_worker_type_for_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
@ -35,8 +35,6 @@ _MOZ_UPDATE_CHANNEL_PER_BRANCH = {
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
beetmover_description_schema = schema.extend({
Required('depname', default='build'): basestring,
Optional('label'): basestring,
@ -52,15 +50,8 @@ beetmover_description_schema = schema.extend({
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_description_schema, job,
"In beetmover-geckoview ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(beetmover_description_schema)
@transforms.add

14
taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py
View File

@ -11,7 +11,6 @@ from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
@ -22,8 +21,6 @@ from voluptuous import Any, Required, Optional
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
@ -37,15 +34,8 @@ beetmover_checksums_description_schema = schema.extend({
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_checksums_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(beetmover_checksums_description_schema)
@transforms.add

15
taskcluster/taskgraph/transforms/beetmover_push_to_release.py
View File

@ -8,9 +8,7 @@ Transform the beetmover-push-to-release task into a task description.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import (
validate_schema, Schema,
)
from taskgraph.util.schema import Schema
from taskgraph.util.scriptworker import (
get_beetmover_bucket_scope, add_scope_prefix,
get_worker_type_for_scope,
@ -24,7 +22,6 @@ from voluptuous import Any, Required, Optional
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
@ -47,14 +44,8 @@ beetmover_push_to_release_description_schema = Schema({
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job['name']
validate_schema(
beetmover_push_to_release_description_schema, job,
"In beetmover-push-to-release ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(beetmover_push_to_release_description_schema)
@transforms.add

14
taskcluster/taskgraph/transforms/beetmover_repackage.py
View File

@ -14,7 +14,6 @@ from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.partials import (get_balrog_platform_name,
get_partials_artifacts,
get_partials_artifact_map)
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
@ -144,8 +143,6 @@ UPSTREAM_ARTIFACT_SIGNED_MSI_PATHS = [
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
@ -170,15 +167,8 @@ beetmover_description_schema = schema.extend({
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_description_schema, job,
"In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(beetmover_description_schema)
@transforms.add

15
taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
View File

@ -19,7 +19,6 @@ from taskgraph.util.partners import (
from taskgraph.util.schema import (
optionally_keyed_by,
resolve_keyed_by,
validate_schema,
)
from taskgraph.util.scriptworker import (
add_scope_prefix,
@ -40,8 +39,6 @@ logger = logging.getLogger(__name__)
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
@ -63,17 +60,9 @@ beetmover_description_schema = schema.extend({
Optional('shipping-product'): task_description_schema['shipping-product'],
})
transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_description_schema, job,
"In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms.add_validate(beetmover_description_schema)
@transforms.add

14
taskcluster/taskgraph/transforms/beetmover_source_checksums.py
View File

@ -11,7 +11,6 @@ from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
@ -22,8 +21,6 @@ from voluptuous import Any, Required, Optional
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
@ -37,15 +34,8 @@ beetmover_checksums_description_schema = schema.extend({
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_checksums_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(beetmover_checksums_description_schema)
@transforms.add

14
taskcluster/taskgraph/transforms/checksums_signing.py
View File

@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
get_signing_cert_scope,
get_worker_type_for_scope,
@ -23,8 +22,6 @@ from voluptuous import Any, Required, Optional
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
@ -37,15 +34,8 @@ checksums_signing_description_schema = schema.extend({
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
checksums_signing_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(checksums_signing_description_schema)
@transforms.add

13
taskcluster/taskgraph/transforms/diffoscope.py
View File

@ -11,7 +11,6 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import (
Schema,
validate_schema,
)
from taskgraph.util.taskcluster import get_artifact_path, get_artifact_url
from voluptuous import (
@ -20,8 +19,6 @@ from voluptuous import (
Required,
)
transforms = TransformSequence()
index_or_string = Any(
basestring,
{Required('index-search'): basestring},
@ -49,14 +46,8 @@ diff_description_schema = Schema({
Optional('extra-args'): basestring,
})
@transforms.add
def validate(config, tasks):
for task in tasks:
validate_schema(
diff_description_schema, task,
"In diff task {!r}:".format(task.get('name', 'unknown')))
yield task
transforms = TransformSequence()
transforms.add_validate(diff_description_schema)
@transforms.add

9
taskcluster/taskgraph/transforms/docker_image.py
View File

@ -17,7 +17,6 @@ from taskgraph.util.docker import (
from taskgraph.util.cached_tasks import add_optimization
from taskgraph.util.schema import (
Schema,
validate_schema,
)
from voluptuous import (
Optional,
@ -54,13 +53,7 @@ docker_image_schema = Schema({
})
@transforms.add
def validate(config, tasks):
for task in tasks:
validate_schema(
docker_image_schema, task,
"In docker image {!r}:".format(task.get('name', 'unknown')))
yield task
transforms.add_validate(docker_image_schema)
def order_image_tasks(config, tasks):

14
taskcluster/taskgraph/transforms/fetch.py
View File

@ -24,7 +24,6 @@ from ..util.cached_tasks import (
)
from ..util.schema import (
Schema,
validate_schema,
)
from ..util.treeherder import (
join_symbol,
@ -33,8 +32,6 @@ from ..util.treeherder import (
CACHE_TYPE = 'content.v1'
transforms = TransformSequence()
FETCH_SCHEMA = Schema({
# Name of the task.
Required('name'): basestring,
@ -77,15 +74,8 @@ FETCH_SCHEMA = Schema({
},
})
@transforms.add
def validate(config, jobs):
for job in jobs:
validate_schema(
FETCH_SCHEMA, job,
'In fetch task {!r}:'.format(job.get('name', 'unknown')))
yield job
transforms = TransformSequence()
transforms.add_validate(FETCH_SCHEMA)
@transforms.add

15
taskcluster/taskgraph/transforms/google_play_strings.py
View File

@ -9,20 +9,16 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from taskgraph.util.schema import resolve_keyed_by, Schema, validate_schema
from taskgraph.util.schema import resolve_keyed_by, Schema
from voluptuous import Required
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
google_play_description_schema = Schema({
Required('name'): basestring,
Required('label'): task_description_schema['label'],
Required('description'): task_description_schema['description'],
Required('job-from'): task_description_schema['job-from'],
Required('attributes'): task_description_schema['attributes'],
@ -34,15 +30,14 @@ google_play_description_schema = Schema({
Required('worker'): object,
})
transforms = TransformSequence()
transforms.add_validate(google_play_description_schema)
@transforms.add
def validate_jobs_schema(config, jobs):
def set_label(config, jobs):
for job in jobs:
job['label'] = job['name']
validate_schema(
google_play_description_schema, job,
"In GooglePlayStrings ({!r} kind) task for {!r}:".format(config.kind, job['label'])
)
yield job

9
taskcluster/taskgraph/transforms/job/__init__.py
View File

@ -109,14 +109,7 @@ job_description_schema = Schema({
})
transforms = TransformSequence()
@transforms.add
def validate(config, jobs):
for job in jobs:
validate_schema(job_description_schema, job,
"In job {!r}:".format(job.get('name', job.get('label'))))
yield job
transforms.add_validate(job_description_schema)
@transforms.add

15
taskcluster/taskgraph/transforms/l10n.py
View File

@ -16,7 +16,6 @@ from taskgraph.transforms.base import (
TransformSequence,
)
from taskgraph.util.schema import (
validate_schema,
optionally_keyed_by,
resolve_keyed_by,
)
@ -214,12 +213,7 @@ def copy_in_useful_magic(config, jobs):
yield job
@transforms.add
def validate_early(config, jobs):
for job in jobs:
validate_schema(l10n_description_schema, job,
"In job {!r}:".format(job.get('name', 'unknown')))
yield job
transforms.add_validate(l10n_description_schema)
@transforms.add
@ -375,12 +369,7 @@ def mh_options_replace_project(config, jobs):
yield job
@transforms.add
def validate_again(config, jobs):
for job in jobs:
validate_schema(l10n_description_schema, job,
"In job {!r}:".format(job.get('name', 'unknown')))
yield job
transforms.add_validate(l10n_description_schema)
@transforms.add

17
taskcluster/taskgraph/transforms/push_apk.py
View File

@ -11,15 +11,12 @@ import re
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema, validate_schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from taskgraph.util.scriptworker import get_push_apk_scope
from taskgraph.util.taskcluster import get_artifact_prefix
from voluptuous import Optional, Required
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
@ -51,16 +48,8 @@ REQUIRED_ARCHITECTURES = {
}
PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly')
@transforms.add
def validate_jobs_schema_transform_partial(config, jobs):
for job in jobs:
label = job.get('label', '?no-label?')
validate_schema(
push_apk_description_schema, job,
"In PushApk ({!r} kind) task for {!r}:".format(config.kind, label)
)
yield job
transforms = TransformSequence()
transforms.add_validate(push_apk_description_schema)
@transforms.add

11
taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py
View File

@ -11,7 +11,7 @@ from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, optionally_keyed_by, resolve_keyed_by
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope)
from taskgraph.transforms.task import task_description_schema
@ -63,14 +63,7 @@ def set_label(config, jobs):
yield job
@transforms.add
def validate(config, jobs):
for job in jobs:
validate_schema(
beetmover_description_schema, job,
"In beetmover ({!r} kind) task for {!r}:".format(config.kind, job['label'])
)
yield job
transforms.add_validate(beetmover_description_schema)
@transforms.add

14
taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
View File

@ -9,7 +9,6 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope,
@ -39,8 +38,6 @@ CHECKSUMS_SIGNING_ARTIFACTS = [
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
release_generate_checksums_beetmover_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'): basestring,
@ -57,15 +54,8 @@ release_generate_checksums_beetmover_schema = schema.extend({
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
release_generate_checksums_beetmover_schema, job,
"In ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(release_generate_checksums_beetmover_schema)
@transforms.add

14
taskcluster/taskgraph/transforms/release_generate_checksums_signing.py
View File

@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
get_signing_cert_scope,
get_worker_type_for_scope,
@ -24,8 +23,6 @@ from voluptuous import Required, Optional
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
release_generate_checksums_signing_schema = schema.extend({
Required('depname', default='release-generate-checksums'): basestring,
Optional('label'): basestring,
@ -34,15 +31,8 @@ release_generate_checksums_signing_schema = schema.extend({
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
release_generate_checksums_signing_schema, job,
"In ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(release_generate_checksums_signing_schema)
@transforms.add

16
taskcluster/taskgraph/transforms/release_sign_and_push_langpacks.py
View File

@ -10,19 +10,14 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, resolve_keyed_by, optionally_keyed_by
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required
transforms = TransformSequence()
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
langpack_sign_push_description_schema = schema.extend({
Required('label'): basestring,
Required('description'): basestring,
@ -51,14 +46,7 @@ def set_label(config, jobs):
yield job
@transforms.add
def validate(config, jobs):
for job in jobs:
validate_schema(
langpack_sign_push_description_schema, job,
'In sign-and-push-langpacks ({} kind) task for {}:'.format(config.kind, job['label'])
)
yield job
transforms.add_validate(langpack_sign_push_description_schema)
@transforms.add

18
taskcluster/taskgraph/transforms/release_snap_push.py
View File

@ -9,18 +9,14 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema, validate_schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from voluptuous import Optional, Required
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
push_snap_description_schema = Schema({
Required('name'): basestring,
Required('job-from'): task_description_schema['job-from'],
@ -36,16 +32,8 @@ push_snap_description_schema = Schema({
Optional('extra'): task_description_schema['extra'],
})
@transforms.add
def validate_jobs_schema_transform(config, jobs):
for job in jobs:
label = job.get('label', '?no-label?')
validate_schema(
push_snap_description_schema, job,
"In release_snap_push ({!r} kind) task for {!r}:".format(config.kind, label)
)
yield job
transforms = TransformSequence()
transforms.add_validate(push_snap_description_schema)
@transforms.add

14
taskcluster/taskgraph/transforms/repackage.py
View File

@ -13,7 +13,6 @@ from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import (
validate_schema,
optionally_keyed_by,
resolve_keyed_by,
)
@ -23,8 +22,6 @@ from taskgraph.util.workertypes import worker_type_implementation
from taskgraph.transforms.job import job_description_schema
from voluptuous import Any, Required, Optional
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
@ -156,15 +153,8 @@ MOZHARNESS_EXPANSIONS = [
'stub-installer-tag', 'sfx-stub', 'wsx-stub',
]
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
packaging_description_schema, job,
"In packaging ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(packaging_description_schema)
@transforms.add

15
taskcluster/taskgraph/transforms/repackage_partner.py
View File

@ -13,7 +13,6 @@ from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import (
validate_schema,
optionally_keyed_by,
resolve_keyed_by,
)
@ -25,8 +24,6 @@ from taskgraph.transforms.task import task_description_schema
from taskgraph.transforms.repackage import PACKAGE_FORMATS
from voluptuous import Any, Required, Optional
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
@ -75,17 +72,9 @@ packaging_description_schema = schema.extend({
}
})
transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
packaging_description_schema, job,
"In packaging ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms.add_validate(packaging_description_schema)
@transforms.add

12
taskcluster/taskgraph/transforms/repackage_routes.py
View File

@ -8,22 +8,10 @@ Add indexes to repackage kinds
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema
from taskgraph.transforms.job import job_description_schema
transforms = TransformSequence()
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job['label']
validate_schema(
job_description_schema, job,
"In repackage-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def add_indexes(config, jobs):
for job in jobs:

14
taskcluster/taskgraph/transforms/repackage_signing.py
View File

@ -12,7 +12,6 @@ import os
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
add_scope_prefix,
get_signing_cert_scope_per_platform,
@ -25,8 +24,6 @@ from voluptuous import Required, Optional
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
repackage_signing_description_schema = schema.extend({
Required('depname', default='repackage'): basestring,
Optional('label'): basestring,
@ -43,15 +40,8 @@ SIGNING_FORMATS = {
"target.installer.msi": ["sha2signcode"],
}
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
repackage_signing_description_schema, job,
"In repackage-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(repackage_signing_description_schema)
@transforms.add

12
taskcluster/taskgraph/transforms/repackage_signing_partner.py
View File

@ -11,7 +11,6 @@ from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.partners import check_if_partners_enabled
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
add_scope_prefix,
get_signing_cert_scope_per_platform,
@ -35,16 +34,7 @@ repackage_signing_description_schema = schema.extend({
})
transforms.add(check_if_partners_enabled)
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
repackage_signing_description_schema, job,
"In repackage-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms.add_validate(repackage_signing_description_schema)
@transforms.add

10
taskcluster/taskgraph/transforms/signing.py
View File

@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
add_scope_prefix,
get_signing_cert_scope_per_platform,
@ -78,14 +77,7 @@ def set_defaults(config, jobs):
yield job
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
signing_description_schema, job,
"In signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms.add_validate(signing_description_schema)
@transforms.add

14
taskcluster/taskgraph/transforms/source_checksums_signing.py
View File

@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
get_signing_cert_scope,
get_worker_type_for_scope,
@ -23,8 +22,6 @@ from voluptuous import Any, Required, Optional
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
@ -37,15 +34,8 @@ checksums_signing_description_schema = schema.extend({
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
checksums_signing_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
transforms = TransformSequence()
transforms.add_validate(checksums_signing_description_schema)
@transforms.add

9
taskcluster/taskgraph/transforms/source_test.py
View File

@ -1,4 +1,3 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
@ -16,7 +15,6 @@ from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.job import job_description_schema
from taskgraph.util.attributes import keymatch
from taskgraph.util.schema import (
validate_schema,
resolve_keyed_by,
)
from taskgraph.util.treeherder import join_symbol, split_symbol
@ -72,12 +70,7 @@ def set_defaults(config, jobs):
yield job
@transforms.add
def validate(config, jobs):
for job in jobs:
validate_schema(source_test_description_schema, job,
"In job {!r}:".format(job['name']))
yield job
transforms.add_validate(source_test_description_schema)
@transforms.add

8
taskcluster/taskgraph/transforms/tests.py
View File

@ -25,7 +25,6 @@ from taskgraph.util.schema import resolve_keyed_by, OptimizationSchema
from taskgraph.util.treeherder import split_symbol, join_symbol, add_suffix
from taskgraph.util.platforms import platform_family
from taskgraph.util.schema import (
validate_schema,
optionally_keyed_by,
Schema,
)
@ -481,12 +480,7 @@ def set_defaults(config, tests):
yield test
@transforms.add
def validate(config, tests):
for test in tests:
validate_schema(test_description_schema, test,
"In test {!r}:".format(test['test-name']))
yield test
transforms.add_validate(test_description_schema)
@transforms.add