Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-25 13:51:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1430006: Allow specifying the scope prefix to be used for script workers; r=aki

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D382

--HG--
extra : rebase_source : d011bc53c290468c846c1c3ed77e083a43fda2fa
This commit is contained in:
Tom Prince 2018-01-12 01:08:54 -07:00
parent d2e34db9b3
commit 3a19777b1f
3 changed files with 121 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
taskcluster/ci/config.yml
View File

@ -98,6 +98,8 @@ try:
- 'sm-compacting-win32'
scriptworker:
# See additional configuration in taskcluster/taskgraph/util/scriptworker.py
scope-prefix: 'project:releng'
worker-types:
'scriptworker-prov-v1/signing-linux-v1':
- 'project:releng:signing:cert:release-signing'

2
taskcluster/taskgraph/config.py
View File

@ -27,6 +27,8 @@ graph_config_schema = Schema({
Required('ridealong-builds', default={}): {basestring: [basestring]},
},
Required('scriptworker'): {
# Prefix to add to scopes controlling scriptworkers
Required('scope-prefix'): basestring,
# Mapping of scriptworker types to scopes they accept
Required('worker-types'): {basestring: [basestring]}
},

194
taskcluster/taskgraph/util/scriptworker.py
View File

@ -55,9 +55,9 @@ SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
"""Map the signing scope aliases to the actual scopes.
"""
SIGNING_CERT_SCOPES = {
'all-release-branches': 'project:releng:signing:cert:release-signing',
'all-nightly-branches': 'project:releng:signing:cert:nightly-signing',
'default': 'project:releng:signing:cert:dep-signing',
'all-release-branches': 'signing:cert:release-signing',
'all-nightly-branches': 'signing:cert:nightly-signing',
'default': 'signing:cert:dep-signing',
}
DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
@ -67,8 +67,8 @@ DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
]]
DEVEDITION_SIGNING_CERT_SCOPES = {
'beta': 'project:releng:signing:cert:nightly-signing',
'default': 'project:releng:signing:cert:dep-signing',
'beta': 'signing:cert:nightly-signing',
'default': 'signing:cert:dep-signing',
}
"""Map beetmover scope aliases to sets of projects.
@ -93,7 +93,7 @@ Used for both `BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK` and `get_release_build_numb
BEETMOVER_CANDIDATES_TARGET_TASKS = set([
'promote_fennec',
'promote_firefox',
'promote_devedition'
'promote_devedition',
])
BEETMOVER_PUSH_TARGET_TASKS = set([
'push_fennec',
@ -130,24 +130,24 @@ BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK = [[
"""
BEETMOVER_BUCKET_SCOPES = {
'all-candidates-tasks': {
'all-release-branches': 'project:releng:beetmover:bucket:release',
'all-release-branches': 'beetmover:bucket:release',
},
'all-push-tasks': {
'all-release-branches': 'project:releng:beetmover:bucket:release',
'all-release-branches': 'beetmover:bucket:release',
},
'all-nightly-tasks': {
'all-nightly-branches': 'project:releng:beetmover:bucket:nightly',
'all-nightly-branches': 'beetmover:bucket:nightly',
},
'default': 'project:releng:beetmover:bucket:dep',
'default': 'beetmover:bucket:dep',
}
"""Map the beetmover tasks aliases to the actual action scopes.
"""
BEETMOVER_ACTION_SCOPES = {
'all-candidates-tasks': 'project:releng:beetmover:action:push-to-candidates',
'all-push-tasks': 'project:releng:beetmover:action:push-to-releases',
'all-nightly-tasks': 'project:releng:beetmover:action:push-to-nightly',
'default': 'project:releng:beetmover:action:push-to-staging',
'all-candidates-tasks': 'beetmover:action:push-to-candidates',
'all-push-tasks': 'beetmover:action:push-to-releases',
'all-nightly-tasks': 'beetmover:action:push-to-nightly',
'default': 'beetmover:action:push-to-staging',
}
@ -184,54 +184,54 @@ BALROG_SCOPE_ALIAS_TO_PROJECT = [[
"""Map the balrog scope aliases to the actual scopes.
"""
BALROG_SERVER_SCOPES = {
'nightly': 'project:releng:balrog:server:nightly',
'aurora': 'project:releng:balrog:server:aurora',
'beta': 'project:releng:balrog:server:beta',
'release': 'project:releng:balrog:server:release',
'esr': 'project:releng:balrog:server:esr',
'default': 'project:releng:balrog:server:dep',
'nightly': 'balrog:server:nightly',
'aurora': 'balrog:server:aurora',
'beta': 'balrog:server:beta',
'release': 'balrog:server:release',
'esr': 'balrog:server:esr',
'default': 'balrog:server:dep',
}
"""Map the balrog scope aliases to the actual channel scopes.
"""
BALROG_CHANNEL_SCOPES = {
'nightly': [
'project:releng:balrog:channel:nightly',
'project:releng:balrog:channel:nightly-old-id',
'project:releng:balrog:channel:aurora'
'balrog:channel:nightly',
'balrog:channel:nightly-old-id',
'balrog:channel:aurora',
],
'aurora': [
'project:releng:balrog:channel:aurora'
'balrog:channel:aurora',
],
'beta': [
'project:releng:balrog:channel:beta',
'project:releng:balrog:channel:beta-localtest',
'project:releng:balrog:channel:beta-cdntest'
'balrog:channel:beta',
'balrog:channel:beta-localtest',
'balrog:channel:beta-cdntest',
],
'release': [
'project:releng:balrog:channel:release',
'project:releng:balrog:channel:release-localtest',
'project:releng:balrog:channel:release-cdntest'
'balrog:channel:release',
'balrog:channel:release-localtest',
'balrog:channel:release-cdntest',
],
'esr': [
'project:releng:balrog:channel:esr',
'project:releng:balrog:channel:esr-localtest',
'project:releng:balrog:channel:esr-cdntest'
'balrog:channel:esr',
'balrog:channel:esr-localtest',
'balrog:channel:esr-cdntest',
],
'default': [
'project:releng:balrog:channel:nightly',
'project:releng:balrog:channel:nightly-old-id',
'project:releng:balrog:channel:aurora'
'project:releng:balrog:channel:beta',
'project:releng:balrog:channel:beta-localtest',
'project:releng:balrog:channel:beta-cdntest',
'project:releng:balrog:channel:release',
'project:releng:balrog:channel:release-localtest',
'project:releng:balrog:channel:release-cdntest',
'project:releng:balrog:channel:esr',
'project:releng:balrog:channel:esr-localtest',
'project:releng:balrog:channel:esr-cdntest'
]
'balrog:channel:nightly',
'balrog:channel:nightly-old-id',
'balrog:channel:aurora',
'balrog:channel:beta',
'balrog:channel:beta-localtest',
'balrog:channel:beta-cdntest',
'balrog:channel:release',
'balrog:channel:release-localtest',
'balrog:channel:release-cdntest',
'balrog:channel:esr',
'balrog:channel:esr-localtest',
'balrog:channel:esr-cdntest',
],
}
@ -251,26 +251,64 @@ PUSH_APK_SCOPE_ALIAS_TO_PROJECT = [[
PUSH_APK_SCOPES = {
'central': 'project:releng:googleplay:aurora',
'beta': 'project:releng:googleplay:beta',
'release': 'project:releng:googleplay:release',
'default': 'project:releng:googleplay:invalid',
'central': 'googleplay:aurora',
'beta': 'googleplay:beta',
'release': 'googleplay:release',
'default': 'googleplay:invalid',
}
def add_scope_prefix(config, scope):
"""
Prepends the scriptworker scope prefix from the :ref:`graph config
<taskgraph-graph-config>`.
Args:
config (TransformConfig): The configuration for the kind being transformed.
scope (string): The suffix of the scope
Returns:
string: the scope to use.
"""
return "{prefix}:{scope}".format(
prefix=config.graph_config['scriptworker']['scope-prefix'],
scope=scope,
)
def with_scope_prefix(f):
"""
Wraps a function, calling :py:func:`add_scope_prefix` on the result of
calling the wrapped function.
Args:
f (callable): A function that takes a ``config`` and some keyword
arguments, and returns a scope suffix.
Returns:
callable: the wrapped function
"""
@functools.wraps(f)
def wrapper(config, **kwargs):
scope_or_scopes = f(config, **kwargs)
if isinstance(scope_or_scopes, list):
return map(functools.partial(add_scope_prefix, config), scope_or_scopes)
else:
return add_scope_prefix(config, scope_or_scopes)
return wrapper
# scope functions {{{1
def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
@with_scope_prefix
def get_scope_from_project(config, alias_to_project_map, alias_to_scope_map):
"""Determine the restricted scope from `config.params['project']`.
Args:
config (TransformConfig): The configuration for the kind being transformed.
alias_to_project_map (list of lists): each list pair contains the
alias and the set of projects that match. This is ordered.
alias_to_scope_map (dict): the alias alias to scope
config (TransformConfig): The configuration for the kind being transformed.
Returns:
string: the scope to use.
@ -281,14 +319,15 @@ def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
return alias_to_scope_map['default']
def get_scope_from_target_method(alias_to_tasks_map, alias_to_scope_map, config):
@with_scope_prefix
def get_scope_from_target_method(config, alias_to_tasks_map, alias_to_scope_map):
"""Determine the restricted scope from `config.params['target_tasks_method']`.
Args:
config (TransformConfig): The configuration for the kind being transformed.
alias_to_tasks_map (list of lists): each list pair contains the
alias and the set of target methods that match. This is ordered.
alias_to_scope_map (dict): the alias alias to scope
config (TransformConfig): The configuration for the kind being transformed.
Returns:
string: the scope to use.
@ -299,8 +338,9 @@ def get_scope_from_target_method(alias_to_tasks_map, alias_to_scope_map, config)
return alias_to_scope_map['default']
def get_scope_from_target_method_and_project(alias_to_tasks_map, alias_to_project_map,
aliases_to_scope_map, config):
@with_scope_prefix
def get_scope_from_target_method_and_project(config, alias_to_tasks_map,
alias_to_project_map, aliases_to_scope_map):
"""Determine the restricted scope from both `target_tasks_method` and `project`.
On certain branches, we'll need differing restricted scopes based on
@ -309,12 +349,12 @@ def get_scope_from_target_method_and_project(alias_to_tasks_map, alias_to_projec
checks both.
Args:
config (TransformConfig): The configuration for the kind being transformed.
alias_to_tasks_map (list of lists): each list pair contains the
alias and the set of target methods that match. This is ordered.
alias_to_project_map (list of lists): each list pair contains the
alias and the set of projects that match. This is ordered.
aliases_to_scope_map (dict of dicts): the task alias to project alias to scope
config (TransformConfig): The configuration for the kind being transformed.
Returns:
string: the scope to use.
@ -329,14 +369,14 @@ def get_scope_from_target_method_and_project(alias_to_tasks_map, alias_to_projec
return aliases_to_scope_map['default']
def get_phase_from_target_method(alias_to_tasks_map, alias_to_phase_map, config):
def get_phase_from_target_method(config, alias_to_tasks_map, alias_to_phase_map):
"""Determine the phase from `config.params['target_tasks_method']`.
Args:
config (TransformConfig): The configuration for the kind being transformed.
alias_to_tasks_map (list of lists): each list pair contains the
alias and the set of target methods that match. This is ordered.
alias_to_phase_map (dict): the alias to phase map
config (TransformConfig): The configuration for the kind being transformed.
Returns:
string: the phase to use.
@ -349,51 +389,51 @@ def get_phase_from_target_method(alias_to_tasks_map, alias_to_phase_map, config)
get_signing_cert_scope = functools.partial(
get_scope_from_project,
SIGNING_SCOPE_ALIAS_TO_PROJECT,
SIGNING_CERT_SCOPES
alias_to_project_map=SIGNING_SCOPE_ALIAS_TO_PROJECT,
alias_to_scope_map=SIGNING_CERT_SCOPES,
)
get_devedition_signing_cert_scope = functools.partial(
get_scope_from_project,
DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT,
DEVEDITION_SIGNING_CERT_SCOPES
alias_to_project_map=DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT,
alias_to_scope_map=DEVEDITION_SIGNING_CERT_SCOPES,
)
get_beetmover_bucket_scope = functools.partial(
get_scope_from_target_method_and_project,
BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
BEETMOVER_BUCKET_SCOPES
alias_to_tasks_map=BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
aliases_to_scope_map=BEETMOVER_BUCKET_SCOPES,
)
get_beetmover_action_scope = functools.partial(
get_scope_from_target_method,
BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
BEETMOVER_ACTION_SCOPES
alias_to_tasks_map=BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
alias_to_scope_map=BEETMOVER_ACTION_SCOPES,
)
get_phase = functools.partial(
get_phase_from_target_method,
BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
PHASES
alias_to_tasks_map=BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
alias_to_phase_map=PHASES,
)
get_balrog_server_scope = functools.partial(
get_scope_from_project,
BALROG_SCOPE_ALIAS_TO_PROJECT,
BALROG_SERVER_SCOPES
alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT,
alias_to_scope_map=BALROG_SERVER_SCOPES,
)
get_balrog_channel_scopes = functools.partial(
get_scope_from_project,
BALROG_SCOPE_ALIAS_TO_PROJECT,
BALROG_CHANNEL_SCOPES
alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT,
alias_to_scope_map=BALROG_CHANNEL_SCOPES,
)
get_push_apk_scope = functools.partial(
get_scope_from_project,
PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
PUSH_APK_SCOPES
alias_to_project_map=PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
alias_to_scope_map=PUSH_APK_SCOPES,
)
@ -451,7 +491,7 @@ def get_signing_cert_scope_per_platform(build_platform, is_nightly, config):
elif is_nightly:
return get_signing_cert_scope(config)
else:
return 'project:releng:signing:cert:dep-signing'
return add_scope_prefix(config, 'signing:cert:dep-signing')
def get_worker_type_for_scope(config, scope):