mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-16 06:45:42 +00:00
Bug 1373326 - Reformat repackage tasks taskgraph to support new input style. r=dustin
(For Landing more OSX Nightly Support from date to central) MozReview-Commit-ID: F8Dl1502D4L --HG-- extra : rebase_source : 0507e659995a25bfa90995476ec5e24751dae57f
This commit is contained in:
Justin Wood
parent
28aa07b4c1
commit
f0c41b41cb
@ -7,6 +7,7 @@ loader: taskgraph.loader.single_dep:loader
|
||||
transforms:
|
||||
- taskgraph.transforms.repackage_l10n:transforms
|
||||
- taskgraph.transforms.repackage:transforms
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
|
||||
@ -6,6 +6,7 @@ loader: taskgraph.loader.single_dep:loader
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.repackage:transforms
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
|
||||
@ -95,7 +95,7 @@ transforms = TransformSequence()
|
||||
def validate(config, jobs):
|
||||
for job in jobs:
|
||||
yield validate_schema(job_description_schema, job,
|
||||
"In job {!r}:".format(job['name']))
|
||||
"In job {!r}:".format(job.get('name', job.get('label'))))
|
||||
|
||||
|
||||
@transforms.add
|
||||
@ -133,7 +133,7 @@ def make_task_description(config, jobs):
|
||||
if 'name' not in job:
|
||||
raise Exception("job has neither a name nor a label")
|
||||
job['label'] = '{}-{}'.format(config.kind, job['name'])
|
||||
if job['name']:
|
||||
if job.get('name'):
|
||||
del job['name']
|
||||
|
||||
impl, os = worker_type_implementation(job['worker-type'])
|
||||
|
||||
@ -12,9 +12,12 @@ from __future__ import absolute_import, print_function, unicode_literals
|
||||
SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
|
||||
|
||||
|
||||
def docker_worker_add_workspace_cache(config, job, taskdesc):
|
||||
def docker_worker_add_workspace_cache(config, job, taskdesc, extra=None):
|
||||
"""Add the workspace cache based on the build platform/type and level,
|
||||
except on try where workspace caches are not used."""
|
||||
except on try where workspace caches are not used.
|
||||
|
||||
extra, is an optional kwarg passed in that supports extending the cache
|
||||
key name to avoid undesired conflicts with other caches."""
|
||||
if config.params['project'] == 'try':
|
||||
return
|
||||
|
||||
@ -27,6 +30,10 @@ def docker_worker_add_workspace_cache(config, job, taskdesc):
|
||||
),
|
||||
'mount-point': "/home/worker/workspace",
|
||||
})
|
||||
if extra:
|
||||
taskdesc['worker']['caches'][-1]['name'] += '-{}'.format(
|
||||
extra
|
||||
)
|
||||
|
||||
|
||||
def docker_worker_add_tc_vcs_cache(config, job, taskdesc):
|
||||
|
||||
@ -49,6 +49,10 @@ mozharness_run_schema = Schema({
|
||||
# --custom-build-variant-cfg value (not supported on Windows)
|
||||
Optional('custom-build-variant-cfg'): basestring,
|
||||
|
||||
# Extra metadata to use toward the workspace caching.
|
||||
# Only supported on docker-worker
|
||||
Optional('extra-workspace-cache-key'): basestring,
|
||||
|
||||
# If not false, tooltool downloads will be enabled via relengAPIProxy
|
||||
# for either just public files, or all files. Not supported on Windows
|
||||
Required('tooltool-downloads', default=False): Any(
|
||||
@ -94,7 +98,8 @@ def mozharness_on_docker_worker_setup(config, job, taskdesc):
|
||||
worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
|
||||
|
||||
docker_worker_add_public_artifacts(config, job, taskdesc)
|
||||
docker_worker_add_workspace_cache(config, job, taskdesc)
|
||||
docker_worker_add_workspace_cache(config, job, taskdesc,
|
||||
extra=run.get('extra-workspace-cache-key'))
|
||||
support_vcs_checkout(config, job, taskdesc)
|
||||
|
||||
env = worker.setdefault('env', {})
|
||||
|
||||
@ -67,16 +67,18 @@ def make_repackage_description(config, jobs):
|
||||
label = job.get('label',
|
||||
dep_job.label.replace("signing-", "repackage-"))
|
||||
job['label'] = label
|
||||
cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
|
||||
cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
|
||||
|
||||
yield job
|
||||
|
||||
|
||||
@transforms.add
|
||||
def make_task_description(config, jobs):
|
||||
def make_job_description(config, jobs):
|
||||
for job in jobs:
|
||||
dep_job = job['dependent-task']
|
||||
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
|
||||
if len(dep_job.dependencies) > 1:
|
||||
raise NotImplementedError(
|
||||
"Can't repackage a signing task with multiple dependencies")
|
||||
|
||||
treeherder = job.get('treeherder', {})
|
||||
treeherder.setdefault('symbol', 'tc(Nr)')
|
||||
@ -85,88 +87,79 @@ def make_task_description(config, jobs):
|
||||
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
|
||||
treeherder.setdefault('tier', 1)
|
||||
treeherder.setdefault('kind', 'build')
|
||||
signing_task = None
|
||||
for dependency in dependencies.keys():
|
||||
if 'signing' in dependency:
|
||||
signing_task = dependency
|
||||
signing_task_ref = "<{}>".format(signing_task)
|
||||
|
||||
attributes = {
|
||||
'nightly': dep_job.attributes.get('nightly', False),
|
||||
'build_platform': dep_job.attributes.get('build_platform'),
|
||||
'build_type': dep_job.attributes.get('build_type'),
|
||||
}
|
||||
|
||||
command = ['/home/worker/bin/run-task',
|
||||
# Various caches/volumes are default owned by root:root.
|
||||
'--chown-recursive', '/home/worker/workspace',
|
||||
'--chown-recursive', '/home/worker/tooltool-cache',
|
||||
'--vcs-checkout', '/home/worker/workspace/build/src',
|
||||
'--tools-checkout', '/home/worker/workspace/build/tools',
|
||||
'--',
|
||||
'/home/worker/workspace/build/src/taskcluster/scripts/builder/repackage.sh'
|
||||
]
|
||||
|
||||
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
|
||||
if job.get('locale'):
|
||||
input_string = 'https://queue.taskcluster.net/v1/task/<nightly-l10n-signing>/' + \
|
||||
'artifacts/public/build/{}/target.tar.gz'
|
||||
input_string = input_string.format(job['locale'])
|
||||
else:
|
||||
input_string = 'https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/' + \
|
||||
'public/build/target.tar.gz'
|
||||
signed_input = {'task-reference': input_string}
|
||||
attributes['locale'] = job['locale']
|
||||
|
||||
level = config.params['level']
|
||||
|
||||
task_env = {}
|
||||
locale_output_path = ""
|
||||
if attributes['build_platform'].startswith('macosx'):
|
||||
if job.get('locale'):
|
||||
input_string = 'https://queue.taskcluster.net/v1/task/' + \
|
||||
'{}/artifacts/public/build/{}/target.tar.gz'
|
||||
input_string = input_string.format(signing_task_ref, job['locale'])
|
||||
locale_output_path = "{}/".format(job['locale'])
|
||||
else:
|
||||
input_string = 'https://queue.taskcluster.net/v1/task/' + \
|
||||
'{}/artifacts/public/build/target.tar.gz'.format(signing_task_ref)
|
||||
task_env.update(
|
||||
SIGNED_INPUT={'task-reference': input_string},
|
||||
)
|
||||
mozharness_config = ['repackage/osx_signed.py']
|
||||
output_files = [{
|
||||
'type': 'file',
|
||||
'path': '/home/worker/workspace/build/artifacts/target.dmg',
|
||||
'name': 'public/build/{}target.dmg'.format(locale_output_path),
|
||||
}]
|
||||
else:
|
||||
raise Exception("Unexpected build platform for repackage")
|
||||
|
||||
run = {
|
||||
'using': 'mozharness',
|
||||
'script': 'mozharness/scripts/repackage.py',
|
||||
'config': mozharness_config,
|
||||
'job-script': 'taskcluster/scripts/builder/repackage.sh',
|
||||
'actions': ['download_input', 'setup', 'repackage'],
|
||||
'extra-workspace-cache-key': 'repackage',
|
||||
}
|
||||
|
||||
if attributes["build_platform"].startswith('macosx'):
|
||||
worker = {
|
||||
'docker-image': {"in-tree": "desktop-build"},
|
||||
'artifacts': output_files,
|
||||
'env': task_env,
|
||||
'chain-of-trust': True,
|
||||
'max-run-time': 3600
|
||||
}
|
||||
run["tooltool-downloads"] = 'internal'
|
||||
worker_type = 'aws-provisioner-v1/gecko-%s-b-macosx64' % level
|
||||
cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
|
||||
cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
|
||||
|
||||
task = {
|
||||
'label': job['label'],
|
||||
'description': "{} Repackage".format(
|
||||
dep_job.task["metadata"]["description"]),
|
||||
'worker-type': 'aws-provisioner-v1/gecko-%s-b-macosx64' % level,
|
||||
'worker-type': worker_type,
|
||||
'dependencies': dependencies,
|
||||
'attributes': attributes,
|
||||
'run-on-projects': dep_job.attributes.get('run_on_projects'),
|
||||
'treeherder': treeherder,
|
||||
'routes': job.get('routes', []),
|
||||
'extra': job.get('extra', {}),
|
||||
'scopes':
|
||||
['docker-worker:relengapi-proxy:tooltool.download.internal',
|
||||
'secrets:get:project/taskcluster/gecko/hgfingerprint',
|
||||
'docker-worker:relengapi-proxy:tooltool.download.public',
|
||||
'project:releng:signing:format:dmg'],
|
||||
'worker': {'implementation': 'docker-worker',
|
||||
'os': 'linux',
|
||||
'docker-image': {"in-tree": "desktop-build"},
|
||||
'caches': [{
|
||||
'type': 'persistent',
|
||||
'name': 'tooltool-cache',
|
||||
'mount-point': '/home/worker/tooltool-cache',
|
||||
}, {
|
||||
'type': 'persistent',
|
||||
'name': 'level-%s-checkouts-v1' % level,
|
||||
'mount-point': '/home/worker/checkouts',
|
||||
}],
|
||||
'artifacts': [{
|
||||
'type': 'file',
|
||||
'path': '/home/worker/workspace/build/upload/target.dmg',
|
||||
'name': 'public/build/target.dmg',
|
||||
}],
|
||||
'env': {
|
||||
'SIGNED_INPUT': signed_input,
|
||||
'MOZ_BUILD_DATE': config.params['moz_build_date'],
|
||||
'MH_BUILD_POOL': 'taskcluster',
|
||||
'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
|
||||
'GECKO_HEAD_REV': config.params['head_rev'],
|
||||
'MH_BRANCH': config.params['project'],
|
||||
'MOZ_SCM_LEVEL': level,
|
||||
'MOZHARNESS_ACTIONS': 'download_input setup repackage',
|
||||
'NEED_XVFB': 'true',
|
||||
'GECKO_BASE_REPOSITORY': config.params['base_repository'],
|
||||
'TOOLTOOL_CACHE': '/home/worker/tooltool-cache',
|
||||
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
|
||||
'MOZHARNESS_CONFIG': 'repackage/osx_signed.py',
|
||||
'USE_SCCACHE': '1',
|
||||
'MOZHARNESS_SCRIPT': 'mozharness/scripts/repackage.py'
|
||||
},
|
||||
'command': command,
|
||||
'chain-of-trust': True,
|
||||
'relengapi-proxy': True,
|
||||
'max-run-time': 3600
|
||||
}
|
||||
'worker': worker,
|
||||
'run': run,
|
||||
}
|
||||
yield task
|
||||
|
||||
@ -17,15 +17,17 @@ def split_locales(config, jobs):
|
||||
for job in jobs:
|
||||
dep_job = job['dependent-task']
|
||||
for locale in dep_job.attributes.get('chunk_locales', []):
|
||||
job['locale'] = locale
|
||||
|
||||
label = dep_job.label.replace("signing-", "repackage-{}-".format(locale))
|
||||
label = "repackage-{}-{}/{}".format(locale,
|
||||
dep_job.attributes['build_platform'],
|
||||
dep_job.attributes['build_type'],)
|
||||
job['label'] = label
|
||||
|
||||
treeherder = job.get('treeherder', {})
|
||||
treeherder['symbol'] = 'tc-L10n-Rpk({})'.format(locale)
|
||||
job['treeherder'] = treeherder
|
||||
yield job
|
||||
|
||||
yield {
|
||||
'locale': locale,
|
||||
'label': label,
|
||||
'treeherder': treeherder,
|
||||
'dependent-task': dep_job,
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user