Bug 1557269 - part 2: GPG-sign geckoview artifacts r=mtabara

Differential Revision: https://phabricator.services.mozilla.com/D34866

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Johan Lorenzo 2019-07-01 14:01:26 +00:00
parent 45b8110131
commit f3a9b736d3
12 changed files with 199 additions and 110 deletions

View File

@ -10,7 +10,7 @@ transforms:
- taskgraph.transforms.task:transforms
kind-dependencies:
- build # geckoview builds aren't signed
- build-signing
only-for-attributes:
- nightly
@ -40,7 +40,6 @@ not-for-build-platforms:
job-template:
attributes:
artifact_map: taskcluster/taskgraph/manifests/fennec_geckoview.yml
artifact_prefix: public/build/maven
run-on-projects: ['mozilla-central', 'mozilla-release']
run-on-hg-branches:
by-project:

View File

@ -3,6 +3,8 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
job-defaults:
attributes:
artifact_map: taskcluster/taskgraph/manifests/fennec_geckoview.yml
index:
product: mobile
worker:

View File

@ -8,66 +8,96 @@ default_locales: # Ignored for geckoview
- en-US
tasktype_map: # Map task reference to task type.
build: build
base_artifact_prefix: public/build/maven/
build-signing: signing
# A default entry, which the mappings below extend and override.
# Final 'destinations' will be the product of:
# s3_bucket_paths + destinations + locale_prefix + pretty_name
default: &default
from:
- build
locale_prefix: ''
source_path_modifier: org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date}
source_path_modifier: maven/org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date}
description: "TO_BE_OVERRIDDEN"
destinations: # locale_prefix is appended
- org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date}
mapping:
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom
${artifact_id}-${major_version}.${minor_version}.${build_date}.aar:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar
${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5:
${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.asc:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5
from: ['build-signing']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.asc
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.asc
${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.asc:
<<: *default
from: ['build-signing']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.asc
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.asc
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar
${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.asc:
<<: *default
from: ['build-signing']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.asc
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.asc
${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha1:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar
${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.asc:
<<: *default
from: ['build-signing']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.asc
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.asc
${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1:
<<: *default
from: ['build']
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1

View File

@ -7,37 +7,25 @@ Transform the beetmover task into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
import re
from copy import deepcopy
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import \
craft_release_properties as beetmover_craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.declarative_artifacts import (
get_geckoview_template_vars,
get_geckoview_upstream_artifacts,
get_geckoview_artifact_id,
)
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
generate_beetmover_upstream_artifacts,
get_worker_type_for_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
_ARTIFACT_ID_PER_PLATFORM = {
'android-aarch64': 'geckoview{update_channel}-arm64-v8a',
'android-api-16': 'geckoview{update_channel}-armeabi-v7a',
'android-x86': 'geckoview{update_channel}-x86',
'android-x86_64': 'geckoview{update_channel}-x86_64',
'android-geckoview-fat-aar': 'geckoview{update_channel}',
}
_MOZ_UPDATE_CHANNEL_PER_BRANCH = {
'mozilla-release': '',
'mozilla-beta': '-beta',
'mozilla-central': '-nightly',
'try': '-nightly-try',
'maple': '-nightly-maple',
}
beetmover_description_schema = schema.extend({
Required('depname', default='build'): basestring,
Optional('label'): basestring,
@ -97,7 +85,8 @@ def make_task_description(config, jobs):
)
)
dependencies = {dep_job.kind: dep_job.label}
dependencies = deepcopy(dep_job.dependencies)
dependencies[dep_job.kind] = dep_job.label
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
@ -125,54 +114,35 @@ def make_task_description(config, jobs):
@transforms.add
def make_task_worker(config, jobs):
for job in jobs:
valid_beetmover_job = len(job['dependencies']) == 1 and 'build' in job['dependencies']
valid_beetmover_job = set(job['dependencies'].keys()) == {'build', 'build-signing'}
if not valid_beetmover_job:
raise NotImplementedError(
'Beetmover-geckoview must have a single dependency. Got: {}'.format(
'Beetmover-geckoview must have 2 dependencies: build and build-signing. '
'Got: {}'.format(
job['dependencies']
)
)
worker = {
job['worker'] = {
'artifact-map': generate_beetmover_artifact_map(
config,
job,
**get_geckoview_template_vars(config, job['attributes']['build_platform'])
),
'implementation': 'beetmover-maven',
'release-properties': craft_release_properties(config, job),
'upstream-artifacts': get_geckoview_upstream_artifacts(config, job),
}
version_groups = re.match(r'(\d+).(\d+).*', config.params['version'])
if version_groups:
major_version, minor_version = version_groups.groups()
template_vars = {
'artifact_id': worker['release-properties']['artifact-id'],
'build_date': config.params['moz_build_date'],
'major_version': major_version,
'minor_version': minor_version,
}
worker['artifact-map'] = generate_beetmover_artifact_map(
config, job, **template_vars
)
upstream_artifacts = generate_beetmover_upstream_artifacts(
config, job, platform='', **template_vars
)
worker['upstream-artifacts'] = [{
key: value for key, value in upstream_artifact.items()
if key != 'locale'
} for upstream_artifact in upstream_artifacts]
job["worker"] = worker
yield job
def craft_release_properties(config, job):
props = beetmover_craft_release_properties(config, job)
release_properties = beetmover_craft_release_properties(config, job)
platform = props['platform']
update_channel = _MOZ_UPDATE_CHANNEL_PER_BRANCH.get(
props['branch'], '-UNKNOWN_MOZ_UPDATE_CHANNEL'
release_properties['artifact-id'] = get_geckoview_artifact_id(
job['attributes']['build_platform'], release_properties['branch']
)
artifact_id = _ARTIFACT_ID_PER_PLATFORM[platform].format(update_channel=update_channel)
props['artifact-id'] = artifact_id
props['app-name'] = 'geckoview' # this beetmover job is not about pushing Fennec
release_properties['app-name'] = 'geckoview'
return props
return release_properties

View File

@ -8,6 +8,7 @@ Transform the signing task into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.signed_artifacts import generate_specifications_of_artifacts_to_sign
from taskgraph.util.taskcluster import get_artifact_path
@ -50,8 +51,11 @@ def define_upstream_artifacts(config, jobs):
dep_job = job['primary-dependency']
build_platform = dep_job.attributes.get('build_platform')
job['attributes'] = copy_attributes_from_dependent_job(dep_job)
artifacts_specifications = generate_specifications_of_artifacts_to_sign(
dep_job,
config,
job,
keep_locale_template=False,
kind=config.kind,
)

View File

@ -8,6 +8,7 @@ Transform the signing task into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.signed_artifacts import generate_specifications_of_artifacts_to_sign
from taskgraph.util.treeherder import join_symbol
@ -37,8 +38,11 @@ def define_upstream_artifacts(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
job['attributes'] = copy_attributes_from_dependent_job(dep_job)
locale_specifications = generate_specifications_of_artifacts_to_sign(
dep_job,
config,
job,
keep_locale_template=True,
)

View File

@ -26,6 +26,7 @@ def define_upstream_artifacts(config, jobs):
dep_job = job['primary-dependency']
repack_id = job['extra']['repack_id']
artifacts_specifications = generate_specifications_of_artifacts_to_sign(
config,
dep_job,
keep_locale_template=True,
kind=config.kind,

View File

@ -42,6 +42,9 @@ signing_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname'): basestring,
# attributes for this task
Optional('attributes'): {basestring: object},
# unique label to describe this signing task, defaults to {dep.label}-signing
Optional('label'): basestring,
@ -137,7 +140,8 @@ def make_task_description(config, jobs):
)
)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes = job['attributes'] if job.get('attributes') else \
copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):

View File

@ -35,6 +35,7 @@ RELEASE_PROMOTION_PROJECTS = {
} | RELEASE_PROJECTS
_OPTIONAL_ATTRIBUTES = (
'artifact_map',
'artifact_prefix',
'l10n_chunk',
'locale',

View File

@ -0,0 +1,51 @@
from __future__ import absolute_import, unicode_literals
import re
from taskgraph.util.scriptworker import generate_beetmover_upstream_artifacts
_ARTIFACT_ID_PER_PLATFORM = {
'android-aarch64-nightly': 'geckoview{update_channel}-arm64-v8a',
'android-api-16-nightly': 'geckoview{update_channel}-armeabi-v7a',
'android-x86-nightly': 'geckoview{update_channel}-x86',
'android-x86_64-nightly': 'geckoview{update_channel}-x86_64',
'android-geckoview-fat-aar-nightly': 'geckoview{update_channel}',
}
_MOZ_UPDATE_CHANNEL_PER_PROJECT = {
'mozilla-release': '',
'mozilla-beta': '-beta',
'mozilla-central': '-nightly',
'try': '-nightly-try',
'maple': '-nightly-maple',
}
def get_geckoview_upstream_artifacts(config, job):
upstream_artifacts = generate_beetmover_upstream_artifacts(
config, job, platform='',
**get_geckoview_template_vars(config, job['attributes']['build_platform'])
)
return [{
key: value for key, value in upstream_artifact.items()
if key != 'locale'
} for upstream_artifact in upstream_artifacts]
def get_geckoview_template_vars(config, platform):
version_groups = re.match(r'(\d+).(\d+).*', config.params['version'])
if version_groups:
major_version, minor_version = version_groups.groups()
return {
'artifact_id': get_geckoview_artifact_id(platform, config.params['project']),
'build_date': config.params['moz_build_date'],
'major_version': major_version,
'minor_version': minor_version,
}
def get_geckoview_artifact_id(platform, project):
update_channel = _MOZ_UPDATE_CHANNEL_PER_PROJECT.get(project, '-UNKNOWN_MOZ_UPDATE_CHANNEL')
return _ARTIFACT_ID_PER_PLATFORM[platform].format(update_channel=update_channel)

View File

@ -445,7 +445,12 @@ def generate_beetmover_upstream_artifacts(
locales = [locale]
if not dependencies:
dependencies = job['dependencies'].keys()
if job.get('dependencies'):
dependencies = job['dependencies'].keys()
elif job.get('primary-dependency'):
dependencies = [job['primary-dependency'].kind]
else:
raise Exception('Unsupported type of dependency. Got job: {}'.format(job))
for locale, dep in itertools.product(locales, dependencies):
paths = list()
@ -476,10 +481,14 @@ def generate_beetmover_upstream_artifacts(
jsone.render(filename, kwargs),
))
if getattr(job['dependencies'][dep], 'release_artifacts', None):
if (
job.get('dependencies') and
getattr(job['dependencies'][dep], 'release_artifacts', None)
):
paths = [
path for path in paths
if path in job['dependencies'][dep].release_artifacts]
if path in job['dependencies'][dep].release_artifacts
]
if not paths:
continue

View File

@ -7,6 +7,7 @@ Defines artifacts to sign before repackage.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.util.taskcluster import get_artifact_path
from taskgraph.util.declarative_artifacts import get_geckoview_upstream_artifacts
def is_partner_kind(kind):
@ -15,23 +16,26 @@ def is_partner_kind(kind):
def generate_specifications_of_artifacts_to_sign(
task, keep_locale_template=True, kind=None
config, job, keep_locale_template=True, kind=None
):
build_platform = task.attributes.get('build_platform')
use_stub = task.attributes.get('stub-installer')
build_platform = job['attributes'].get('build_platform')
use_stub = job['attributes'].get('stub-installer')
if kind == 'release-source-signing':
artifacts_specifications = [{
'artifacts': [
get_artifact_path(task, 'source.tar.xz')
get_artifact_path(job, 'source.tar.xz')
],
'formats': ['autograph_gpg'],
}]
elif 'android' in build_platform:
artifacts_specifications = [{
'artifacts': [
get_artifact_path(task, '{locale}/target.apk'),
get_artifact_path(job, '{locale}/target.apk'),
],
'formats': ['autograph_apk_fennec_sha1'],
}, {
'artifacts': get_geckoview_artifacts_to_sign(config, job),
'formats': ['autograph_gpg'],
}]
# XXX: Mars aren't signed here (on any platform) because internals will be
# signed at after this stage of the release
@ -41,29 +45,29 @@ def generate_specifications_of_artifacts_to_sign(
else:
extension = 'dmg'
artifacts_specifications = [{
'artifacts': [get_artifact_path(task, '{{locale}}/target.{}'.format(extension))],
'artifacts': [get_artifact_path(job, '{{locale}}/target.{}'.format(extension))],
'formats': ['macapp', 'autograph_widevine', 'autograph_omnija'],
}]
elif 'win' in build_platform:
artifacts_specifications = [{
'artifacts': [
get_artifact_path(task, '{locale}/setup.exe'),
get_artifact_path(job, '{locale}/setup.exe'),
],
'formats': ['sha2signcode'],
}, {
'artifacts': [
get_artifact_path(task, '{locale}/target.zip'),
get_artifact_path(job, '{locale}/target.zip'),
],
'formats': ['sha2signcode', 'autograph_widevine', 'autograph_omnija'],
}]
if use_stub:
artifacts_specifications[0]['artifacts'] += [
get_artifact_path(task, '{locale}/setup-stub.exe')
get_artifact_path(job, '{locale}/setup-stub.exe')
]
elif 'linux' in build_platform:
artifacts_specifications = [{
'artifacts': [get_artifact_path(task, '{locale}/target.tar.bz2')],
'artifacts': [get_artifact_path(job, '{locale}/target.tar.bz2')],
'formats': ['autograph_gpg', 'autograph_widevine', 'autograph_omnija'],
}]
else:
@ -117,3 +121,13 @@ def get_signed_artifacts(input, formats, behavior=None):
artifacts.add('{}.asc'.format(input))
return artifacts
def get_geckoview_artifacts_to_sign(config, job):
upstream_artifacts = get_geckoview_upstream_artifacts(config, job)
return [
path
for upstream_artifact in upstream_artifacts
for path in upstream_artifact['paths']
if not path.endswith('.md5') and not path.endswith('.sha1')
]