Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-24 05:11:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1466714 - declarative artifacts in-tree work for Fennec. r=mtabara

Browse Source
This commit is contained in:
Simon Fraser 2018-11-28 11:13:52 +00:00
parent 80c35b4119
commit 9582b26afc
23 changed files with 940 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
taskcluster/ci/beetmover-checksums/kind.yml
View File

@ -17,3 +17,9 @@ only-for-attributes:
job-template:
shipping-phase: promote
attributes:
artifact_prefix: public
artifact_map:
by-platform:
android.*: taskcluster/taskgraph/manifests/fennec_nightly_checksums.yml
default: taskcluster/taskgraph/manifests/firefox_nightly_checksums.yml

2
taskcluster/ci/beetmover-geckoview/kind.yml
View File

@ -30,6 +30,8 @@ not-for-build-platforms:
- win64-asan-reporter-nightly/opt
job-template:
attributes:
artifact_map: taskcluster/taskgraph/manifests/fennec_geckoview.yml
run-on-projects: ['mozilla-central', 'mozilla-release']
run-on-hg-branches:
by-project:

2
taskcluster/ci/beetmover-l10n/kind.yml
View File

@ -30,3 +30,5 @@ not-for-build-platforms:
job-template:
shipping-phase: promote
attributes:
artifact_map: taskcluster/taskgraph/manifests/fennec_nightly.yml

2
taskcluster/ci/beetmover-source/kind.yml
View File

@ -15,3 +15,5 @@ kind-dependencies:
job-template:
shipping-phase: promote
attributes:
artifact_map: taskcluster/taskgraph/manifests/source_files.yml

2
taskcluster/ci/beetmover/kind.yml
View File

@ -31,3 +31,5 @@ not-for-build-platforms:
job-template:
shipping-phase: promote
attributes:
artifact_map: taskcluster/taskgraph/manifests/fennec_nightly.yml

2
taskcluster/ci/release-beetmover-source-checksums/kind.yml
View File

@ -14,3 +14,5 @@ kind-dependencies:
job-template:
shipping-phase: promote
attributes:
artifact_map: taskcluster/taskgraph/manifests/source_checksums.yml

2
taskcluster/ci/release-generate-checksums-beetmover/kind.yml
View File

@ -13,3 +13,5 @@ kind-dependencies:
job-template:
shipping-phase: promote
attributes:
artifact_map: taskcluster/taskgraph/manifests/release_checksums.yml

5
taskcluster/docs/attributes.rst
View File

@ -244,6 +244,11 @@ lot of places. To support private artifacts, we've moved this to the
``artifact_prefix`` attribute. It will default to ``public/build`` but will be
overrideable per-task.
artifact_map
===============
For beetmover jobs, this indicates which yaml file should be used to
generate the upstream artifacts and payload instructions to the task.
enable-full-crashsymbols
========================
In automation, full crashsymbol package generation is normally disabled. For

83
taskcluster/taskgraph/manifests/fennec_geckoview.yml Normal file
View File

@ -0,0 +1,83 @@
---
metadata:
name: "Beetmover for checksums artifacts"
description: "Uploads checksum files to s3 buckets"
owner: "release@mozilla.com"
s3_bucket_paths:
- maven2
default_locales: # Ignored for geckoview
- en-US
tasktype_map: # Map task reference to task type.
build: build
base_artifact_prefix: ''
platform_names:
path_platform: ''
filename_platform: ''
# A default entry, which the mappings below extend and override.
# Final 'destinations' will be the product of:
# s3_bucket_paths + destinations + locale_prefix + pretty_name
default: &default
from:
- build
locale_prefix: ''
source_path_modifier: org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date}
description: "TODO"
destinations: # locale_prefix is appended
- org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date}
# Configuration for individual files. Extends 'default', above.
upstream_mapping:
target.maven.zip:
from:
- build
mapping:
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}.pom:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom
${artifact_id}-${major_version}.${minor_version}.${build_date}.aar:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar
${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5
${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha1:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha1
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha1
${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar:
<<: *default
pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar
checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar

153
taskcluster/taskgraph/manifests/fennec_nightly.yml Normal file
View File

@ -0,0 +1,153 @@
---
metadata:
description: Fennec nightly artifacts
name: Beetmover declarative artifacts manifest
owner: release@mozilla.com
s3_bucket_paths:
- pub/mobile/nightly
default_locales: # if given an empty locale, use these locales
- en-US
- multi
tasktype_map: # Map task reference to task type.
build: build
build-signing: signing
nightly-l10n-signing: signing
platform_names:
path_platform:
by-platform:
android-x86-nightly: 'android-x86'
android-x86_64-nightly: 'android-x86_64'
android-api-16-nightly: 'android-api-16'
android-aarch64-nightly: 'android-aarch64'
filename_platform:
by-platform:
android-x86-nightly: 'android-i386'
android-x86_64-nightly: 'android-x86_64'
android-api-16-nightly: 'android-arm'
android-aarch64-nightly: 'android-aarch64'
# A default entry, which the mappings below extend and override.
# Final 'destinations' will be the product of:
# s3_bucket_paths + destinations + locale_prefix + pretty_name
default: &default
from:
- build
all_locales: false
description: "TODO"
locale_prefix:
by-locale:
default: ''
en-US: '${locale}'
source_path_modifier: # Anything between artifact prefix and filename
by-locale:
default: '${locale}'
multi: ''
destinations: # locale_prefix is appended
by-locale:
multi:
- latest-${branch}-${path_platform}
- ${year}/${month}/${upload_date}-${branch}-${path_platform}
en-US:
- latest-${branch}-${path_platform}
- ${year}/${month}/${upload_date}-${branch}-${path_platform}
default:
- latest-${branch}-${path_platform}-l10n
- ${year}/${month}/${upload_date}-${branch}-${path_platform}-l10n
# Configuration for individual files. Extends 'default', above.
mapping:
buildhub.json:
<<: *default
all_locales: true
locale_prefix: ''
pretty_name: fennec-${version}.${locale}.${filename_platform}.buildhub.json
checksums_path: fennec-${version}.${locale}.${filename_platform}.buildhub.json
target.common.tests.tar.gz:
<<: *default
pretty_name: fennec-${version}.${locale}.${filename_platform}.common.tests.tar.gz
checksums_path: fennec-${version}.${locale}.${filename_platform}.common.tests.tar.gz
target.cppunittest.tests.tar.gz:
<<: *default
pretty_name: fennec-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz
checksums_path: fennec-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz
target.crashreporter-symbols.zip:
<<: *default
pretty_name: fennec-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip
checksums_path: fennec-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip
target.json:
<<: *default
all_locales: true
pretty_name: fennec-${version}.${locale}.${filename_platform}.json
checksums_path: fennec-${version}.${locale}.${filename_platform}.json
target.mochitest.tests.tar.gz:
<<: *default
pretty_name: fennec-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz
checksums_path: fennec-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz
target.mozinfo.json:
<<: *default
all_locales: true
pretty_name: fennec-${version}.${locale}.${filename_platform}.mozinfo.json
checksums_path: fennec-${version}.${locale}.${filename_platform}.mozinfo.json
target.reftest.tests.tar.gz:
<<: *default
pretty_name: fennec-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz
checksums_path: fennec-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz
target.talos.tests.tar.gz:
<<: *default
pretty_name: fennec-${version}.${locale}.${filename_platform}.talos.tests.tar.gz
checksums_path: fennec-${version}.${locale}.${filename_platform}.talos.tests.tar.gz
target.awsy.tests.tar.gz:
<<: *default
pretty_name: fennec-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz
checksums_path: fennec-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz
target.test_packages.json:
<<: *default
all_locales: true
pretty_name: fennec-${version}.${locale}.${filename_platform}.test_packages.json
checksums_path: fennec-${version}.${locale}.${filename_platform}.test_packages.json
target.txt:
<<: *default
all_locales: true
pretty_name: fennec-${version}.${locale}.${filename_platform}.txt
checksums_path: fennec-${version}.${locale}.${filename_platform}.txt
target.web-platform.tests.tar.gz:
<<: *default
description: "Web platform test suite <TODO>"
pretty_name: fennec-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz
checksums_path: fennec-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz
target.xpcshell.tests.tar.gz:
<<: *default
description: "XPCShell test suite <TODO>"
pretty_name: fennec-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz
checksums_path: fennec-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz
target_info.txt:
<<: *default
all_locales: true
description: "File containing the buildID for the particular release"
pretty_name: fennec-${version}.${locale}.${filename_platform}_info.txt
checksums_path: fennec-${version}.${locale}.${filename_platform}_info.txt
mozharness.zip:
<<: *default
pretty_name: mozharness.zip
checksums_path: mozharness.zip
robocop.apk:
<<: *default
all_locales: true
pretty_name: robocop.apk
checksums_path: robocop.apk
target.jsshell.zip:
<<: *default
pretty_name: jsshell-${filename_platform}.zip
checksums_path: jsshell-${filename_platform}.zip
target.apk:
<<: *default
all_locales: true
from:
- build-signing
- nightly-l10n-signing
pretty_name: fennec-${version}.${locale}.${filename_platform}.apk
checksums_path: fennec-${version}.${locale}.${filename_platform}.apk
update_balrog_manifest:
by-locale:
multi: true
default: false

60
taskcluster/taskgraph/manifests/fennec_nightly_checksums.yml Normal file
View File

@ -0,0 +1,60 @@
---
metadata:
name: "Beetmover for checksums artifacts"
description: "Uploads checksum files to s3 buckets"
owner: "release@mozilla.com"
s3_bucket_paths:
- pub/mobile/nightly
default_locales: # if given an empty locale, use these locales
- en-US
tasktype_map: # Map task reference to task type.
checksums-signing: signing
platform_names:
path_platform:
by-platform:
android-x86-nightly: 'android-x86'
android-x86_64-nightly: 'android-x86_64'
android-api-16-nightly: 'android-api-16'
android-aarch64-nightly: 'android-aarch64'
filename_platform:
by-platform:
android-x86-nightly: 'android-i386'
android-x86_64-nightly: 'android-x86_64'
android-api-16-nightly: 'android-arm'
android-aarch64-nightly: 'android-aarch64'
# A default entry, which the mappings below extend and override.
# Final 'destinations' will be the product of:
# s3_bucket_paths + destinations + locale_prefix + pretty_name
default: &default
from:
- checksums-signing
all_locales: true
description: "TODO"
locale_prefix:
by-locale:
default: ''
en-US: '${locale}'
source_path_modifier: ''
destinations: # locale_prefix is appended
by-locale:
multi:
- latest-${branch}-${path_platform}
- ${year}/${month}/${upload_date}-${branch}-${path_platform}
en-US:
- latest-${branch}-${path_platform}
- ${year}/${month}/${upload_date}-${branch}-${path_platform}
default:
- latest-${branch}-${path_platform}-l10n
- ${year}/${month}/${upload_date}-${branch}-${path_platform}-l10n
# Configuration for individual files. Extends 'default', above.
mapping:
target.checksums:
<<: *default
pretty_name: fennec-${version}.${locale}.${filename_platform}.checksums
checksums_path: fennec-${version}.${locale}.${filename_platform}.checksums
target.checksums.asc:
<<: *default
pretty_name: fennec-${version}.${locale}.${filename_platform}.checksums.asc
checksums_path: fennec-${version}.${locale}.${filename_platform}.checksums.asc

55
taskcluster/taskgraph/manifests/firefox_nightly_checksums.yml Normal file
View File

@ -0,0 +1,55 @@
---
metadata:
name: "Beetmover for checksums artifacts"
description: "Uploads checksum files to s3 buckets"
owner: "release@mozilla.com"
s3_bucket_paths:
by-platform:
.*devedition.*:
- pub/devedition/nightly
default:
- pub/firefox/nightly
default_locales: # if given an empty locale, use these locales
- en-US
tasktype_map: # Map task reference to task type.
checksums-signing: signing
platform_names:
path_platform: ''
filename_platform:
by-platform:
linux-.*: 'linux-i686'
linux64.*: 'linux-x86_64'
macosx64.*: 'mac'
win32.*: 'win32'
win64.*: 'win64'
# A default entry, which the mappings below extend and override.
# Final 'destinations' will be the product of:
# s3_bucket_paths + destinations + locale_prefix + pretty_name
default: &default
from:
- checksums-signing
all_locales: true
description: "TODO"
locale_prefix: ''
source_path_modifier: ''
destinations: # locale_prefix is appended
by-locale:
en-US:
- latest-${branch}
- latest-${branch}-l10n
- ${year}/${month}/${upload_date}-${branch}-${path_platform}
default:
- latest-${branch}-l10n
- ${year}/${month}/${upload_date}-${branch}-l10n
# Configuration for individual files. Extends 'default', above.
mapping:
target.checksums:
<<: *default
pretty_name: firefox-${version}.${locale}.${filename_platform}.checksums
checksums_path: firefox-${version}.${locale}.${filename_platform}.checksums
target.checksums.asc:
<<: *default
pretty_name: firefox-${version}.${locale}.${filename_platform}.checksums.asc
checksums_path: firefox-${version}.${locale}.${filename_platform}.checksums.asc

69
taskcluster/taskgraph/manifests/release_checksums.yml Normal file
View File

@ -0,0 +1,69 @@
---
metadata:
name: "Beetmover for checksums artifacts"
description: "Uploads checksum files to s3 buckets"
owner: "release@mozilla.com"
s3_bucket_paths:
by-platform:
fennec-release:
- pub/mobile/candidates
devedition-release:
- pub/devedition/candidates
firefox-release:
- pub/firefox/candidates
default_locales: # if given an empty locale, use these locales
- en-US
tasktype_map: # Map task reference to task type.
release-generate-checksums-signing: signing
release-generate-checksums: build
platform_names:
path_platform: ''
filename_platform: ''
# A default entry, which the mappings below extend and override.
# Final 'destinations' will be the product of:
# s3_bucket_paths + destinations + locale_prefix + pretty_name
default: &default
from:
- release-generate-checksums-signing
all_locales: true
description: "TODO"
locale_prefix: ''
source_path_modifier: ''
destinations: # locale_prefix is appended
- ${version}-candidates/build${build_number}
# Configuration for individual files. Extends 'default', above.
mapping:
SHA256SUMMARY:
<<: *default
from:
- release-generate-checksums
pretty_name: SHA256SUMMARY
checksums_path: SHA256SUMMARY
SHA512SUMMARY:
<<: *default
from:
- release-generate-checksums
pretty_name: SHA512SUMMARY
checksums_path: SHA512SUMMARY
KEY:
<<: *default
pretty_name: KEY
checksums_path: KEY
SHA256SUMS:
<<: *default
pretty_name: SHA256SUMS
checksums_path: SHA256SUMS
SHA256SUMS.asc:
<<: *default
pretty_name: SHA256SUMS.asc
checksums_path: SHA256SUMS.asc
SHA512SUMS:
<<: *default
pretty_name: SHA512SUMS
checksums_path: SHA512SUMS
SHA512SUMS.asc:
<<: *default
pretty_name: SHA512SUMS.asc
checksums_path: SHA512SUMS.asc

50
taskcluster/taskgraph/manifests/source_checksums.yml Normal file
View File

@ -0,0 +1,50 @@
---
metadata:
description: Source file archives
name: Beetmover source archive manifest
owner: release@mozilla.com
s3_bucket_paths:
by-platform:
fennec-source:
- pub/mobile/candidates
devedition-source:
- pub/devedition/candidates
firefox-source:
- pub/firefox/candidates
default_locales: # if given an empty locale, use these locales
- en-US
tasktype_map: # Map task reference to task type.
release-source-checksums-signing: signing
platform_names:
path_platform: ''
filename_platform: ''
# A default entry, which the mappings below extend and override.
# Final 'destinations' will be the product of:
# s3_bucket_paths + destinations + locale_prefix + pretty_name
default: &default
from:
- release-source-checksums-signing
all_locales: false
description: "TODO"
locale_prefix: ''
source_path_modifier: ''
destinations: # locale_prefix is appended
- ${version}-candidates/build${build_number}/beetmover-checksums/source
# Configuration for individual files. Extends 'default', above.
mapping:
target-source.checksums:
<<: *default
pretty_name:
by-platform:
firefox-source: firefox-${version}.checksums.beet
devedition-source: firefox-${version}.checksums.beet
fennec-source: fennec-${version}.checksums.beet
target-source.checksums.asc:
<<: *default
pretty_name:
by-platform:
firefox-source: firefox-${version}.checksums.asc
devedition-source: firefox-${version}.checksums.asc
fennec-source: fennec-${version}.checksums.asc

60
taskcluster/taskgraph/manifests/source_files.yml Normal file
View File

@ -0,0 +1,60 @@
---
metadata:
description: Source file archives
name: Beetmover source archive manifest
owner: release@mozilla.com
s3_bucket_paths:
by-platform:
fennec-source:
- pub/mobile/candidates
devedition-source:
- pub/devedition/candidates
firefox-source:
- pub/firefox/candidates
default_locales: # if given an empty locale, use these locales
- en-US
tasktype_map: # Map task reference to task type.
release-source-signing: signing
platform_names:
path_platform: ''
filename_platform: ''
# A default entry, which the mappings below extend and override.
# Final 'destinations' will be the product of:
# s3_bucket_paths + destinations + locale_prefix + pretty_name
default: &default
from:
- release-source-signing
all_locales: false
description: "TODO"
locale_prefix: ''
source_path_modifier: ''
destinations: # locale_prefix is appended
- ${version}-candidates/build${build_number}/source
# Configuration for individual files. Extends 'default', above.
mapping:
source.tar.xz:
<<: *default
pretty_name:
by-platform:
firefox-source: firefox-${version}.source.tar.xz
devedition-source: firefox-${version}.source.tar.xz
fennec-source: fennec-${version}.source.tar.xz
checksums_path:
by-platform:
firefox-source: firefox-${version}.source.tar.xz
devedition-source: firefox-${version}.source.tar.xz
fennec-source: fennec-${version}.source.tar.xz
source.tar.xz.asc:
<<: *default
pretty_name:
by-platform:
firefox-source: firefox-${version}.source.tar.xz.asc
devedition-source: firefox-${version}.source.tar.xz.asc
fennec-source: fennec-${version}.source.tar.xz.asc
checksums_path:
by-platform:
firefox-source: firefox-${version}.source.tar.xz.asc
devedition-source: firefox-${version}.source.tar.xz.asc
fennec-source: fennec-${version}.source.tar.xz.asc

37
taskcluster/taskgraph/transforms/beetmover.py
View File

@ -7,16 +7,19 @@ Transform the beetmover task into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
from voluptuous import Any, Optional, Required
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
generate_beetmover_upstream_artifacts,
get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope,
should_use_artifact_map)
from taskgraph.util.taskcluster import get_artifact_prefix
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
@ -138,6 +141,7 @@ beetmover_description_schema = schema.extend({
Required('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('attributes'): task_description_schema['attributes'],
})
@ -181,6 +185,7 @@ def make_task_description(config, jobs):
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
@ -253,7 +258,7 @@ def generate_upstream_artifacts(job, signing_task_ref, build_task_ref, platform,
"paths": ["{}/{}".format(artifact_prefix, p)
for p in build_mapping[multi_platform]],
"locale": "multi",
}, {
}, {
"taskId": {"task-reference": signing_task_ref},
"taskType": "signing",
"paths": ["{}/{}".format(artifact_prefix, p)
@ -311,14 +316,24 @@ def make_task_worker(config, jobs):
signing_task_ref = "<" + str(signing_task) + ">"
build_task_ref = "<" + str(build_task) + ">"
if should_use_artifact_map(platform, config.params['project']):
upstream_artifacts = generate_beetmover_upstream_artifacts(
job, platform, locale
)
else:
upstream_artifacts = generate_upstream_artifacts(
job, signing_task_ref, build_task_ref, platform, locale
)
worker = {
'implementation': 'beetmover',
'release-properties': craft_release_properties(config, job),
'upstream-artifacts': generate_upstream_artifacts(
job, signing_task_ref, build_task_ref, platform, locale
)
'upstream-artifacts': upstream_artifacts,
}
if should_use_artifact_map(platform, config.params['project']):
worker['artifact-map'] = generate_beetmover_artifact_map(
config, job, platform=platform, locale=locale)
if locale:
worker["locale"] = locale
job["worker"] = worker

31
taskcluster/taskgraph/transforms/beetmover_checksums.py
View File

@ -7,15 +7,19 @@ Transform the checksums signing task into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
generate_beetmover_upstream_artifacts,
get_beetmover_action_scope,
get_worker_type_for_scope)
get_beetmover_bucket_scope,
get_worker_type_for_scope,
should_use_artifact_map)
from voluptuous import Any, Optional, Required
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
@ -27,6 +31,7 @@ taskref_or_string = Any(
beetmover_checksums_description_schema = schema.extend({
Required('depname', default='build'): basestring,
Required('attributes'): {basestring: object},
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('locale'): basestring,
@ -80,6 +85,7 @@ def make_beetmover_checksums_description(config, jobs):
dependencies = {dependent_kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'BMcs({})'.format(dep_job.attributes.get('locale'))
@ -142,11 +148,24 @@ def make_beetmover_checksums_worker(config, jobs):
worker = {
'implementation': 'beetmover',
'release-properties': craft_release_properties(config, job),
'upstream-artifacts': generate_upstream_artifacts(
refs, platform, locale
),
}
if should_use_artifact_map(platform, config.params['project']):
upstream_artifacts = generate_beetmover_upstream_artifacts(
job, platform, locale
)
worker['artifact-map'] = generate_beetmover_artifact_map(
config, job, platform=platform, locale=locale)
else:
upstream_artifacts = generate_upstream_artifacts(
refs, platform, locale
)
# Clean up un-used artifact map, to avoid confusion
if job['attributes'].get('artifact_map'):
del job['attributes']['artifact_map']
worker['upstream-artifacts'] = upstream_artifacts
if locale:
worker["locale"] = locale
job["worker"] = worker

29
taskcluster/taskgraph/transforms/beetmover_geckoview.py
View File

@ -7,13 +7,17 @@ Transform the beetmover task into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
import re
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import \
craft_release_properties as beetmover_craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.util.scriptworker import get_worker_type_for_scope
from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
generate_beetmover_compressed_upstream_artifacts,
get_worker_type_for_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
@ -48,6 +52,7 @@ beetmover_description_schema = schema.extend({
'project', task_description_schema['shipping-phase']
),
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('attributes'): task_description_schema['attributes'],
})
transforms = TransformSequence()
@ -97,6 +102,7 @@ def make_task_description(config, jobs):
dependencies = {dependent_kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
@ -138,15 +144,28 @@ def make_task_worker(config, jobs):
)
)
build_task = list(job["dependencies"].keys())[0]
build_task_ref = "<" + str(build_task) + ">"
worker = {
'implementation': 'beetmover-maven',
'release-properties': craft_release_properties(config, job),
'upstream-artifacts': generate_upstream_artifacts(build_task_ref)
}
upstream_artifacts = generate_beetmover_compressed_upstream_artifacts(job)
worker['upstream-artifacts'] = upstream_artifacts
version_groups = re.match(r'(\d+).(\d+).*', config.params['version'])
if version_groups:
major_version, minor_version = version_groups.groups()
template_vars = {
'artifact_id': worker['release-properties']['artifact-id'],
'build_date': config.params['moz_build_date'],
'major_version': major_version,
'minor_version': minor_version,
}
worker['artifact-map'] = generate_beetmover_artifact_map(
config, job, **template_vars)
job["worker"] = worker
yield job

2
taskcluster/taskgraph/transforms/beetmover_l10n.py
View File

@ -33,5 +33,7 @@ def make_beetmover_description(config, jobs):
'treeherder': treeherder,
'locale': locale,
'shipping-phase': job['shipping-phase'],
'attributes': job['attributes'],
}
yield beet_description

22
taskcluster/taskgraph/transforms/beetmover_source_checksums.py
View File

@ -11,9 +11,12 @@ from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
generate_beetmover_upstream_artifacts,
get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
get_worker_type_for_scope,
should_use_artifact_map)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
@ -32,6 +35,7 @@ beetmover_checksums_description_schema = schema.extend({
Optional('locale'): basestring,
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('attributes'): task_description_schema['attributes'],
})
transforms = TransformSequence()
@ -73,6 +77,7 @@ def make_beetmover_checksums_description(config, jobs):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
@ -141,14 +146,21 @@ def make_beetmover_checksums_worker(config, jobs):
raise NotImplementedError(
"Beetmover checksums must have a beetmover and signing dependency!")
if should_use_artifact_map(platform, config.params['project']):
upstream_artifacts = generate_beetmover_upstream_artifacts(job, platform, locale)
else:
upstream_artifacts = generate_upstream_artifacts(refs, platform, locale)
worker = {
'implementation': 'beetmover',
'release-properties': craft_release_properties(config, job),
'upstream-artifacts': generate_upstream_artifacts(
refs, platform, locale
),
'upstream-artifacts': upstream_artifacts,
}
if should_use_artifact_map(platform, config.params['project']):
worker['artifact-map'] = generate_beetmover_artifact_map(
config, job, platform=platform)
if locale:
worker["locale"] = locale
job["worker"] = worker

27
taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
View File

@ -9,9 +9,12 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
generate_beetmover_upstream_artifacts,
get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope,
should_use_artifact_map,
)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.transforms.beetmover import craft_release_properties
@ -65,6 +68,7 @@ def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-SGenChcks')
@ -121,7 +125,7 @@ def generate_upstream_artifacts(job, signing_task_ref, build_task_ref):
"paths": ["{}/{}".format(artifact_prefix, p)
for p in build_mapping],
"locale": "en-US",
}, {
}, {
"taskId": {"task-reference": signing_task_ref},
"taskType": "signing",
"paths": ["{}/{}".format(artifact_prefix, p)
@ -154,10 +158,25 @@ def make_task_worker(config, jobs):
worker = {
'implementation': 'beetmover',
'release-properties': craft_release_properties(config, job),
'upstream-artifacts': generate_upstream_artifacts(
}
platform = job["attributes"]["build_platform"]
# Works with Firefox/Devedition. Commented for migration.
if should_use_artifact_map(platform, config.params['project']):
upstream_artifacts = generate_beetmover_upstream_artifacts(
job, platform=None, locale=None
)
else:
upstream_artifacts = generate_upstream_artifacts(
job, signing_task_ref, build_task_ref
)
}
worker['upstream-artifacts'] = upstream_artifacts
# Works with Firefox/Devedition. Commented for migration.
if should_use_artifact_map(platform, config.params['project']):
worker['artifact-map'] = generate_beetmover_artifact_map(
config, job, platform=platform)
job["worker"] = worker

6
taskcluster/taskgraph/transforms/task.py
View File

@ -921,6 +921,7 @@ def build_binary_transparency_payload(config, task, task_def):
# locale is used to map upload path and allow for duplicate simple names
Required('locale'): basestring,
}],
Optional('artifact-map'): object,
})
def build_beetmover_payload(config, task, task_def):
worker = task['worker']
@ -942,6 +943,8 @@ def build_beetmover_payload(config, task, task_def):
}
if worker.get('locale'):
task_def['payload']['locale'] = worker['locale']
if worker.get('artifact-map'):
task_def['payload']['artifactMap'] = worker['artifact-map']
if worker.get('partner-public'):
task_def['payload']['is_partner_repack_public'] = worker['partner-public']
if release_config:
@ -983,11 +986,14 @@ def build_beetmover_push_to_release_payload(config, task, task_def):
Required('paths'): [basestring],
Required('zipExtract', default=False): bool,
}],
Optional('artifact-map'): object,
})
def build_beetmover_maven_payload(config, task, task_def):
build_beetmover_payload(config, task, task_def)
task_def['payload']['artifact_id'] = task['worker']['release-properties']['artifact-id']
if task['worker'].get('artifact-map'):
task_def['payload']['artifactMap'] = task['worker']['artifact-map']
del task_def['payload']['releaseProperties']['hashType']
del task_def['payload']['releaseProperties']['platform']

264
taskcluster/taskgraph/util/scriptworker.py
View File

@ -19,7 +19,14 @@ from __future__ import absolute_import, print_function, unicode_literals
import functools
import json
import os
import itertools
from copy import deepcopy
from datetime import datetime
import jsone
from .schema import resolve_keyed_by
from .taskcluster import get_artifact_prefix
from .yaml import load_yaml
# constants {{{1
"""Map signing scope aliases to sets of projects.
@ -387,3 +394,260 @@ def get_worker_type_for_scope(config, scope):
),
)
)
# generate_beetmover_upstream_artifacts {{{1
def generate_beetmover_upstream_artifacts(job, platform, locale=None, dependencies=None):
"""Generate the upstream artifacts for beetmover, using the artifact map.
Currently only applies to beetmover tasks.
Args:
job (dict): The current job being generated
dependencies (list): A list of the job's dependency labels.
platform (str): The current build platform
locale (str): The current locale being beetmoved.
Returns:
list: A list of dictionaries conforming to the upstream_artifacts spec.
"""
base_artifact_prefix = get_artifact_prefix(job)
resolve_keyed_by(job, 'attributes.artifact_map', 'artifact map', platform=platform)
map_config = load_yaml(*os.path.split(job['attributes']['artifact_map']))
upstream_artifacts = list()
if not locale:
locales = map_config['default_locales']
else:
locales = [locale]
if not dependencies:
dependencies = job['dependencies'].keys()
for locale, dep in itertools.product(locales, dependencies):
paths = list()
for filename in map_config['mapping']:
if dep not in map_config['mapping'][filename]['from']:
continue
if locale != 'en-US' and not map_config['mapping'][filename]['all_locales']:
continue
# The next time we look at this file it might be a different locale.
file_config = deepcopy(map_config['mapping'][filename])
resolve_keyed_by(file_config, "source_path_modifier",
'source path modifier', locale=locale)
paths.append(os.path.join(
base_artifact_prefix,
jsone.render(file_config['source_path_modifier'], {'locale': locale}),
filename,
))
if not paths:
continue
upstream_artifacts.append({
"taskId": {
"task-reference": "<{}>".format(dep)
},
"taskType": map_config['tasktype_map'].get(dep),
"paths": sorted(paths),
"locale": locale,
})
return upstream_artifacts
# generate_beetmover_compressed_upstream_artifacts {{{1
def generate_beetmover_compressed_upstream_artifacts(job, dependencies=None):
"""Generate compressed file upstream artifacts for beetmover.
These artifacts will not be beetmoved directly, but will be
decompressed from upstream_mapping and the contents beetmoved
using the `mapping` entry in the artifact map.
Currently only applies to beetmover tasks.
Args:
job (dict): The current job being generated
dependencies (list): A list of the job's dependency labels.
Returns:
list: A list of dictionaries conforming to the upstream_artifacts spec.
"""
base_artifact_prefix = get_artifact_prefix(job)
map_config = load_yaml(*os.path.split(job['attributes']['artifact_map']))
upstream_artifacts = list()
if not dependencies:
dependencies = job['dependencies'].keys()
for dep in dependencies:
paths = list()
for filename in map_config['upstream_mapping']:
if dep not in map_config['upstream_mapping'][filename]['from']:
continue
paths.append(os.path.join(
base_artifact_prefix,
filename,
))
if not paths:
continue
upstream_artifacts.append({
"taskId": {
"task-reference": "<{}>".format(dep)
},
"taskType": map_config['tasktype_map'].get(dep),
"paths": sorted(paths),
"zipExtract": True,
})
return upstream_artifacts
# generate_beetmover_artifact_map {{{1
def generate_beetmover_artifact_map(config, job, **kwargs):
"""Generate the beetmover artifact map.
Currently only applies to beetmover tasks.
Args:
config (): Current taskgraph configuration.
job (dict): The current job being generated
Common kwargs:
platform (str): The current build platform
locale (str): The current locale being beetmoved.
Returns:
list: A list of dictionaries containing source->destination
maps for beetmover.
"""
platform = kwargs.get('platform', '')
resolve_keyed_by(job, 'attributes.artifact_map', 'artifact map', platform=platform)
map_config = load_yaml(*os.path.split(job['attributes']['artifact_map']))
base_artifact_prefix = map_config.get('base_artifact_prefix', get_artifact_prefix(job))
artifacts = list()
dependencies = job['dependencies'].keys()
if kwargs.get('locale'):
locales = [kwargs['locale']]
else:
locales = map_config['default_locales']
resolve_keyed_by(map_config, 's3_bucket_paths', 's3_bucket_paths', platform=platform)
for locale, dep in itertools.product(locales, dependencies):
paths = dict()
for filename in map_config['mapping']:
# Relevancy checks
if dep not in map_config['mapping'][filename]['from']:
# We don't get this file from this dependency.
continue
if locale != 'en-US' and not map_config['mapping'][filename]['all_locales']:
# This locale either doesn't produce or shouldn't upload this file.
continue
# Filling in destinations
# deepcopy because the next time we look at this file the locale will differ.
file_config = deepcopy(map_config['mapping'][filename])
for field in [
'destinations',
'locale_prefix',
'source_path_modifier',
'update_balrog_manifest',
'pretty_name',
'checksums_path'
]:
resolve_keyed_by(file_config, field, field, locale=locale, platform=platform)
# This format string should ideally be in the configuration file,
# but this would mean keeping variable names in sync between code + config.
destinations = [
"{s3_bucket_path}/{dest_path}/{locale_prefix}{filename}".format(
s3_bucket_path=bucket_path,
dest_path=dest_path,
locale_prefix=file_config['locale_prefix'],
filename=file_config.get('pretty_name', filename),
)
for dest_path, bucket_path
in itertools.product(file_config['destinations'], map_config['s3_bucket_paths'])
]
# Creating map entries
# Key must be artifact path, to avoid trampling duplicates, such
# as public/build/target.apk and public/build/en-US/target.apk
key = os.path.join(
base_artifact_prefix,
file_config['source_path_modifier'],
filename,
)
paths[key] = {
'destinations': destinations,
}
if file_config.get('checksums_path'):
paths[key]['checksums_path'] = file_config['checksums_path']
# Optional flags.
if file_config.get('update_balrog_manifest'):
paths[key]['update_balrog_manifest'] = True
if file_config.get('balrog_format'):
paths[key]['balrog_format'] = file_config['balrog_format']
if not paths:
# No files for this dependency/locale combination.
continue
# Render all variables for the artifact map
platforms = deepcopy(map_config['platform_names'])
if platform:
for key in platforms.keys():
resolve_keyed_by(platforms, key, key, platform=platform)
upload_date = datetime.fromtimestamp(config.params['build_date'])
kwargs.update({
'locale': locale,
'version': config.params['app_version'],
'branch': config.params['project'],
'build_number': config.params['build_number'],
'filename_platform': platforms['filename_platform'],
'path_platform': platforms['path_platform'],
'year': upload_date.year,
'month': upload_date.strftime("%m"), # zero-pad the month
'upload_date': upload_date.strftime("%Y-%m-%d-%H-%M-%S")
})
paths = jsone.render(paths, kwargs)
artifacts.append({
'taskId': {'task-reference': "<{}>".format(dep)},
'locale': locale,
'paths': paths,
})
return artifacts
# should_use_artifact_map {{{
def should_use_artifact_map(platform, project):
"""Return True if this task uses the beetmover artifact map.
This function exists solely for the beetmover artifact map
migration.
"""
platforms = ['android', 'fennec']
#FIXME: once we're ready to switch fully to declarative artifacts on other
# branches, we can expand this
projects = ['mozilla-central']
if any([pl in platform for pl in platforms]) and any([pj in project for pj in projects]):
return True
return False