Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-24 13:21:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1816992 - Add attribution job for DMGs r=taskgraph-reviewers,releng-reviewers,bhearsum

Browse Source 
Depends on D189404

Differential Revision: https://phabricator.services.mozilla.com/D189405
This commit is contained in:
Heitor Neiva 2023-10-06 20:32:00 +00:00
parent 8c6a1ab185
commit fd8c1dc88c
8 changed files with 171 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
taskcluster/ci/attribution-l10n/kind.yml
View File

@ -5,6 +5,7 @@
loader: gecko_taskgraph.loader.transform:loader
transforms:
- gecko_taskgraph.transforms.attribution:resolve_keyed_by_transforms
- gecko_taskgraph.transforms.split_by_locale:transforms
- gecko_taskgraph.transforms.attribution:transforms
- taskgraph.transforms.task_context
@ -15,6 +16,8 @@ kind-dependencies:
- toolchain
# Windows
- repackage-signing-l10n
# Mac
- repackage-l10n
job-defaults:
description: Attribute release builds
@ -24,21 +27,38 @@ job-defaults:
worker-type: b-linux-gcp
locales-file: browser/locales/l10n-changesets.json
properties-with-locale:
- label
- dependencies.repackage-signing-l10n
- treeherder.symbol
- fetches.repackage-signing-l10n
by-build-platform:
win.*:
- label
- dependencies.repackage-signing-l10n
- treeherder.symbol
- fetches.repackage-signing-l10n
mac.*:
- label
- dependencies.repackage-l10n
- treeherder.symbol
- fetches.repackage-l10n
attributes:
build_type: opt
release_artifacts:
- public/build/target.installer.exe
by-build-platform:
win.*:
- public/build/target.installer.exe
mac.*:
- public/build/target.dmg
treeherder:
symbol: Attr-L10n({locale})
kind: other
tier: 1
fetches:
repackage-signing-l10n:
- "{locale}/target.installer.exe"
by-build-platform:
win.*:
repackage-signing-l10n:
- "{locale}/target.installer.exe"
mac.*:
repackage-l10n:
- "{locale}/target.dmg"
toolchain:
- "linux64-libdmg"
task-context:
from-file: browser/installer/attribution.yml
substitution-fields:
@ -57,17 +77,27 @@ job-defaults:
using: run-task
cwd: "{checkout}"
command:
- ./mach
- python
- python/mozrelease/mozrelease/attribute_builds.py
- --attribution
- dlsource={attribution_code[json][dlsource]}
- --output
- /builds/worker/artifacts
# All jobs require attribution of target.installer.exe. Some may require
# attribution of other files, which they can append themselves.
- --input
- /builds/worker/fetches/target.installer.exe
by-build-platform:
win.*:
- ./mach
- python
- python/mozrelease/mozrelease/attribute_builds.py
- --attribution
- dlsource={attribution_code[json][dlsource]}
- --output
- /builds/worker/artifacts
# All jobs require attribution of target.installer.exe. Some may require
# attribution of other files, which they can append themselves.
- --input
- /builds/worker/fetches/target.installer.exe
mac.*:
- mkdir -p /builds/worker/artifacts &&
- /builds/worker/fetches/dmg/dmg
- attribute
- /builds/worker/fetches/target.dmg
- /builds/worker/artifacts/target.dmg
- __MOZCUSTOM__
- __MOZCUSTOM__dlsource%3D{attribution_code[json][dlsource]}
use-system-python: false
jobs:
@ -133,3 +163,24 @@ jobs:
platform: win64-aarch64-devedition/opt
dependencies:
repackage-signing-l10n: repackage-signing-l10n-{locale}-win64-aarch64-devedition/opt
macosx64-devedition/opt:
label: attribution-macosx64-{locale}-devedition/opt
shipping-product: devedition
attributes:
build_platform: macosx64-devedition
treeherder:
platform: macosx64-devedition/opt
dependencies:
repackage-l10n: repackage-l10n-{locale}-macosx64-devedition/opt
macosx64-shippable/opt:
label: attribution-macosx64-{locale}-shippable/opt
locale-file-platform: macosx64
shipping-product: firefox
attributes:
build_platform: macosx64-shippable
treeherder:
platform: macosx64-shippable/opt
dependencies:
repackage-l10n: repackage-l10n-{locale}-macosx64-shippable/opt

74
taskcluster/ci/attribution/kind.yml
View File

@ -5,6 +5,7 @@
loader: gecko_taskgraph.loader.transform:loader
transforms:
- gecko_taskgraph.transforms.attribution:resolve_keyed_by_transforms
- gecko_taskgraph.transforms.attribution:transforms
- taskgraph.transforms.task_context
- gecko_taskgraph.transforms.job:transforms
@ -14,6 +15,8 @@ kind-dependencies:
- toolchain
# Windows
- repackage-signing
# Mac
- repackage
job-defaults:
description: Attribute release builds
@ -22,11 +25,22 @@ job-defaults:
run-on-projects: []
worker-type: b-linux-gcp
fetches:
repackage-signing:
- target.installer.exe
by-build-platform:
win.*:
repackage-signing:
- target.installer.exe
mac.*:
repackage:
- target.dmg
toolchain:
- linux64-libdmg
attributes:
release_artifacts:
- public/build/target.installer.exe
by-build-platform:
win.*:
- public/build/target.installer.exe
mac.*:
- public/build/target.dmg
treeherder:
symbol: Attr
kind: other
@ -49,17 +63,27 @@ job-defaults:
using: run-task
cwd: "{checkout}"
command:
- ./mach
- python
- python/mozrelease/mozrelease/attribute_builds.py
- --attribution
- dlsource={attribution_code[json][dlsource]}
- --output
- /builds/worker/artifacts
# All jobs require attribution of target.installer.exe. Some may require
# attribution of other files, which they can append themselves.
- --input
- /builds/worker/fetches/target.installer.exe
by-build-platform:
win.*:
- ./mach
- python
- python/mozrelease/mozrelease/attribute_builds.py
- --attribution
- dlsource={attribution_code[json][dlsource]}
- --output
- /builds/worker/artifacts
# All jobs require attribution of target.installer.exe. Some may require
# attribution of other files, which they can append themselves.
- --input
- /builds/worker/fetches/target.installer.exe
mac.*:
- mkdir -p /builds/worker/artifacts &&
- /builds/worker/fetches/dmg/dmg
- attribute
- /builds/worker/fetches/target.dmg
- /builds/worker/artifacts/target.dmg
- __MOZCUSTOM__
- __MOZCUSTOM__dlsource%3D{attribution_code[json][dlsource]}
use-system-python: false
jobs:
@ -128,3 +152,25 @@ jobs:
platform: win64-aarch64-devedition/opt
dependencies:
repackage-signing: repackage-signing-win64-aarch64-devedition/opt
macosx64-devedition/opt:
label: attribution-macosx64-devedition/opt
shipping-product: devedition
attributes:
build_platform: macosx64-devedition
build_type: opt
treeherder:
platform: macosx64-devedition/opt
dependencies:
repackage: repackage-macosx64-devedition/opt
macosx64-shippable/opt:
label: attribution-macosx64-shippable/opt
shipping-product: firefox
attributes:
build_platform: macosx64-shippable
build_type: opt
treeherder:
platform: macosx64-shippable/opt
dependencies:
repackage: repackage-macosx64-shippable/opt

2
taskcluster/ci/beetmover-repackage/kind.yml
View File

@ -32,9 +32,7 @@ kind-dependencies:
- repackage-signing-shippable-l10n-msix
- mar-signing
- mar-signing-l10n
# Windows
- attribution
# Windows
- attribution-l10n
- repackage-deb
- repackage-deb-l10n

2
taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml
View File

@ -314,7 +314,7 @@ mapping:
description: "Main package disk image for Mac OS X platforms"
all_locales: true
from:
- repackage
- attribution
only_for_platforms:
- macosx64-shippable
- macosx64-devedition

2
taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml
View File

@ -358,7 +358,7 @@ mapping:
description: "Main package disk image for Mac OS X platforms"
all_locales: true
from:
- repackage
- attribution
only_for_platforms:
- macosx64-shippable
pretty_name: firefox-${version}.${locale}.${filename_platform}.dmg

34
taskcluster/gecko_taskgraph/transforms/attribution.py
View File

@ -3,8 +3,25 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import resolve_keyed_by
transforms = TransformSequence()
resolve_keyed_by_transforms = TransformSequence()
@resolve_keyed_by_transforms.add
def attribution_keyed_by(config, jobs):
keyed_by_fields = (
"fetches",
"attributes.release_artifacts",
"run.command",
"properties-with-locale", # properties-with-locale only exists in the l10n task
)
for job in jobs:
build_platform = {"build-platform": job["attributes"]["build_platform"]}
for field in keyed_by_fields:
resolve_keyed_by(item=job, field=field, item_name=field, **build_platform)
yield job
@transforms.add
@ -30,3 +47,20 @@ def stub_installer(config, jobs):
"public/build/target.stub-installer.exe"
)
yield job
@transforms.add
def mac_attribution(config, jobs):
"""Adds \t padding to the mac attribution data. Implicitly assumes that the
attribution data is the last thing in job.run.command
"""
for job in jobs:
if "macosx" in job["attributes"]["build_platform"]:
# Last argument of command should be the attribution data
command = job["run"]["command"]
attribution_arg = command[-1]
while len(attribution_arg) < 1024:
attribution_arg += "\t"
command[-1] = attribution_arg
job["run"]["command"] = " ".join(command)
yield job

1
taskcluster/gecko_taskgraph/transforms/repackage.py
View File

@ -321,6 +321,7 @@ def copy_in_useful_magic(config, jobs):
job["build-platform"] = dep.attributes.get("build_platform")
job["shipping-product"] = dep.attributes.get("shipping_product")
job["build-type"] = dep.attributes.get("build_type")
yield job

11
taskcluster/gecko_taskgraph/transforms/repackage_partner.py
View File

@ -23,11 +23,6 @@ from gecko_taskgraph.util.partners import get_partner_config_by_kind
from gecko_taskgraph.util.platforms import archive_format, executable_extension
from gecko_taskgraph.util.workertypes import worker_type_implementation
def _by_platform(arg):
return optionally_keyed_by("build-platform", arg)
# When repacking the stub installer we need to pass a zip file and package name to the
# repackage task. This is not needed for vanilla stub but analogous to the full installer.
PACKAGE_FORMATS = copy.deepcopy(PACKAGE_FORMATS_VANILLA)
@ -45,11 +40,13 @@ packaging_description_schema = Schema(
# Shipping product and phase
Optional("shipping-product"): task_description_schema["shipping-product"],
Optional("shipping-phase"): task_description_schema["shipping-phase"],
Required("package-formats"): _by_platform([str]),
Required("package-formats"): optionally_keyed_by(
"build-platform", "build-type", [str]
),
# All l10n jobs use mozharness
Required("mozharness"): {
# Config files passed to the mozharness script
Required("config"): _by_platform([str]),
Required("config"): optionally_keyed_by("build-platform", [str]),
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional("config-paths"): [str],