From 8111a424952cfad3c7b6d1dd3f7b9968f4f81406 Mon Sep 17 00:00:00 2001 From: Butkovits Atila Date: Sat, 7 Oct 2023 16:46:14 +0300 Subject: [PATCH] Backed out 2 changesets (bug 1816992) for causing Build bustages. a=backout Backed out changeset 1d25ed23e85a (bug 1816992) Backed out changeset 7628ecf70973 (bug 1816992) --- python/mozbuild/mozbuild/mach_commands.py | 7 +- python/mozbuild/mozbuild/repackaging/dmg.py | 3 +- python/mozbuild/mozpack/dmg.py | 34 +------ taskcluster/ci/attribution-l10n/kind.yml | 89 ++++--------------- taskcluster/ci/attribution/kind.yml | 74 +++------------ taskcluster/ci/beetmover-repackage/kind.yml | 2 + .../kind.yml | 6 +- .../release-partner-repack-repackage/kind.yml | 6 +- taskcluster/ci/repackage-l10n/kind.yml | 6 +- taskcluster/ci/repackage/kind.yml | 6 +- .../manifests/firefox_candidates.yml | 2 +- .../manifests/firefox_nightly.yml | 2 +- .../gecko_taskgraph/transforms/attribution.py | 34 ------- .../gecko_taskgraph/transforms/repackage.py | 14 +-- .../transforms/repackage_partner.py | 11 ++- 15 files changed, 55 insertions(+), 241 deletions(-) diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index d75fdf059c98..6f834965c053 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -2405,17 +2405,14 @@ def repackage_deb_l10n( @SubCommand("repackage", "dmg", description="Repackage a tar file into a .dmg for OSX") @CommandArgument("--input", "-i", type=str, required=True, help="Input filename") @CommandArgument("--output", "-o", type=str, required=True, help="Output filename") -@CommandArgument( - "--attribution_sentinel", type=str, required=False, help="DMGs with attribution." -) -def repackage_dmg(command_context, input, output, attribution_sentinel): +def repackage_dmg(command_context, input, output): if not os.path.exists(input): print("Input file does not exist: %s" % input) return 1 from mozbuild.repackaging.dmg import repackage_dmg - repackage_dmg(input, output, attribution_sentinel) + repackage_dmg(input, output) @SubCommand("repackage", "pkg", description="Repackage a tar file into a .pkg for OSX") diff --git a/python/mozbuild/mozbuild/repackaging/dmg.py b/python/mozbuild/mozbuild/repackaging/dmg.py index d83ab0cb77df..5054fdec8e10 100644 --- a/python/mozbuild/mozbuild/repackaging/dmg.py +++ b/python/mozbuild/mozbuild/repackaging/dmg.py @@ -12,7 +12,7 @@ from mozbuild.bootstrap import bootstrap_toolchain from mozbuild.repackaging.application_ini import get_application_ini_value -def repackage_dmg(infile, output, attribution_sentinel=None): +def repackage_dmg(infile, output): if not tarfile.is_tarfile(infile): raise Exception("Input file %s is not a valid tarfile." % infile) @@ -52,5 +52,4 @@ def repackage_dmg(infile, output, attribution_sentinel=None): dmg_tool=Path(dmg_tool), hfs_tool=Path(hfs_tool), mkfshfs_tool=Path(mkfshfs_tool), - attribution_sentinel=attribution_sentinel, ) diff --git a/python/mozbuild/mozpack/dmg.py b/python/mozbuild/mozpack/dmg.py index 4e094648fe09..334f3a69cc3d 100644 --- a/python/mozbuild/mozpack/dmg.py +++ b/python/mozbuild/mozpack/dmg.py @@ -85,37 +85,17 @@ def create_dmg_from_staged( volume_name: str, hfs_tool: Path = None, dmg_tool: Path = None, - mkfshfs_tool: Path = None, - attribution_sentinel: str = None, ): "Given a prepared directory stagedir, produce a DMG at output_dmg." - if is_linux: # The dmg tool doesn't create the destination directories, and silently # returns success if the parent directory doesn't exist. ensureParentDir(output_dmg) + hfs = os.path.join(tmpdir, "staged.hfs") subprocess.check_call([hfs_tool, hfs, "addall", stagedir]) - - dmg_cmd = [dmg_tool, "build", hfs, output_dmg] - if attribution_sentinel: - while len(attribution_sentinel) < 1024: - attribution_sentinel += "\t" - subprocess.check_call( - [ - hfs_tool, - hfs, - "setattr", - f"{volume_name}.app", - "com.apple.application-instance", - attribution_sentinel, - ] - ) - subprocess.check_call(["cp", hfs, str(Path(output_dmg).parent)]) - dmg_cmd.append(attribution_sentinel) - subprocess.check_call( - dmg_cmd, + [dmg_tool, "build", hfs, output_dmg], # dmg is seriously chatty stdout=subprocess.DEVNULL, ) @@ -160,7 +140,6 @@ def create_dmg( dmg_tool: Path, hfs_tool: Path, mkfshfs_tool: Path, - attribution_sentinel: str = None, ): """ Create a DMG disk image at the path output_dmg from source_directory. @@ -192,14 +171,7 @@ def create_dmg( set_folder_icon(stagedir, tmpdir, hfs_tool) chmod(stagedir) create_dmg_from_staged( - stagedir, - output_dmg, - tmpdir, - volume_name, - hfs_tool, - dmg_tool, - mkfshfs_tool, - attribution_sentinel, + stagedir, output_dmg, tmpdir, volume_name, hfs_tool, dmg_tool ) diff --git a/taskcluster/ci/attribution-l10n/kind.yml b/taskcluster/ci/attribution-l10n/kind.yml index 811d18f55101..132f6fdc8ccb 100644 --- a/taskcluster/ci/attribution-l10n/kind.yml +++ b/taskcluster/ci/attribution-l10n/kind.yml @@ -5,7 +5,6 @@ loader: gecko_taskgraph.loader.transform:loader transforms: - - gecko_taskgraph.transforms.attribution:resolve_keyed_by_transforms - gecko_taskgraph.transforms.split_by_locale:transforms - gecko_taskgraph.transforms.attribution:transforms - taskgraph.transforms.task_context @@ -16,8 +15,6 @@ kind-dependencies: - toolchain # Windows - repackage-signing-l10n - # Mac - - repackage-l10n job-defaults: description: Attribute release builds @@ -27,38 +24,21 @@ job-defaults: worker-type: b-linux-gcp locales-file: browser/locales/l10n-changesets.json properties-with-locale: - by-build-platform: - win.*: - - label - - dependencies.repackage-signing-l10n - - treeherder.symbol - - fetches.repackage-signing-l10n - mac.*: - - label - - dependencies.repackage-l10n - - treeherder.symbol - - fetches.repackage-l10n + - label + - dependencies.repackage-signing-l10n + - treeherder.symbol + - fetches.repackage-signing-l10n attributes: + build_type: opt release_artifacts: - by-build-platform: - win.*: - - public/build/target.installer.exe - mac.*: - - public/build/target.dmg + - public/build/target.installer.exe treeherder: symbol: Attr-L10n({locale}) kind: other tier: 1 fetches: - by-build-platform: - win.*: - repackage-signing-l10n: - - "{locale}/target.installer.exe" - mac.*: - repackage-l10n: - - "{locale}/target.dmg" - toolchain: - - "linux64-libdmg" + repackage-signing-l10n: + - "{locale}/target.installer.exe" task-context: from-file: browser/installer/attribution.yml substitution-fields: @@ -77,27 +57,17 @@ job-defaults: using: run-task cwd: "{checkout}" command: - by-build-platform: - win.*: - - ./mach - - python - - python/mozrelease/mozrelease/attribute_builds.py - - --attribution - - dlsource={attribution_code[json][dlsource]} - - --output - - /builds/worker/artifacts - # All jobs require attribution of target.installer.exe. Some may require - # attribution of other files, which they can append themselves. - - --input - - /builds/worker/fetches/target.installer.exe - mac.*: - - mkdir -p /builds/worker/artifacts && - - /builds/worker/fetches/dmg/dmg - - attribute - - /builds/worker/fetches/target.dmg - - /builds/worker/artifacts/target.dmg - - __MOZCUSTOM__ - - __MOZCUSTOM__dlsource%3D{attribution_code[json][dlsource]} + - ./mach + - python + - python/mozrelease/mozrelease/attribute_builds.py + - --attribution + - dlsource={attribution_code[json][dlsource]} + - --output + - /builds/worker/artifacts + # All jobs require attribution of target.installer.exe. Some may require + # attribution of other files, which they can append themselves. + - --input + - /builds/worker/fetches/target.installer.exe use-system-python: false jobs: @@ -163,24 +133,3 @@ jobs: platform: win64-aarch64-devedition/opt dependencies: repackage-signing-l10n: repackage-signing-l10n-{locale}-win64-aarch64-devedition/opt - - macosx64-devedition/opt: - label: attribution-macosx64-{locale}-devedition/opt - shipping-product: devedition - attributes: - build_platform: macosx64-devedition - treeherder: - platform: macosx64-devedition/opt - dependencies: - repackage-l10n: repackage-l10n-{locale}-macosx64-devedition/opt - - macosx64-shippable/opt: - label: attribution-macosx64-{locale}-shippable/opt - locale-file-platform: macosx64 - shipping-product: firefox - attributes: - build_platform: macosx64-shippable - treeherder: - platform: macosx64-shippable/opt - dependencies: - repackage-l10n: repackage-l10n-{locale}-macosx64-shippable/opt diff --git a/taskcluster/ci/attribution/kind.yml b/taskcluster/ci/attribution/kind.yml index 1c8fcb5153b4..34994fb3dc0f 100644 --- a/taskcluster/ci/attribution/kind.yml +++ b/taskcluster/ci/attribution/kind.yml @@ -5,7 +5,6 @@ loader: gecko_taskgraph.loader.transform:loader transforms: - - gecko_taskgraph.transforms.attribution:resolve_keyed_by_transforms - gecko_taskgraph.transforms.attribution:transforms - taskgraph.transforms.task_context - gecko_taskgraph.transforms.job:transforms @@ -15,8 +14,6 @@ kind-dependencies: - toolchain # Windows - repackage-signing - # Mac - - repackage job-defaults: description: Attribute release builds @@ -25,22 +22,11 @@ job-defaults: run-on-projects: [] worker-type: b-linux-gcp fetches: - by-build-platform: - win.*: - repackage-signing: - - target.installer.exe - mac.*: - repackage: - - target.dmg - toolchain: - - linux64-libdmg + repackage-signing: + - target.installer.exe attributes: release_artifacts: - by-build-platform: - win.*: - - public/build/target.installer.exe - mac.*: - - public/build/target.dmg + - public/build/target.installer.exe treeherder: symbol: Attr kind: other @@ -63,27 +49,17 @@ job-defaults: using: run-task cwd: "{checkout}" command: - by-build-platform: - win.*: - - ./mach - - python - - python/mozrelease/mozrelease/attribute_builds.py - - --attribution - - dlsource={attribution_code[json][dlsource]} - - --output - - /builds/worker/artifacts - # All jobs require attribution of target.installer.exe. Some may require - # attribution of other files, which they can append themselves. - - --input - - /builds/worker/fetches/target.installer.exe - mac.*: - - mkdir -p /builds/worker/artifacts && - - /builds/worker/fetches/dmg/dmg - - attribute - - /builds/worker/fetches/target.dmg - - /builds/worker/artifacts/target.dmg - - __MOZCUSTOM__ - - __MOZCUSTOM__dlsource%3D{attribution_code[json][dlsource]} + - ./mach + - python + - python/mozrelease/mozrelease/attribute_builds.py + - --attribution + - dlsource={attribution_code[json][dlsource]} + - --output + - /builds/worker/artifacts + # All jobs require attribution of target.installer.exe. Some may require + # attribution of other files, which they can append themselves. + - --input + - /builds/worker/fetches/target.installer.exe use-system-python: false jobs: @@ -152,25 +128,3 @@ jobs: platform: win64-aarch64-devedition/opt dependencies: repackage-signing: repackage-signing-win64-aarch64-devedition/opt - - macosx64-devedition/opt: - label: attribution-macosx64-devedition/opt - shipping-product: devedition - attributes: - build_platform: macosx64-devedition - build_type: opt - treeherder: - platform: macosx64-devedition/opt - dependencies: - repackage: repackage-macosx64-devedition/opt - - macosx64-shippable/opt: - label: attribution-macosx64-shippable/opt - shipping-product: firefox - attributes: - build_platform: macosx64-shippable - build_type: opt - treeherder: - platform: macosx64-shippable/opt - dependencies: - repackage: repackage-macosx64-shippable/opt diff --git a/taskcluster/ci/beetmover-repackage/kind.yml b/taskcluster/ci/beetmover-repackage/kind.yml index 8746bb5b44c7..380dd0e0b4ee 100644 --- a/taskcluster/ci/beetmover-repackage/kind.yml +++ b/taskcluster/ci/beetmover-repackage/kind.yml @@ -32,7 +32,9 @@ kind-dependencies: - repackage-signing-shippable-l10n-msix - mar-signing - mar-signing-l10n + # Windows - attribution + # Windows - attribution-l10n - repackage-deb - repackage-deb-l10n diff --git a/taskcluster/ci/release-eme-free-repack-repackage/kind.yml b/taskcluster/ci/release-eme-free-repack-repackage/kind.yml index 4f5c05af2bc6..78bc18fe6d2f 100644 --- a/taskcluster/ci/release-eme-free-repack-repackage/kind.yml +++ b/taskcluster/ci/release-eme-free-repack-repackage/kind.yml @@ -55,10 +55,6 @@ jobs: - repackage/win64_partner.py package-formats: by-build-platform: - macosx64\b.*: - by-build-type: - # Debug builds can't be attributable - debug: [dmg] - default: [dmg-attrib] + macosx64\b.*: [dmg] win32\b.*: [installer] win64\b.*: [installer] diff --git a/taskcluster/ci/release-partner-repack-repackage/kind.yml b/taskcluster/ci/release-partner-repack-repackage/kind.yml index 025cbe6b9b7e..099c55240f22 100644 --- a/taskcluster/ci/release-partner-repack-repackage/kind.yml +++ b/taskcluster/ci/release-partner-repack-repackage/kind.yml @@ -56,10 +56,6 @@ jobs: - repackage/win64_partner.py package-formats: by-build-platform: - macosx64\b.*: - by-build-type: - # Debug builds can't be attributable - debug: [dmg] - default: [dmg-attrib] + macosx64\b.*: [dmg] win32\b.*: [installer] win64\b.*: [installer] diff --git a/taskcluster/ci/repackage-l10n/kind.yml b/taskcluster/ci/repackage-l10n/kind.yml index b0d492457283..d0ede27f1eec 100644 --- a/taskcluster/ci/repackage-l10n/kind.yml +++ b/taskcluster/ci/repackage-l10n/kind.yml @@ -71,12 +71,8 @@ jobs: - repackage/win64_signed.py package-formats: by-build-platform: - macosx64\b.*: - by-build-type: - # Debug builds can't be attributable - debug: [dmg] - default: [dmg-attrib] linux.*: [mar] + macosx64\b.*: [mar, dmg, pkg] win32\b.*: [mar, installer] win64\b.*: [mar, installer] fetches: diff --git a/taskcluster/ci/repackage/kind.yml b/taskcluster/ci/repackage/kind.yml index d907b924b383..8e2f9e71c80c 100644 --- a/taskcluster/ci/repackage/kind.yml +++ b/taskcluster/ci/repackage/kind.yml @@ -85,12 +85,8 @@ jobs: - repackage/win64_signed.py package-formats: by-build-platform: - macosx64\b.*: - by-build-type: - # Debug builds can't be attributable - debug: [dmg] - default: [dmg-attrib] linux.*: [mar] + macosx64\b.*: [mar, dmg, pkg] win32\b.*: [mar, installer] win64\b.*: [mar, installer] fetches: diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml index d647a20e7f7b..e8d7061c786a 100644 --- a/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml +++ b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml @@ -314,7 +314,7 @@ mapping: description: "Main package disk image for Mac OS X platforms" all_locales: true from: - - attribution + - repackage only_for_platforms: - macosx64-shippable - macosx64-devedition diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml index 23c35e9b890d..d2e4f6adf024 100644 --- a/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml +++ b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml @@ -358,7 +358,7 @@ mapping: description: "Main package disk image for Mac OS X platforms" all_locales: true from: - - attribution + - repackage only_for_platforms: - macosx64-shippable pretty_name: firefox-${version}.${locale}.${filename_platform}.dmg diff --git a/taskcluster/gecko_taskgraph/transforms/attribution.py b/taskcluster/gecko_taskgraph/transforms/attribution.py index 25451acbc84b..935c274c03ff 100644 --- a/taskcluster/gecko_taskgraph/transforms/attribution.py +++ b/taskcluster/gecko_taskgraph/transforms/attribution.py @@ -3,25 +3,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from taskgraph.transforms.base import TransformSequence -from taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() -resolve_keyed_by_transforms = TransformSequence() - - -@resolve_keyed_by_transforms.add -def attribution_keyed_by(config, jobs): - keyed_by_fields = ( - "fetches", - "attributes.release_artifacts", - "run.command", - "properties-with-locale", # properties-with-locale only exists in the l10n task - ) - for job in jobs: - build_platform = {"build-platform": job["attributes"]["build_platform"]} - for field in keyed_by_fields: - resolve_keyed_by(item=job, field=field, item_name=field, **build_platform) - yield job @transforms.add @@ -47,20 +30,3 @@ def stub_installer(config, jobs): "public/build/target.stub-installer.exe" ) yield job - - -@transforms.add -def mac_attribution(config, jobs): - """Adds \t padding to the mac attribution data. Implicitly assumes that the - attribution data is the last thing in job.run.command - """ - for job in jobs: - if "macosx" in job["attributes"]["build_platform"]: - # Last argument of command should be the attribution data - command = job["run"]["command"] - attribution_arg = command[-1] - while len(attribution_arg) < 1024: - attribution_arg += "\t" - command[-1] = attribution_arg - job["run"]["command"] = " ".join(command) - yield job diff --git a/taskcluster/gecko_taskgraph/transforms/repackage.py b/taskcluster/gecko_taskgraph/transforms/repackage.py index 61ac03f7ed42..90247a4a5f16 100644 --- a/taskcluster/gecko_taskgraph/transforms/repackage.py +++ b/taskcluster/gecko_taskgraph/transforms/repackage.py @@ -42,7 +42,7 @@ packaging_description_schema = Schema( Optional("shipping-product"): job_description_schema["shipping-product"], Optional("shipping-phase"): job_description_schema["shipping-phase"], Required("package-formats"): optionally_keyed_by( - "build-platform", "release-type", "build-type", [str] + "build-platform", "release-type", [str] ), Optional("msix"): { Optional("channel"): optionally_keyed_by( @@ -204,17 +204,6 @@ PACKAGE_FORMATS = { }, "output": "target.dmg", }, - "dmg-attrib": { - "args": [ - "dmg", - "--attribution_sentinel", - "__MOZCUSTOM__", - ], - "inputs": { - "input": "target{archive_format}", - }, - "output": "target.dmg", - }, "pkg": { "args": ["pkg"], "inputs": { @@ -321,7 +310,6 @@ def copy_in_useful_magic(config, jobs): job["build-platform"] = dep.attributes.get("build_platform") job["shipping-product"] = dep.attributes.get("shipping_product") - job["build-type"] = dep.attributes.get("build_type") yield job diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_partner.py b/taskcluster/gecko_taskgraph/transforms/repackage_partner.py index ad91d101da0d..b3e86b0ad2ff 100644 --- a/taskcluster/gecko_taskgraph/transforms/repackage_partner.py +++ b/taskcluster/gecko_taskgraph/transforms/repackage_partner.py @@ -23,6 +23,11 @@ from gecko_taskgraph.util.partners import get_partner_config_by_kind from gecko_taskgraph.util.platforms import archive_format, executable_extension from gecko_taskgraph.util.workertypes import worker_type_implementation + +def _by_platform(arg): + return optionally_keyed_by("build-platform", arg) + + # When repacking the stub installer we need to pass a zip file and package name to the # repackage task. This is not needed for vanilla stub but analogous to the full installer. PACKAGE_FORMATS = copy.deepcopy(PACKAGE_FORMATS_VANILLA) @@ -40,13 +45,11 @@ packaging_description_schema = Schema( # Shipping product and phase Optional("shipping-product"): task_description_schema["shipping-product"], Optional("shipping-phase"): task_description_schema["shipping-phase"], - Required("package-formats"): optionally_keyed_by( - "build-platform", "build-type", [str] - ), + Required("package-formats"): _by_platform([str]), # All l10n jobs use mozharness Required("mozharness"): { # Config files passed to the mozharness script - Required("config"): optionally_keyed_by("build-platform", [str]), + Required("config"): _by_platform([str]), # Additional paths to look for mozharness configs in. These should be # relative to the base of the source checkout Optional("config-paths"): [str],