From 311e167747e6b0cea647d79b9d9c6bfbf4d22f51 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 30 Jul 2020 22:21:28 +0000 Subject: [PATCH] Bug 1654994 - Switch full symbols archive to tar.zst. r=firefox-build-system-reviewers,rstewart,tomprince Differential Revision: https://phabricator.services.mozilla.com/D85252 --- Makefile.in | 6 +-- build/.lldbinit.in | 2 +- python/mozbuild/mozbuild/artifacts.py | 2 +- taskcluster/ci/upload-symbols/kind.yml | 20 ++++++++-- .../taskgraph/transforms/upload_symbols.py | 1 + toolkit/crashreporter/tools/symbolstore.py | 40 +++---------------- .../crashreporter/tools/unit-symbolstore.py | 3 +- toolkit/mozapps/installer/packager.mk | 6 +-- toolkit/mozapps/installer/upload-files.mk | 2 +- 9 files changed, 34 insertions(+), 48 deletions(-) diff --git a/Makefile.in b/Makefile.in index bfb6187c6b51..febccd5ecb1e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -199,8 +199,8 @@ endif .PHONY: symbolsfullarchive symbolsfullarchive: prepsymbolsarchive - $(RM) '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' - $(call py_action,symbols_archive,'$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' \ + $(RM) '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst' + $(call py_action,symbols_archive,'$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst' \ $(abspath $(DIST)/crashreporter-symbols) \ --full-archive) @@ -224,7 +224,7 @@ endif uploadsymbols: ifdef MOZ_CRASHREPORTER - $(PYTHON3) -u $(topsrcdir)/toolkit/crashreporter/tools/upload_symbols.py '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' + $(PYTHON3) -u $(topsrcdir)/toolkit/crashreporter/tools/upload_symbols.py '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst' endif .PHONY: update-packaging diff --git a/build/.lldbinit.in b/build/.lldbinit.in index 1dd536840334..3c566e0b999b 100644 --- a/build/.lldbinit.in +++ b/build/.lldbinit.in @@ -11,7 +11,7 @@ settings set symbols.enable-external-lookup true settings append target.exec-search-paths @topobjdir@/toolkit/library settings append target.exec-search-paths @topobjdir@/mozglue/build -# This is where artifact builds unpacks "crashreporter-symbols-full.zip" uncompressed ELF debug symbols. +# This is where artifact builds unpacks "crashreporter-symbols-full" uncompressed ELF debug symbols. settings append target.debug-file-search-paths @topobjdir@/dist/crashreporter-symbols # These are specific paths encoded into Mozilla's automation outputs. diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py index 6e6c429a8a1a..9608016c6e57 100644 --- a/python/mozbuild/mozbuild/artifacts.py +++ b/python/mozbuild/mozbuild/artifacts.py @@ -159,7 +159,7 @@ class ArtifactJob(object): self._substs = substs self._symbols_archive_suffix = None if download_symbols == 'full': - self._symbols_archive_suffix = 'crashreporter-symbols-full.zip' + self._symbols_archive_suffix = 'crashreporter-symbols-full.tar.zst' elif download_symbols: self._symbols_archive_suffix = 'crashreporter-symbols.zip' self._mozbuild = mozbuild diff --git a/taskcluster/ci/upload-symbols/kind.yml b/taskcluster/ci/upload-symbols/kind.yml index c7a8397b6a5d..655e2c83b8b7 100644 --- a/taskcluster/ci/upload-symbols/kind.yml +++ b/taskcluster/ci/upload-symbols/kind.yml @@ -2,7 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. --- -loader: taskgraph.loader.single_dep:loader +loader: taskgraph.loader.multi_dep:loader + +group-by: platform transforms: - taskgraph.transforms.upload_symbols:transforms @@ -11,24 +13,36 @@ transforms: kind-dependencies: - build + - toolchain + +primary-dependency: build not-for-build-platforms: # Address Sanitizer builds don't have symbols to upload. - linux64-asan-reporter-shippable/opt - win64-asan-reporter-shippable/opt + - linux-geckodriver/opt + - linux64-geckodriver/opt + - macosx64-geckodriver/opt + - win32-geckodriver/opt + - win64-geckodriver/opt job-template: description: Upload Symbols worker-type: b-linux worker: - docker-image: {in-tree: "lint"} + docker-image: {in-tree: "debian10-base"} max-run-time: 900 env: # {level} gets replaced in the upload_symbols transform SYMBOL_SECRET: "project/releng/gecko/build/level-{level}/gecko-symbol-upload" + MAKECAB: /builds/worker/fetches/makecab/makecab run: using: mach - mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py "} + mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py "} sparse-profile: upload-symbols scopes: - secrets:get:project/releng/gecko/build/level-{level}/gecko-symbol-upload + fetches: + toolchain: + - linux64-makecab diff --git a/taskcluster/taskgraph/transforms/upload_symbols.py b/taskcluster/taskgraph/transforms/upload_symbols.py index ceb5d2957faf..f2bc62330263 100644 --- a/taskcluster/taskgraph/transforms/upload_symbols.py +++ b/taskcluster/taskgraph/transforms/upload_symbols.py @@ -40,6 +40,7 @@ def check_nightlies(config, tasks): def fill_template(config, tasks): for task in tasks: dep = task['primary-dependency'] + task.pop('dependent-tasks', None) # Fill out the dynamic fields in the task description task['label'] = dep.label + '-upload-symbols' diff --git a/toolkit/crashreporter/tools/symbolstore.py b/toolkit/crashreporter/tools/symbolstore.py index 1a0e94ca92fa..c6a5a68f1795 100755 --- a/toolkit/crashreporter/tools/symbolstore.py +++ b/toolkit/crashreporter/tools/symbolstore.py @@ -752,32 +752,10 @@ class Dumper_Win32(Dumper): def CopyDebug(self, file, debug_file, guid, code_file, code_id): file = locate_pdb(file) - def compress(path): - compressed_file = path[:-1] + "_" - # ignore makecab's output - makecab = buildconfig.substs["MAKECAB"] - wine = buildconfig.substs.get("WINE") - if wine and makecab.lower().endswith(".exe"): - cmd = [wine, makecab] - else: - cmd = [makecab] - success = subprocess.call( - cmd + ["-D", "CompressionType=MSZIP", path, compressed_file], - stdout=open(os.devnull, "w"), - stderr=subprocess.STDOUT, - ) - if success == 0 and os.path.exists(compressed_file): - os.unlink(path) - return True - return False - rel_path = os.path.join(debug_file, guid, debug_file).replace("\\", "/") full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path)) shutil.copyfile(file, full_path) - if compress(full_path): - print(rel_path[:-1] + "_") - else: - print(rel_path) + print(rel_path) # Copy the binary file as well if code_file and code_id: @@ -793,10 +771,7 @@ class Dumper_Win32(Dumper): if e.errno != errno.EEXIST: raise shutil.copyfile(full_code_path, full_path) - if compress(full_path): - print(rel_path[:-1] + "_") - else: - print(rel_path) + print(rel_path) def SourceServerIndexing(self, debug_file, guid, sourceFileStream, vcs_root): # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing @@ -862,9 +837,7 @@ class Dumper_Linux(Dumper): rel_path = os.path.join(debug_file, guid, debug_file + ".dbg") full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path)) shutil.move(file_dbg, full_path) - # gzip the shipped debug files - os.system("gzip -4 -f %s" % full_path) - print(rel_path + ".gz") + print(rel_path) else: if os.path.isfile(file_dbg): os.unlink(file_dbg) @@ -989,16 +962,15 @@ class Dumper_Mac(Dumper): def CopyDebug(self, file, debug_file, guid, code_file, code_id): """ProcessFile has already produced a dSYM bundle, so we should just copy that to the destination directory. However, we'll package it - into a .tar.bz2 because the debug symbols are pretty huge, and - also because it's a bundle, so it's a directory. |file| here is the + into a .tar because it's a bundle, so it's a directory. |file| here is the original filename.""" dsymbundle = file + ".dSYM" rel_path = os.path.join( - debug_file, guid, os.path.basename(dsymbundle) + ".tar.bz2" + debug_file, guid, os.path.basename(dsymbundle) + ".tar" ) full_path = os.path.abspath(os.path.join(self.symbol_path, rel_path)) success = subprocess.call( - ["tar", "cjf", full_path, os.path.basename(dsymbundle)], + ["tar", "cf", full_path, os.path.basename(dsymbundle)], cwd=os.path.dirname(dsymbundle), stdout=open(os.devnull, "w"), stderr=subprocess.STDOUT, diff --git a/toolkit/crashreporter/tools/unit-symbolstore.py b/toolkit/crashreporter/tools/unit-symbolstore.py index b621ec85cbdb..e8da009f963c 100755 --- a/toolkit/crashreporter/tools/unit-symbolstore.py +++ b/toolkit/crashreporter/tools/unit-symbolstore.py @@ -192,7 +192,6 @@ class TestCopyDebug(HelperMixin, unittest.TestCase): d.Process(os.path.join(self.test_dir, add_extension(["foo"])[0])) self.assertEqual(1, len(copied)) - @patch.dict("buildconfig.substs._dict", {"MAKECAB": "makecab"}) def test_copy_debug_copies_binaries(self): """ Test that CopyDebug copies binaries as well on Windows. @@ -219,7 +218,7 @@ class TestCopyDebug(HelperMixin, unittest.TestCase): d.Process(test_file) self.assertTrue( os.path.isfile( - os.path.join(self.symbol_dir, code_file, code_id, code_file[:-1] + "_") + os.path.join(self.symbol_dir, code_file, code_id, code_file) ) ) diff --git a/toolkit/mozapps/installer/packager.mk b/toolkit/mozapps/installer/packager.mk index 94a3e30ea586..64040fdd8e01 100644 --- a/toolkit/mozapps/installer/packager.mk +++ b/toolkit/mozapps/installer/packager.mk @@ -60,9 +60,9 @@ ifdef MOZ_ARTIFACT_BUILD_SYMBOLS cd $(DIST)/crashreporter-symbols && \ zip -r5D '../$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip' . -i '*.sym' -i '*.txt' ifeq ($(MOZ_ARTIFACT_BUILD_SYMBOLS),full) - $(NSINSTALL) -D $(DIST)/$(PKG_PATH) - cd $(DIST)/crashreporter-symbols && \ - zip -r5D '../$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' . + $(call py_action,symbols_archive,'../$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst' \ + $(abspath $(DIST)/crashreporter-symbols) \ + --full-archive) endif endif # MOZ_ARTIFACT_BUILD_SYMBOLS endif # MOZ_AUTOMATION diff --git a/toolkit/mozapps/installer/upload-files.mk b/toolkit/mozapps/installer/upload-files.mk index ee08995ae92b..e5a7fd6d425c 100644 --- a/toolkit/mozapps/installer/upload-files.mk +++ b/toolkit/mozapps/installer/upload-files.mk @@ -380,7 +380,7 @@ UPLOAD_FILES= \ $(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \ $(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \ $(call QUOTED_WILDCARD,$(PKG_JSSHELL)) \ - $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip) \ + $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst) \ $(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup.exe) \ $(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup-stub.exe) \ $(call QUOTED_WILDCARD,$(topsrcdir)/toolchains.json) \