Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-23 12:51:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1654994 - Switch full symbols archive to tar.zst. r=firefox-build-system-reviewers,rstewart,tomprince

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D85252
This commit is contained in:
Mike Hommey 2020-07-30 22:21:28 +00:00
parent 0ecfde84be
commit 311e167747
9 changed files with 34 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile.in
View File

@ -199,8 +199,8 @@ endif
.PHONY: symbolsfullarchive
symbolsfullarchive: prepsymbolsarchive
$(RM) '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
$(call py_action,symbols_archive,'$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' \
$(RM) '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst'
$(call py_action,symbols_archive,'$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst' \
$(abspath $(DIST)/crashreporter-symbols) \
--full-archive)
@ -224,7 +224,7 @@ endif
uploadsymbols:
ifdef MOZ_CRASHREPORTER
$(PYTHON3) -u $(topsrcdir)/toolkit/crashreporter/tools/upload_symbols.py '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
$(PYTHON3) -u $(topsrcdir)/toolkit/crashreporter/tools/upload_symbols.py '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst'
endif
.PHONY: update-packaging

2
build/.lldbinit.in
View File

@ -11,7 +11,7 @@ settings set symbols.enable-external-lookup true
settings append target.exec-search-paths @topobjdir@/toolkit/library
settings append target.exec-search-paths @topobjdir@/mozglue/build
# This is where artifact builds unpacks "crashreporter-symbols-full.zip" uncompressed ELF debug symbols.
# This is where artifact builds unpacks "crashreporter-symbols-full" uncompressed ELF debug symbols.
settings append target.debug-file-search-paths @topobjdir@/dist/crashreporter-symbols
# These are specific paths encoded into Mozilla's automation outputs.

2
python/mozbuild/mozbuild/artifacts.py
View File

@ -159,7 +159,7 @@ class ArtifactJob(object):
self._substs = substs
self._symbols_archive_suffix = None
if download_symbols == 'full':
self._symbols_archive_suffix = 'crashreporter-symbols-full.zip'
self._symbols_archive_suffix = 'crashreporter-symbols-full.tar.zst'
elif download_symbols:
self._symbols_archive_suffix = 'crashreporter-symbols.zip'
self._mozbuild = mozbuild

20
taskcluster/ci/upload-symbols/kind.yml
View File

@ -2,7 +2,9 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.single_dep:loader
loader: taskgraph.loader.multi_dep:loader
group-by: platform
transforms:
- taskgraph.transforms.upload_symbols:transforms
@ -11,24 +13,36 @@ transforms:
kind-dependencies:
- build
- toolchain
primary-dependency: build
not-for-build-platforms:
# Address Sanitizer builds don't have symbols to upload.
- linux64-asan-reporter-shippable/opt
- win64-asan-reporter-shippable/opt
- linux-geckodriver/opt
- linux64-geckodriver/opt
- macosx64-geckodriver/opt
- win32-geckodriver/opt
- win64-geckodriver/opt
job-template:
description: Upload Symbols
worker-type: b-linux
worker:
docker-image: {in-tree: "lint"}
docker-image: {in-tree: "debian10-base"}
max-run-time: 900
env:
# {level} gets replaced in the upload_symbols transform
SYMBOL_SECRET: "project/releng/gecko/build/level-{level}/gecko-symbol-upload"
MAKECAB: /builds/worker/fetches/makecab/makecab
run:
using: mach
mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py <build/public/build/target.crashreporter-symbols-full.zip>"}
mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py <build/public/build/target.crashreporter-symbols-full.tar.zst>"}
sparse-profile: upload-symbols
scopes:
- secrets:get:project/releng/gecko/build/level-{level}/gecko-symbol-upload
fetches:
toolchain:
- linux64-makecab

1
taskcluster/taskgraph/transforms/upload_symbols.py
View File

@ -40,6 +40,7 @@ def check_nightlies(config, tasks):
def fill_template(config, tasks):
for task in tasks:
dep = task['primary-dependency']
task.pop('dependent-tasks', None)
# Fill out the dynamic fields in the task description
task['label'] = dep.label + '-upload-symbols'

40
toolkit/crashreporter/tools/symbolstore.py
View File

@ -752,32 +752,10 @@ class Dumper_Win32(Dumper):
def CopyDebug(self, file, debug_file, guid, code_file, code_id):
file = locate_pdb(file)
def compress(path):
compressed_file = path[:-1] + "_"
# ignore makecab's output
makecab = buildconfig.substs["MAKECAB"]
wine = buildconfig.substs.get("WINE")
if wine and makecab.lower().endswith(".exe"):
cmd = [wine, makecab]
else:
cmd = [makecab]
success = subprocess.call(
cmd + ["-D", "CompressionType=MSZIP", path, compressed_file],
stdout=open(os.devnull, "w"),
stderr=subprocess.STDOUT,
)
if success == 0 and os.path.exists(compressed_file):
os.unlink(path)
return True
return False
rel_path = os.path.join(debug_file, guid, debug_file).replace("\\", "/")
full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
shutil.copyfile(file, full_path)
if compress(full_path):
print(rel_path[:-1] + "_")
else:
print(rel_path)
print(rel_path)
# Copy the binary file as well
if code_file and code_id:
@ -793,10 +771,7 @@ class Dumper_Win32(Dumper):
if e.errno != errno.EEXIST:
raise
shutil.copyfile(full_code_path, full_path)
if compress(full_path):
print(rel_path[:-1] + "_")
else:
print(rel_path)
print(rel_path)
def SourceServerIndexing(self, debug_file, guid, sourceFileStream, vcs_root):
# Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
@ -862,9 +837,7 @@ class Dumper_Linux(Dumper):
rel_path = os.path.join(debug_file, guid, debug_file + ".dbg")
full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path))
shutil.move(file_dbg, full_path)
# gzip the shipped debug files
os.system("gzip -4 -f %s" % full_path)
print(rel_path + ".gz")
print(rel_path)
else:
if os.path.isfile(file_dbg):
os.unlink(file_dbg)
@ -989,16 +962,15 @@ class Dumper_Mac(Dumper):
def CopyDebug(self, file, debug_file, guid, code_file, code_id):
"""ProcessFile has already produced a dSYM bundle, so we should just
copy that to the destination directory. However, we'll package it
into a .tar.bz2 because the debug symbols are pretty huge, and
also because it's a bundle, so it's a directory. |file| here is the
into a .tar because it's a bundle, so it's a directory. |file| here is
the original filename."""
dsymbundle = file + ".dSYM"
rel_path = os.path.join(
debug_file, guid, os.path.basename(dsymbundle) + ".tar.bz2"
debug_file, guid, os.path.basename(dsymbundle) + ".tar"
)
full_path = os.path.abspath(os.path.join(self.symbol_path, rel_path))
success = subprocess.call(
["tar", "cjf", full_path, os.path.basename(dsymbundle)],
["tar", "cf", full_path, os.path.basename(dsymbundle)],
cwd=os.path.dirname(dsymbundle),
stdout=open(os.devnull, "w"),
stderr=subprocess.STDOUT,

3
toolkit/crashreporter/tools/unit-symbolstore.py
View File

@ -192,7 +192,6 @@ class TestCopyDebug(HelperMixin, unittest.TestCase):
d.Process(os.path.join(self.test_dir, add_extension(["foo"])[0]))
self.assertEqual(1, len(copied))
@patch.dict("buildconfig.substs._dict", {"MAKECAB": "makecab"})
def test_copy_debug_copies_binaries(self):
"""
Test that CopyDebug copies binaries as well on Windows.
@ -219,7 +218,7 @@ class TestCopyDebug(HelperMixin, unittest.TestCase):
d.Process(test_file)
self.assertTrue(
os.path.isfile(
os.path.join(self.symbol_dir, code_file, code_id, code_file[:-1] + "_")
os.path.join(self.symbol_dir, code_file, code_id, code_file)
)
)

6
toolkit/mozapps/installer/packager.mk
View File

@ -60,9 +60,9 @@ ifdef MOZ_ARTIFACT_BUILD_SYMBOLS
cd $(DIST)/crashreporter-symbols && \
zip -r5D '../$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip' . -i '*.sym' -i '*.txt'
ifeq ($(MOZ_ARTIFACT_BUILD_SYMBOLS),full)
$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
cd $(DIST)/crashreporter-symbols && \
zip -r5D '../$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' .
$(call py_action,symbols_archive,'../$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst' \
$(abspath $(DIST)/crashreporter-symbols) \
--full-archive)
endif
endif # MOZ_ARTIFACT_BUILD_SYMBOLS
endif # MOZ_AUTOMATION

2
toolkit/mozapps/installer/upload-files.mk
View File

@ -380,7 +380,7 @@ UPLOAD_FILES= \
$(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \
$(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \
$(call QUOTED_WILDCARD,$(PKG_JSSHELL)) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip) \
$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).tar.zst) \
$(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup.exe) \
$(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup-stub.exe) \
$(call QUOTED_WILDCARD,$(topsrcdir)/toolchains.json) \