mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-28 12:45:27 +00:00
servo: Merge #12244 - Create .tar.gz package deterministically using Python's "tarfile" (from rev112:package-repro-pure); r=aneeshusa
<!-- Please describe your changes on the following line: --> A development of #12108, creates a .tar.gz package using the `tarfile` and `gzip` modules, without external dependencies. Fixes #11981. Also this fixes the issue when the existing `resources/` directory didn't allow to create a new package and failed with "File exists" error. --- <!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: --> - [X] `./mach build -d` does not report any errors - [X] `./mach test-tidy` does not report any errors - [X] These changes fix #11981 (github issue number if applicable). <!-- Either: --> - [ ] There are tests for these changes OR - [X] These changes do not require tests because "more general approach to reproducibility testing is needed" <!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. --> Source-Repo: https://github.com/servo/servo Source-Revision: 1086df67e04ce97fed080ba7c79ed8e1615202eb
This commit is contained in:
parent
22abbe3f7c
commit
55e08a6b34
@ -7,12 +7,16 @@
|
||||
# option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
import gzip
|
||||
import itertools
|
||||
import locale
|
||||
import os
|
||||
from os import path
|
||||
import contextlib
|
||||
import subprocess
|
||||
from subprocess import PIPE
|
||||
import sys
|
||||
import tarfile
|
||||
import platform
|
||||
|
||||
import toml
|
||||
@ -33,6 +37,55 @@ def cd(new_path):
|
||||
os.chdir(previous_path)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def setlocale(name):
|
||||
"""Context manager for changing the current locale"""
|
||||
saved_locale = locale.setlocale(locale.LC_ALL)
|
||||
try:
|
||||
yield locale.setlocale(locale.LC_ALL, name)
|
||||
finally:
|
||||
locale.setlocale(locale.LC_ALL, saved_locale)
|
||||
|
||||
|
||||
def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
|
||||
"""Create a .tar.gz archive in a deterministic (reproducible) manner.
|
||||
|
||||
See https://reproducible-builds.org/docs/archives/ for more details."""
|
||||
|
||||
def reset(tarinfo):
|
||||
"""Helper to reset owner/group and modification time for tar entries"""
|
||||
tarinfo.uid = tarinfo.gid = 0
|
||||
tarinfo.uname = tarinfo.gname = "root"
|
||||
tarinfo.mtime = 0
|
||||
return tarinfo
|
||||
|
||||
dest_archive = os.path.abspath(dest_archive)
|
||||
with cd(dir_to_archive):
|
||||
current_dir = "."
|
||||
file_list = [current_dir]
|
||||
for root, dirs, files in os.walk(current_dir):
|
||||
for name in itertools.chain(dirs, files):
|
||||
file_list.append(os.path.join(root, name))
|
||||
|
||||
# Sort file entries with the fixed locale
|
||||
with setlocale('C'):
|
||||
file_list.sort(cmp=locale.strcoll)
|
||||
|
||||
# Use a temporary file and atomic rename to avoid partially-formed
|
||||
# packaging (in case of exceptional situations like running out of disk space).
|
||||
# TODO do this in a temporary folder after #11983 is fixed
|
||||
temp_file = '{}.temp~'.format(dest_archive)
|
||||
with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0644), 'w') as out_file:
|
||||
with gzip.GzipFile('wb', fileobj=out_file, mtime=0) as gzip_file:
|
||||
with tarfile.open(fileobj=gzip_file, mode='w:') as tar_file:
|
||||
for entry in file_list:
|
||||
arcname = entry
|
||||
if prepend_path is not None:
|
||||
arcname = os.path.normpath(os.path.join(prepend_path, arcname))
|
||||
tar_file.add(entry, filter=reset, recursive=False, arcname=arcname)
|
||||
os.rename(temp_file, dest_archive)
|
||||
|
||||
|
||||
def host_triple():
|
||||
os_type = platform.system().lower()
|
||||
if os_type == "linux":
|
||||
|
@ -16,7 +16,6 @@ sys.path.append(path.join(path.dirname(sys.argv[0]), "components", "style", "pro
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
|
||||
from mach.registrar import Registrar
|
||||
from datetime import datetime
|
||||
@ -29,7 +28,14 @@ from mach.decorators import (
|
||||
|
||||
from mako.template import Template
|
||||
|
||||
from servo.command_base import CommandBase, cd, BuildNotFound, is_macosx, is_windows
|
||||
from servo.command_base import (
|
||||
archive_deterministically,
|
||||
BuildNotFound,
|
||||
cd,
|
||||
CommandBase,
|
||||
is_macosx,
|
||||
is_windows,
|
||||
)
|
||||
from servo.post_build_commands import find_dep_path_newest
|
||||
|
||||
|
||||
@ -205,7 +211,10 @@ class PackageCommands(CommandBase):
|
||||
else:
|
||||
dir_to_package = '/'.join(binary_path.split('/')[:-1])
|
||||
dir_to_root = '/'.join(binary_path.split('/')[:-3])
|
||||
shutil.copytree(dir_to_root + '/resources', dir_to_package + '/resources')
|
||||
resources_dir = dir_to_package + '/resources'
|
||||
if os.path.exists(resources_dir):
|
||||
delete(resources_dir)
|
||||
shutil.copytree(dir_to_root + '/resources', resources_dir)
|
||||
browserhtml_path = find_dep_path_newest('browserhtml', binary_path)
|
||||
if browserhtml_path is None:
|
||||
print("Could not find browserhtml package; perhaps you haven't built Servo.")
|
||||
@ -234,9 +243,9 @@ class PackageCommands(CommandBase):
|
||||
time = datetime.utcnow().replace(microsecond=0).isoformat()
|
||||
time = time.replace(':', "-")
|
||||
tar_path += time + "-servo-tech-demo.tar.gz"
|
||||
with tarfile.open(tar_path, "w:gz") as tar:
|
||||
# arcname is to add by relative rather than absolute path
|
||||
tar.add(dir_to_package, arcname='servo/')
|
||||
|
||||
archive_deterministically(dir_to_package, tar_path, prepend_path='servo/')
|
||||
|
||||
print("Packaged Servo into " + tar_path)
|
||||
|
||||
@Command('install',
|
||||
|
Loading…
Reference in New Issue
Block a user