From b69023ce455310420a234c5082482e4e2c4bc3f9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 22 Aug 2026 14:17:51 +0200 Subject: [PATCH] create, import-tar: --digests computes digests over the full file content, see #4699 With "--digests ALGOS", borg computes hash digests over the full content of each regular file and stores them in the item's new "digests" dict (hash algorithm name -> digest), e.g. --digests=blake3 gives {"blake3": ...} (32 bytes). ALGOS is a comma-separated list of hash algorithm names - the same algorithms that are available as "borg list" format keys - or "none". Default: none, digests are opt-in. Unlike the chunk ids, such a digest depends neither on the chunker nor on borg's key, so it is the same digest that e.g. "b3sum" computes for that file. "borg list --format {blake3}" uses a stored digest if the item has one, instead of reading the file content from the repository. The content is hashed while it is read and processed anyway: the first chunk inline, from the second (big enough) chunk on by a background thread. Our hash implementations release the GIL, so the hashing can overlap with chunking, id hashing, compression and encryption instead of adding to their runtime. ContentDigester.update() takes ownership of the chunk data and releases it after hashing, so nothing is copied for this and the memory is still freed timely. Measured (12 cores, medians of 5 runs, relative to --digests=none, which is as fast as borg without this): 1GiB new 1GiB known 20000 small files --digests=blake3 free free +6..12% --digests=blake3,sha256 free +10% +16% --digests=blake3,sha256,sha512 +7% +60% +24% One background thread hashes all requested algorithms one after the other, so their combined throughput is the harmonic sum, while many small files are hashed inline by the main thread. Hashing is free exactly while it is faster than the rest of the pipeline - a single algorithm is, several may not be. That is why digests are opt-in. Digests are not computed if borg does not read the full content: an unchanged file gets them from the files cache and additional hard links from the first hard link, but "borg create --reuse-from" items do not have digests. Files that already are in the files cache without digests get them when they change (and thus are read again), so changing --digests only affects the files that borg reads. --- docs/changes.rst | 23 +++ docs/internals/data-structures.rst | 35 ++++ docs/usage/create.rst.inc | 3 + docs/usage/import-tar.rst.inc | 3 + src/borg/archive.py | 115 +++++++++-- src/borg/archiver/create_cmd.py | 14 +- src/borg/archiver/tar_cmds.py | 13 ++ src/borg/cache.py | 35 ++-- src/borg/constants.py | 12 ++ src/borg/digests.py | 135 +++++++++++++ src/borg/helpers/__init__.py | 1 + src/borg/helpers/parseformat.py | 22 ++- src/borg/item.pyi | 4 + src/borg/item.pyx | 6 + .../testsuite/archiver/create_cmd_test.py | 179 ++++++++++++++++++ src/borg/testsuite/archiver/tar_cmds_test.py | 47 ++++- src/borg/testsuite/cache_test.py | 6 +- src/borg/testsuite/digests_test.py | 138 ++++++++++++++ .../testsuite/helpers/parseformat_test.py | 12 ++ 19 files changed, 759 insertions(+), 44 deletions(-) create mode 100644 src/borg/digests.py create mode 100644 src/borg/testsuite/digests_test.py diff --git a/docs/changes.rst b/docs/changes.rst index dc83a2236..e03109940 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -167,6 +167,29 @@ Version 2.0.0b23 (not released yet) New features: +- create, import-tar: new option "--digests ALGOS", #4699. + + Computes hash digests over the full content of each regular file and stores them in + the item's new "digests" dict (algorithm name -> digest), e.g. --digests=blake3 gives + {"blake3": ...} (32 bytes). ALGOS is a comma-separated list of hash algorithm names - + the same algorithms that are available as "borg list" format keys - or "none". + Default: none, digests are opt-in. + + Unlike the chunk ids, such a digest depends neither on the chunker nor on borg's key, + so it is the same digest as e.g. "b3sum" of that file gives. "borg list --format + {blake3}" uses a stored digest if the item has one, instead of reading the file + content from the repository. + + The content is hashed while it is read and processed anyway, mostly by a background + thread, so one hash algorithm usually does not make borg create slower for bigger + files. Many small files are hashed by the main thread though, and several algorithms + are hashed one after the other, which the background thread may not be able to hide. + + Digests are not computed if borg does not read the full content: an unchanged file gets + them from the files cache and additional hard links from the first hard link, but + "borg create --reuse-from" items do not have digests. Files that already are in the + files cache without digests get them when they change (and thus are read again), so + changing --digests only affects the files that borg reads. - crypto: protect metadata and object header in the modes that do not encrypt, #9104. The "none" and "authenticated" modes had a no-op repo object envelope: only the chunk diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index 71ec05ae5..5a344f6aa 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -388,6 +388,41 @@ dictionary created by the ``Item`` class that contains: * xattrs * acl (various OS-dependent fields) * flags +* digests, hash digests over the full content of a regular file, see :ref:`item_digests` + +.. _item_digests: + +Item digests +~~~~~~~~~~~~ + +If asked to (``--digests ALGOS``, e.g. ``--digests=blake3``), ``borg create`` and +``borg import-tar`` compute hash digests over the full content of each regular file +and store them in the item's ``digests`` dict, mapping the hash algorithm name to +the digest, e.g. ``{"blake3": b"..."}`` (32 bytes / 256 bits for blake3). The digest +of the content is the same as the one an external tool like ``b3sum`` computes - in +contrast to the chunk ids, it does not depend on the chunker or on borg's key. + +The same algorithms are available as ``borg list`` format keys, and +``borg list --format "{blake3}"`` uses a stored digest if the item has one (otherwise +it reads the file content to compute it). + +Digests are off by default (``--digests=none``): the content is hashed while it is +read and processed anyway, mostly by a background thread (our hash implementations +release the GIL), so one hash algorithm usually does not make ``borg create`` slower +for bigger files - but many small files are hashed by the main thread, and several +algorithms are hashed one after the other, which the background thread may not be +able to hide. + +Digests are not computed if borg does not read the full content of a file: + +* an unchanged file (the files cache knows it) is not read again - borg takes the + digests from the files cache. Files that are already in the files cache without + digests only get them when they are read again (that is, when they change), so + changing ``--digests`` only affects files that borg reads. +* the additional hard links to a file are not read again, they get the digests of + the first one. +* ``borg create --reuse-from`` reuses chunks of a reference archive without reading + them, so the resulting item does not have digests. All items are serialized using msgpack and the resulting byte stream is fed into the same chunker algorithm as used for regular file data diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 2f44469d6..eaa7e98cd 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -91,6 +91,8 @@ borg create +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--sparse`` | detect sparse holes in input and seek over them instead of reading them | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--digests ALGOS`` | compute these hash digests over the full content of each file and store them into the archive items. Comma-separated list of hash algorithm names, e.g. "blake3", or "none". default: none | + +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--files-cache MODE`` | operate files cache in MODE. default: ctime,size,inode (on Windows: mtime,size,inode, because ctime is file creation time there). | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--files-changed MODE`` | specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there). | @@ -175,6 +177,7 @@ borg create --noacls do not read and store ACLs into archive --noxattrs do not read and store xattrs into archive --sparse detect sparse holes in input and seek over them instead of reading them + --digests ALGOS compute these hash digests over the full content of each file and store them into the archive items. Comma-separated list of hash algorithm names, e.g. "blake3", or "none". default: none --files-cache MODE operate files cache in MODE. default: ctime,size,inode (on Windows: mtime,size,inode, because ctime is file creation time there). --files-changed MODE specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there). --read-special open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. diff --git a/docs/usage/import-tar.rst.inc b/docs/usage/import-tar.rst.inc index b8e0bf9c7..74ddb49be 100644 --- a/docs/usage/import-tar.rst.inc +++ b/docs/usage/import-tar.rst.inc @@ -47,6 +47,8 @@ borg import-tar +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``-C COMPRESSION``, ``--compression COMPRESSION`` | select compression algorithm, see the output of the "borg help compression" command for details. | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--digests ALGOS`` | compute these hash digests over the full content of each file and store them into the archive items. Comma-separated list of hash algorithm names, e.g. "blake3", or "none". default: none | + +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ .. raw:: html @@ -81,6 +83,7 @@ borg import-tar --timestamp TIMESTAMP manually specify the archive creation date/time (yyyy-mm-ddThh:mm:ss[(+|-)HH:MM] format, (+|-)HH:MM is the UTC offset, default: local time zone). Alternatively, give a reference file/directory. --chunker-params PARAMS specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, NC_LEVEL). default: fastcdc,19,23,21,2 -C COMPRESSION, --compression COMPRESSION select compression algorithm, see the output of the "borg help compression" command for details. + --digests ALGOS compute these hash digests over the full content of each file and store them into the archive items. Comma-separated list of hash algorithm names, e.g. "blake3", or "none". default: none Description diff --git a/src/borg/archive.py b/src/borg/archive.py index 1e14df1b7..6eac61fb8 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -26,6 +26,7 @@ from .chunkers import get_chunker, Chunk, release_chunk_data from .cache import ChunkListEntry, build_chunkindex_from_repo, write_chunkindex_to_repo from .crypto.key import key_factory, UnsupportedPayloadError from .constants import * # NOQA +from .digests import ContentDigester from .crypto.low_level import IntegrityError as IntegrityErrorBase from .helpers import BackupError, BackupRaceConditionError, BackupItemExcluded from .helpers import BackupSymlinkParentError, BackupPathTraversalError @@ -1337,7 +1338,15 @@ class ChunksProcessor: self.add_item = add_item self.rechunkify = rechunkify - def process_file_chunks(self, item, cache, stats, show_progress, chunk_iter, chunk_processor=None, append=False): + def process_file_chunks( + self, item, cache, stats, show_progress, chunk_iter, chunk_processor=None, append=False, digester=None + ): + """ + Process all chunks of an item's content. + + :param digester: if given, the content is also fed to this ContentDigester (see #4699). + Only the default chunk_processor does that, a given one is on its own. + """ if not chunk_processor: def chunk_processor(chunk): @@ -1347,7 +1356,12 @@ class ChunksProcessor: try: chunk_entry = cache.add_chunk(chunk_id, {}, data, stats=stats, ro_type=ROBJ_FILE_STREAM) finally: - release_chunk_data(data) + if digester is None: + release_chunk_data(data) + else: + # the digester takes ownership of data and releases it after hashing. + # we are done with data here, so it can be hashed in the background. + digester.update(data) return chunk_entry if not append: @@ -1457,6 +1471,7 @@ class FilesystemObjectProcessors: read_special_timeout=None, input_map=None, reuse_chunks=None, + digest_algos=DIGEST_ALGOS_DEFAULT, ): self.metadata_collector = metadata_collector self.cache = cache @@ -1470,10 +1485,12 @@ class FilesystemObjectProcessors: self.input_map = input_map # --map: content range info for the single input file, see #4363 self.reuse_chunks = reuse_chunks # --reuse-from: the reference archive item's chunk list, see #4363 - self.hlm = HardLinkManager(id_type=tuple, info_type=(list, type(None))) # (dev, ino) -> chunks or None + self.hlm = HardLinkManager(id_type=tuple, info_type=tuple) # (dev, ino) -> (chunks, digests) self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) self.cwd = os.getcwd() self.chunker = get_chunker(*chunker_params, key=key, sparse=sparse) + # computes item.digests (--digests) while we read a file's content + self.digester = ContentDigester(digest_algos) @contextmanager def create_helper(self, path, st, status=None, hardlinkable=True, strip_prefix=None): @@ -1498,11 +1515,13 @@ class FilesystemObjectProcessors: if hardlinked: status = "h" # hard link nothing = object() - chunks = self.hlm.retrieve(id=(st.st_ino, st.st_dev), default=nothing) - if chunks is nothing: + info = self.hlm.retrieve(id=(st.st_ino, st.st_dev), default=nothing) + if info is nothing: update_map = True - elif chunks is not None: - hl_chunks = chunks + else: + hl_chunks, hl_digests = info + if hl_digests is not None and self.digester.enabled: + item.digests = hl_digests # same content as the first hard link, thus same digests item.hlid = self.hlm.hardlink_id_from_inode(ino=st.st_ino, dev=st.st_dev) yield item, status, hardlinked, hl_chunks maybe_exclude_by_attr(item) @@ -1511,7 +1530,8 @@ class FilesystemObjectProcessors: # remember the hlid of this fs object and if the item has chunks, # also remember them, so we do not have to re-chunk a hard link. chunks = item.chunks if "chunks" in item else None - self.hlm.remember(id=(st.st_ino, st.st_dev), info=chunks) + digests = item.digests if "digests" in item else None + self.hlm.remember(id=(st.st_ino, st.st_dev), info=(chunks, digests)) def process_dir_with_fd(self, *, path, fd, st, strip_prefix): with self.create_helper(path, st, "d", hardlinkable=False, strip_prefix=strip_prefix) as ( @@ -1616,7 +1636,18 @@ class FilesystemObjectProcessors: item.uid = uid if gid is not None: item.gid = gid - self.process_file_chunks(item, cache, self.stats, self.show_progress, backup_io_iter(self.chunker.chunkify(fd))) + self.digester.start() + self.process_file_chunks( + item, + cache, + self.stats, + self.show_progress, + backup_io_iter(self.chunker.chunkify(fd)), + digester=self.digester, + ) + digests = self.digester.finish() + if digests is not None: + item.digests = digests item.get_size(memorize=True) self.stats.nfiles += 1 self.add_item(item, stats=self.stats) @@ -1663,19 +1694,19 @@ class FilesystemObjectProcessors: if self.input_map is not None: # --map: the given map replaces the files cache as content/change information, see #4363. hashed_path = path_hash = None - known, chunks = False, None + known, chunks, digests = False, None, None elif not is_special_file: hashed_path = safe_encode(item.path) # path as in archive item! started_hashing = time.monotonic() path_hash = self.key.id_hash(hashed_path) self.stats.hashing_time += time.monotonic() - started_hashing - known, chunks = cache.file_known_and_unchanged(hashed_path, path_hash, st) + known, chunks, digests = cache.file_known_and_unchanged(hashed_path, path_hash, st) else: # in --read-special mode, we may be called for special files. # there should be no information in the cache about special files processed in # read-special mode, but we better play safe as this was wrong in the past: hashed_path = path_hash = None - known, chunks = False, None + known, chunks, digests = False, None, None if chunks is not None: # Make sure all ids are available for chunk in chunks: @@ -1689,6 +1720,10 @@ class FilesystemObjectProcessors: # process one-by-one, so we will know in item.chunks how far we got cache.reuse_chunk(chunk.id, chunk.size, self.stats) item.chunks.append(chunk) + if digests is not None and self.digester.enabled: + # the file did not change, so its digests are still valid and we + # do not have to read the file to compute them again. + item.digests = StableDict(digests) status = "U" # regular file, unchanged else: status = "M" if known else "A" # regular file, modified or added @@ -1696,6 +1731,7 @@ class FilesystemObjectProcessors: # Only chunkify the file if needed changed_while_backup = False if "chunks" not in item: + self.digester.start() start_reading = time.time_ns() # Do NOT wrap this in backup_io("read"): the source-file reads are already # guarded individually by backup_io_iter() below. Wrapping the whole call would @@ -1717,6 +1753,9 @@ class FilesystemObjectProcessors: # without reading them. Each "read" part gets its own chunkify call, so # chunks never span the gap left by reused parts. plan = build_reuse_plan(self.input_map, self.reuse_chunks, input_size, cache.seen_chunk) + if any(kind == "reuse" for kind, payload in plan): + # reused chunks are not read, so we can not digest the full content. + self.digester.discard() item.chunks = [] for kind, payload in plan: if kind == "reuse": @@ -1733,26 +1772,45 @@ class FilesystemObjectProcessors: self.show_progress, backup_io_iter(chunk_iter), append=True, + digester=self.digester, ) else: fmap = [(start, length, state == MAP_DATA) for start, length, state in self.input_map] chunk_iter = self.chunker.chunkify(None, fd, fmap=fmap, st=st) self.process_file_chunks( - item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) + item, + cache, + self.stats, + self.show_progress, + backup_io_iter(chunk_iter), + digester=self.digester, ) elif read_special_timeout is not None: # all reads go through the timeout-enforcing wrapper (fh stays unused). chunk_iter = self.chunker.chunkify(SpecialFileReader(fd, read_special_timeout), st=st) self.process_file_chunks( - item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) + item, + cache, + self.stats, + self.show_progress, + backup_io_iter(chunk_iter), + digester=self.digester, ) else: # passing st saves FileReader a stat call; regular files take the # direct read path, special files (--read-special) the buffered one. chunk_iter = self.chunker.chunkify(None, fd, st=st) self.process_file_chunks( - item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) + item, + cache, + self.stats, + self.show_progress, + backup_io_iter(chunk_iter), + digester=self.digester, ) + digests = self.digester.finish() + if digests is not None: + item.digests = digests self.stats.chunking_time = self.chunker.chunking_time end_reading = time.time_ns() with backup_io("fstat2"): @@ -1795,7 +1853,7 @@ class FilesystemObjectProcessors: # block or char device will change without its mtime/size/inode changing. # also, we must not memorize a potentially inconsistent/corrupt file that # changed while we backed it up. - cache.memorize_file(hashed_path, path_hash, st, item.chunks) + cache.memorize_file(hashed_path, path_hash, st, item.chunks, digests) self.stats.files_stats[status] += 1 # must be done late if not changed_while_backup: status = None # we already called print_file_status @@ -1816,6 +1874,7 @@ class TarfileObjectProcessors: show_progress, log_json, file_status_printer=None, + digest_algos=DIGEST_ALGOS_DEFAULT, ): self.cache = cache self.key = key @@ -1826,7 +1885,9 @@ class TarfileObjectProcessors: self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) self.chunker = get_chunker(*chunker_params, key=key, sparse=False) - self.hlm = HardLinkManager(id_type=str, info_type=list) # normalized/safe path -> chunks + self.hlm = HardLinkManager(id_type=str, info_type=tuple) # normalized/safe path -> (chunks, digests) + # computes item.digests (--digests) while we read a file's content + self.digester = ContentDigester(digest_algos) @contextmanager def create_helper(self, tarinfo, status=None, type=None): @@ -1906,9 +1967,12 @@ class TarfileObjectProcessors: # create a not hardlinked borg item, reusing the chunks, see HardLinkManager.__doc__ normalized_path = posixpath.normpath(tarinfo.linkname) safe_path = make_path_safe(normalized_path) - chunks = self.hlm.retrieve(safe_path) - if chunks is not None: + info = self.hlm.retrieve(safe_path) + if info is not None: + chunks, digests = info item.chunks = chunks + if digests is not None and self.digester.enabled: + item.digests = digests # same content as the file we link to, thus same digests item.get_size(memorize=True, from_chunks=True) self.stats.nfiles += 1 return status @@ -1918,13 +1982,22 @@ class TarfileObjectProcessors: self.print_file_status(status, item.path) status = None # we already printed the status fd = tar.extractfile(tarinfo) + self.digester.start() self.process_file_chunks( - item, self.cache, self.stats, self.show_progress, backup_io_iter(self.chunker.chunkify(fd)) + item, + self.cache, + self.stats, + self.show_progress, + backup_io_iter(self.chunker.chunkify(fd)), + digester=self.digester, ) + digests = self.digester.finish() + if digests is not None: + item.digests = digests item.get_size(memorize=True, from_chunks=True) self.stats.nfiles += 1 # we need to remember ALL files, see HardLinkManager.__doc__ - self.hlm.remember(id=item.path, info=item.chunks) + self.hlm.remember(id=item.path, info=(item.chunks, digests)) return status diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 0bd2d8f43..0d19f16df 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -16,7 +16,7 @@ from ..archive import FilesystemObjectProcessors, MetadataCollector, ChunksProce from ..cache import Cache from ..constants import * # NOQA from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec -from ..helpers import archivename_validator, FilesCacheMode, files_cache_mode_no_ctime +from ..helpers import archivename_validator, DigestAlgos, FilesCacheMode, files_cache_mode_no_ctime from ..helpers import octal_int, nonnegative_seconds from ..helpers import read_input_map from ..helpers import eval_escapes @@ -360,6 +360,7 @@ class CreateMixIn: read_special_timeout=read_special_timeout, input_map=input_map, reuse_chunks=reuse_chunks, + digest_algos=args.digest_algos, ) create_inner(archive, cache, fso) args.stats |= args.json @@ -1197,6 +1198,17 @@ class CreateMixIn: action="store_true", help="detect sparse holes in input and seek over them instead of reading them", ) + fs_group.add_argument( + "--digests", + metavar="ALGOS", + dest="digest_algos", + action=Highlander, + type=DigestAlgos, + default=DIGEST_ALGOS_DEFAULT, + help="compute these hash digests over the full content of each file and store them into " + 'the archive items. Comma-separated list of hash algorithm names, e.g. "blake3", or "none". ' + "default: %s" % (",".join(DIGEST_ALGOS_DEFAULT) or "none"), + ) fs_group.add_argument( "--files-cache", metavar="MODE", diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index 51594c43d..02b5969dc 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -22,6 +22,7 @@ from ..helpers import msgpack from ..helpers import create_filter_process from ..helpers import ChunkIteratorFileWrapper from ..helpers import archivename_validator, comment_validator, PathSpec, ChunkerParams, CompressionSpec +from ..helpers import DigestAlgos from ..helpers import FilesystemPathSpec from ..helpers import remove_surrogates from ..helpers import timestamp, archive_ts_now @@ -557,6 +558,7 @@ class TarMixIn: cache=cache, key=key, process_file_chunks=cp.process_file_chunks, + digest_algos=args.digest_algos, add_item=archive.add_item, chunker_params=args.chunker_params, show_progress=args.progress, @@ -834,6 +836,17 @@ class TarMixIn: action=Highlander, help="select compression algorithm, see the output of the " '"borg help compression" command for details.', ) + archive_group.add_argument( + "--digests", + metavar="ALGOS", + dest="digest_algos", + action=Highlander, + type=DigestAlgos, + default=DIGEST_ALGOS_DEFAULT, + help="compute these hash digests over the full content of each file and store them into " + 'the archive items. Comma-separated list of hash algorithm names, e.g. "blake3", or "none". ' + "default: %s" % (",".join(DIGEST_ALGOS_DEFAULT) or "none"), + ) subparser.add_argument("name", metavar="NAME", type=archivename_validator, help="specify the archive name") subparser.add_argument( diff --git a/src/borg/cache.py b/src/borg/cache.py index b018879b3..c49277d74 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -69,8 +69,9 @@ def discover_files_cache_names(path, files_cache_name="files"): return [p.name for p in path.iterdir() if p.name.startswith(files_cache_name + ".")] -# chunks is a list of ChunkListEntry -FileCacheEntry = namedtuple("FileCacheEntry", "age inode size ctime mtime chunks") +# chunks is a list of ChunkListEntry, digests is a dict (see item.digests) or None. +# digests has a default, so that a files cache written by a borg without it still loads. +FileCacheEntry = namedtuple("FileCacheEntry", "age inode size ctime mtime chunks digests", defaults=(None,)) def cache_dir(repository, path=None): @@ -334,6 +335,7 @@ class FilesCacheMixin: ctime=int_to_timestamp(ctime_ns), mtime=int_to_timestamp(mtime_ns), chunks=item.chunks, + digests=item.get("digests"), ) # note: if the repo is an a valid state, next line should not fail with KeyError: files[path_hash] = self.compress_entry(entry) @@ -456,25 +458,27 @@ class FilesCacheMixin: :param hashed_path: the file's path as we gave it to hash(hashed_path) :param path_hash: hash(hashed_path), to save some memory in the files cache :param st: the file's stat() result - :return: known, chunks (known is True if we have infos about this file in the cache, - chunks is a list[ChunkListEntry] IF the file has not changed, otherwise None). + :return: known, chunks, digests (known is True if we have infos about this file in the cache, + chunks is a list[ChunkListEntry] IF the file has not changed, otherwise None, + digests is the file's digests dict IF the file has not changed and we have + them, otherwise None). """ if not stat.S_ISREG(st.st_mode): - return False, None + return False, None, None cache_mode = self.cache_mode if "d" in cache_mode: # d(isabled) files_cache_logger.debug("UNKNOWN: files cache disabled") - return False, None + return False, None, None # note: r(echunk) does not need the files cache in this method, but the files cache will # be updated and saved to disk to memorize the files. To preserve previous generations in # the cache, this means that it also needs to get loaded from disk first. if "r" in cache_mode: # r(echunk) files_cache_logger.debug("UNKNOWN: rechunking enforced") - return False, None + return False, None, None entry = self.files.get(path_hash) if not entry: files_cache_logger.debug("UNKNOWN: no file metadata in cache for: %r", hashed_path) - return False, None + return False, None, None # we know the file! try: entry = self.decompress_entry(entry) @@ -482,21 +486,21 @@ class FilesCacheMixin: # the cached entry references a chunk that is no longer in the chunks index (e.g. after # an aborted / out-of-space backup); treat the file as unknown so it gets re-chunked. files_cache_logger.debug("UNKNOWN: cached entry references a missing chunk: %r", hashed_path) - return False, None + return False, None, None if "s" in cache_mode and entry.size != st.st_size: files_cache_logger.debug("KNOWN-CHANGED: file size has changed: %r", hashed_path) - return True, None + return True, None, None if "i" in cache_mode and entry.inode != st.st_ino: files_cache_logger.debug("KNOWN-CHANGED: file inode number has changed: %r", hashed_path) - return True, None + return True, None, None ctime = int_to_timestamp(safe_ns(st.st_ctime_ns)) if "c" in cache_mode and entry.ctime != ctime: files_cache_logger.debug("KNOWN-CHANGED: file ctime has changed: %r", hashed_path) - return True, None + return True, None, None mtime = int_to_timestamp(safe_ns(st.st_mtime_ns)) if "m" in cache_mode and entry.mtime != mtime: files_cache_logger.debug("KNOWN-CHANGED: file mtime has changed: %r", hashed_path) - return True, None + return True, None, None # V = any of the inode number, mtime, ctime values. # we ignored V in the comparison above or it is still the same value. # if it is still the same, replacing it in the tuple doesn't change it. @@ -508,9 +512,9 @@ class FilesCacheMixin: entry = entry._replace(inode=st.st_ino, ctime=ctime, mtime=mtime, age=0) self.files[path_hash] = self.compress_entry(entry) chunks = [ChunkListEntry(*chunk) for chunk in entry.chunks] # convert to list of namedtuple - return True, chunks + return True, chunks, entry.digests - def memorize_file(self, hashed_path, path_hash, st, chunks): + def memorize_file(self, hashed_path, path_hash, st, chunks, digests=None): if not stat.S_ISREG(st.st_mode): return # note: r(echunk) modes will update the files cache, d(isabled) mode won't @@ -526,6 +530,7 @@ class FilesCacheMixin: ctime=int_to_timestamp(ctime_ns), mtime=int_to_timestamp(mtime_ns), chunks=chunks, + digests=digests, ) self.files[path_hash] = self.compress_entry(entry) self._newest_cmtime = max(self._newest_cmtime or 0, ctime_ns) diff --git a/src/borg/constants.py b/src/borg/constants.py index fb8483fb8..5f8a512a6 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -1,14 +1,26 @@ +import hashlib + # this set must be kept complete, otherwise the RobustUnpacker might malfunction: # fmt: off ITEM_KEYS = frozenset(['path', 'source', 'target', 'rdev', 'chunks', 'chunks_healthy', 'hardlink_master', 'hlid', 'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'birthtime', 'size', 'inode', 'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended', + 'digests', 'part']) # fmt: on # this is the set of keys that are always present in items: REQUIRED_ITEM_KEYS = frozenset(["path", "mtime"]) +# hash algorithms borg can compute over the content of a file: the ones from python's stdlib +# (except shake_*, which use an incompatible .digest() to support variable length) and blake3. +# used for "borg create --digests" (item.digests) and for the "borg list" format keys. +HASH_ALGORITHMS = frozenset(hashlib.algorithms_guaranteed - {"shake_128", "shake_256"}) | {"blake3"} + +# what "borg create --digests" computes if the user does not give that option: +# nothing - digests are opt-in, computing them costs time (esp. for many small files). +DIGEST_ALGOS_DEFAULT = () + # this set must be kept complete, otherwise rebuild_manifest might malfunction: # fmt: off ARCHIVE_KEYS = frozenset(['version', 'name', 'hostname', 'username', diff --git a/src/borg/digests.py b/src/borg/digests.py new file mode 100644 index 000000000..19d04a6fd --- /dev/null +++ b/src/borg/digests.py @@ -0,0 +1,135 @@ +"""Compute digests over the full content of a file, see #4699.""" + +import hashlib +import queue +import threading + +from blake3 import blake3 + +from .chunkers.reader import release_chunk_data +from .constants import HASH_ALGORITHMS +from .helpers.datastruct import StableDict +from .logger import create_logger + +logger = create_logger() + + +def new_hasher(algo): + """give a fresh hasher for the hash algorithm (one of HASH_ALGORITHMS)""" + # blake3 hashes single-threaded here (that is its default): we are off the critical path + # anyway and must not compete with borg's own threads for cores. + return blake3() if algo == "blake3" else hashlib.new(algo) + + +class ContentDigester: + """ + Compute digests over the full content of a file, fed chunk by chunk. + + Usage: start() a file, update() it with all its content chunks in order, then either + finish() it to get the digests or discard() it if the content was not fed completely. + + Hashing starts out inline, so that small (single chunk) files do not pay for thread + setup at all. If more chunks follow and they are big enough for the handover to pay + off, the hashing is taken over by a background thread: our hash implementations release + the GIL, so the hashing then overlaps with borg's chunking / id hashing / compression / + encryption of the following chunks rather than adding to their runtime. + + A digester without algorithms ("borg create --digests=none", the default) computes nothing, + but still accepts (and releases) the content given to it. + """ + + # from this chunk size on, giving the hashing to the background thread pays off. + # below it, the per-chunk handover overhead would dominate, so we keep hashing inline. + THREAD_MIN_CHUNK_SIZE = 64 * 1024 + + # how many chunks may wait for the background thread. bounds the memory held by chunks + # that were already processed by the main thread, but not hashed yet. + QUEUE_SIZE = 4 + + def __init__(self, algos): + """:param algos: the hash algorithms to compute, see HASH_ALGORITHMS. empty: compute nothing.""" + assert set(algos) <= HASH_ALGORITHMS, f"unsupported hash algorithm(s): {set(algos) - HASH_ALGORITHMS}" + self.algos = tuple(algos) + self._hashers = None # None: we're not computing digests (no file started or content incomplete) + self._thread = None + self._queue = None + self._big_chunks = False # is handing the hashing over to a background thread worth it? + + @property + def enabled(self): + """do we compute digests at all? (if not, digests must not be stored into items either)""" + return bool(self.algos) + + def start(self): + """begin a new file, forgetting about a previously started one""" + self._stop_thread() + self._hashers = {algo: new_hasher(algo) for algo in self.algos} if self.algos else None + self._big_chunks = False + + def update(self, data): + """ + feed the next piece of content of the current file (pieces must be given in order) + + Takes ownership of data: the caller must not use it afterwards, it is released + (see release_chunk_data) as soon as it has been hashed. + """ + if self._hashers is None: # we're not computing digests for this file + release_chunk_data(data) + elif self._thread is not None: + self._queue.put(data) # the background thread hashes and releases it + elif self._big_chunks: # 2nd big chunk: from here on, let a background thread do the hashing + self._start_thread() + self._queue.put(data) + else: + self._big_chunks = len(data) >= self.THREAD_MIN_CHUNK_SIZE + self._update(data) + + def discard(self): + """the current file's content is not fed completely, so do not compute digests for it""" + self._stop_thread() + self._hashers = None + + def finish(self): + """ + finish the current file + + :return: StableDict mapping the algorithm name to the digest, None if there are no digests. + """ + self._stop_thread() # waits until all content given to the thread has been hashed + if self._hashers is None: + return None + digests = StableDict((algo, hasher.digest()) for algo, hasher in self._hashers.items()) + self._hashers = None + return digests + + def _update(self, data): + """hash data (if we still compute digests for this file) and release it""" + try: + if self._hashers is not None: + for hasher in self._hashers.values(): + hasher.update(data) + finally: + release_chunk_data(data) + + def _worker(self): + while True: + data = self._queue.get() + if data is None: # sentinel: that was all the content of this file + break + try: + self._update(data) + except Exception: # hashing is not expected to fail, but we must not die with a full queue + logger.warning("Computing content digests failed, not storing any.", exc_info=True) + self._hashers = None # the main thread does not touch this while we run, see _stop_thread + + def _start_thread(self): + self._queue = queue.Queue(maxsize=self.QUEUE_SIZE) + self._thread = threading.Thread(target=self._worker, daemon=True) + self._thread.start() + + def _stop_thread(self): + if self._thread is not None: + self._queue.put(None) + self._thread.join() + self._thread = None + self._queue = None diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 2f7420eca..ff8a5a8a5 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -40,6 +40,7 @@ from .parseformat import ( SortBySpec, CompressionSpec, ChunkerParams, + DigestAlgos, FilesCacheMode, files_cache_mode_no_ctime, partial_format, diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index fe2ab1965..a9e94076b 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -414,6 +414,20 @@ def ChunkerParams(s): raise ArgumentTypeError("invalid chunker params") +def DigestAlgos(s): + """--digests: parse a comma-separated list of hash algorithm names (or "none")""" + if isinstance(s, (list, tuple)): # the default value is given as a tuple already, see ChunkerParams + return tuple(s) + if s == "none": + return () + algos = tuple(sorted(set(s.strip().split(",")))) + if not set(algos) <= HASH_ALGORITHMS: + raise ArgumentTypeError( + 'digests must be "none" or a comma-separated list of: %s' % ",".join(sorted(HASH_ALGORITHMS)) + ) + return algos + + def FilesCacheMode(s): ENTRIES_MAP = dict(ctime="c", mtime="m", size="s", inode="i", rechunk="r", disabled="d") VALID_MODES = ("cis", "ims", "cs", "ms", "cr", "mr", "d", "s") # letters in alpha order @@ -1151,9 +1165,7 @@ class ArchiveFormatter(BaseFormatter): class ItemFormatter(BaseFormatter): - # we provide the hash algos from python stdlib (except shake_*) and blake3. - # shake_* is not provided because it uses an incompatible .digest() method to support variable length. - hash_algorithms = set(hashlib.algorithms_guaranteed).difference({"shake_128", "shake_256"}) | {"blake3"} + hash_algorithms = HASH_ALGORITHMS # the same algos that "borg create --digests" can compute KEY_DESCRIPTIONS = { "type": "file type (file, dir, symlink, ...)", "mode": "file mode (as in stat)", @@ -1284,6 +1296,10 @@ class ItemFormatter(BaseFormatter): def hash_item(self, hash_function, item): if "chunks" not in item: return "" + digest = item.get("digests", {}).get(hash_function) + if digest is not None: + # borg create computed this over the file content, so we do not need to read it again. + return bin_to_hex(digest) if hash_function == "blake3": hash = blake3() else: diff --git a/src/borg/item.pyi b/src/borg/item.pyi index 1bc17a0dd..f3c917733 100644 --- a/src/borg/item.pyi +++ b/src/borg/item.pyi @@ -149,6 +149,10 @@ class Item(PropDict): @xattrs.setter def xattrs(self, val: StableDict) -> None: ... @property + def digests(self) -> StableDict: ... + @digests.setter + def digests(self, val: StableDict) -> None: ... + @property def acl_access(self) -> bytes: ... @acl_access.setter def acl_access(self, val: bytes) -> None: ... diff --git a/src/borg/item.pyx b/src/borg/item.pyx index b8388e892..0abc8077d 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -293,6 +293,9 @@ cdef class Item(PropDict): xattrs = PropDictProperty(StableDict) + # digests over the full content of the item, hash algo name (e.g. "blake3") -> 32 bytes digest + digests = PropDictProperty(StableDict) + deleted = PropDictProperty(bool) nlink = PropDictProperty(int) @@ -374,6 +377,9 @@ cdef class Item(PropDict): v = fix_timestamp(v) if k in ('acl_access', 'acl_default', 'acl_extended', 'acl_nfs4'): v = fix_bytes_value(d, k) + if k == 'digests': + if not isinstance(v, StableDict): + v = StableDict(v) if k == 'xattrs': if not isinstance(v, StableDict): v = StableDict(v) diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index 6f1d0f235..ed98549f0 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -1,4 +1,5 @@ import errno +import hashlib import json import os import tempfile @@ -8,6 +9,7 @@ import stat import subprocess import pytest +from blake3 import blake3 from ... import platform from ...constants import * # NOQA @@ -32,6 +34,7 @@ from .. import ( ) from . import ( cmd, + open_archive, generate_archiver_tests, create_test_files, assert_dirs_equal, @@ -1824,3 +1827,179 @@ def test_create_map_reuse_from_errors(archivers, request): "test", "input/file", ) + + +def item_digests(archiver, archive="test"): + """{path: item.digests} as STORED in the items of the newest archive of a series. + + Note: "borg list --format {blake3}" can not be used for this - it falls back to + reading and hashing the file content if the item does not have digests. + """ + archives = cmd(archiver, "repo-list", "--format", "{name} {id}{NL}").splitlines() + archive_id = [line.split()[1] for line in archives if line.split()[0] == archive][-1] + archive_obj, repository = open_archive(archiver.repository_path, f"aid:{archive_id}") + with repository: + return {item.path: item.get("digests") for item in archive_obj.iter_items()} + + +def blake3_digests(data): + """what item.digests must look like for content """ + return {"blake3": blake3(data).digest()} + + +def test_create_digests(archivers, request): + # borg create computes a digest over the full content of each file, see #4699 + archiver = request.getfixturevalue(archivers) + contents = { + "empty": b"", + "small": b"small file contents", # single chunk, hashed by the main thread + "big": os.urandom(512 * 1024), # multiple big chunks, hashed by a background thread + } + for name, data in contents.items(): + create_regular_file(archiver.input_path, name, contents=data) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "--digests=blake3", "--chunker-params=fixed,131072", "test", "input") + digests = item_digests(archiver) + for name, data in contents.items(): + assert digests[f"input/{name}"] == blake3_digests(data) + assert digests["input"] is None # a directory has no content and thus no digests + + +def test_create_digests_from_files_cache(archivers, request): + # an unchanged file is not read again, so its digests must come from the files cache + archiver = request.getfixturevalue(archivers) + contents = b"some file contents" + create_regular_file(archiver.input_path, "file1", contents=contents) + granularity_sleep() # file2 must have newer timestamps than file1, see test_file_status + create_regular_file(archiver.input_path, "file2", contents=b"newer file") + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "--digests=blake3", "test", "input") + output = cmd(archiver, "create", "--digests=blake3", "--list", "--filter=U", "test", "input") + assert "U input/file1" in output # not read again + assert item_digests(archiver)["input/file1"] == blake3_digests(contents) + + +@requires_hardlinks +def test_create_digests_hardlink(archivers, request): + # the 2nd hard link is not read again, its digests come from the first one + archiver = request.getfixturevalue(archivers) + contents = b"some file contents" + create_regular_file(archiver.input_path, "file1", contents=contents) + os.link(os.path.join(archiver.input_path, "file1"), os.path.join(archiver.input_path, "file2")) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "--digests=blake3", "test", "input") + digests = item_digests(archiver) + assert digests["input/file1"] == digests["input/file2"] == blake3_digests(contents) + + +def test_create_digests_holes(archivers, request): + # a hole is not read, but it is part of the file content, so it must be digested as zeros + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + block_size = 4096 + data = os.urandom(block_size) + contents = data + b"\0" * block_size + create_regular_file(archiver.input_path, "file", contents=contents) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {block_size} data\n") + f.write(f"{block_size} {block_size} zero\n") # a hole, borg does not read it + cmd( + archiver, + "create", + "--digests=blake3", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "test", + "input/file", + ) + assert item_digests(archiver)["input/file"] == blake3_digests(contents) + + +def test_create_digests_reuse_from(archivers, request): + # --reuse-from reuses chunks without reading them, so we can not digest the full content + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + block_size = 4096 + blocks = [os.urandom(block_size) for _ in range(2)] + create_regular_file(archiver.input_path, "file", contents=blocks[0] + blocks[1]) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {2 * block_size} data\n") + cmd( + archiver, "create", "--digests=blake3", "--chunker-params", "fixed,4096", "--map", map_path, "ref", "input/file" + ) + assert item_digests(archiver, "ref")["input/file"] == blake3_digests(blocks[0] + blocks[1]) + new_block = os.urandom(block_size) + create_regular_file(archiver.input_path, "file", contents=blocks[0] + new_block) + with open(map_path, "w") as f: + f.write(f"0 {block_size} same\n") + f.write(f"{block_size} {block_size} data\n") + cmd( + archiver, + "create", + "--digests=blake3", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "--reuse-from", + "ref", + "test", + "input/file", + ) + assert item_digests(archiver, "test")["input/file"] is None + + +def test_create_digests_stdin(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + input_data = b"some data from a pipe" + cmd(archiver, "create", "--digests=blake3", "test", "-", input=input_data) + assert item_digests(archiver)["stdin"] == blake3_digests(input_data) + + +def test_create_digests_algos(archivers, request): + # --digests selects the hash algorithms to compute + archiver = request.getfixturevalue(archivers) + contents = b"some file contents" + create_regular_file(archiver.input_path, "file1", contents=contents) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "--digests", "blake3,sha256", "test", "input") + assert item_digests(archiver)["input/file1"] == { + "blake3": blake3(contents).digest(), + "sha256": hashlib.sha256(contents).digest(), + } + + +def test_create_digests_default_is_none(archivers, request): + # digests are opt-in: without --digests, borg computes and stores none + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some file contents") + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input") + assert item_digests(archiver)["input/file1"] is None + + +def test_create_digests_none(archivers, request): + # --digests=none does not compute digests and does not store known ones either + archiver = request.getfixturevalue(archivers) + contents = b"some file contents" + create_regular_file(archiver.input_path, "file1", contents=contents) + granularity_sleep() # file2 must have newer timestamps than file1, see test_file_status + create_regular_file(archiver.input_path, "file2", contents=b"newer file") + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "--digests=blake3", "test", "input") # this one has digests + output = cmd(archiver, "create", "--digests=none", "--list", "--filter=U", "test", "input") + assert "U input/file1" in output + # neither computed nor taken from the files cache (which has them from the first run): + assert item_digests(archiver)["input/file1"] is None + + +def test_create_digests_invalid_algo(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + output = cmd(archiver, "create", "--digests=nosuchhash", "test", "input", exit_code=2) + assert "digests must be" in output and "blake3" in output # the error lists the valid algorithms diff --git a/src/borg/testsuite/archiver/tar_cmds_test.py b/src/borg/testsuite/archiver/tar_cmds_test.py index 0888f944f..8f763891d 100644 --- a/src/borg/testsuite/archiver/tar_cmds_test.py +++ b/src/borg/testsuite/archiver/tar_cmds_test.py @@ -1,3 +1,4 @@ +import hashlib import io import json import os @@ -8,6 +9,7 @@ import sys import tarfile import pytest +from blake3 import blake3 from ... import xattr from ...archiver.tar_cmds import chunks_to_sparse_info, gnu_sparse_10_map, SparseTarInfo @@ -15,7 +17,7 @@ from ...constants import * # NOQA from ...helpers import Error from ...item import ChunkListEntry from .. import changedir -from . import assert_dirs_equal, _extract_hardlinks_setup, cmd, requires_hardlinks, RK_ENCRYPTION +from . import assert_dirs_equal, _extract_hardlinks_setup, cmd, open_archive, requires_hardlinks, RK_ENCRYPTION from . import create_test_files, create_regular_file from . import generate_archiver_tests from ...platform import acl_get, acl_set @@ -154,6 +156,49 @@ def test_import_tar_nfiles(archivers, request): assert info["archives"][0]["stats"]["nfiles"] == 3 +def tar_item_digests(archiver, archive): + """{path: item.digests} as STORED in the items of an archive, see create_cmd_test.item_digests""" + archive_obj, repository = open_archive(archiver.repository_path, archive) + with repository: + return {item.path: item.get("digests") for item in archive_obj.iter_items()} + + +def test_import_tar_digests(archivers, request): + # import-tar computes item.digests like borg create does, see #4699 + archiver = request.getfixturevalue(archivers) + contents = {"dir/file1": os.urandom(512 * 1024), "dir/file2": b"small"} + with tarfile.open("input.tar", "w") as tar: + for name, data in contents.items(): + tarinfo = tarfile.TarInfo(name) + tarinfo.size = len(data) + tar.addfile(tarinfo, io.BytesIO(data)) + tarinfo = tarfile.TarInfo("dir/hardlink1") # no content of its own, links to dir/file1 + tarinfo.type = tarfile.LNKTYPE + tarinfo.linkname = "dir/file1" + tar.addfile(tarinfo) + tarinfo = tarfile.TarInfo("dir/subdir") + tarinfo.type = tarfile.DIRTYPE + tar.addfile(tarinfo) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "import-tar", "--digests=blake3", "--chunker-params=fixed,131072", "dst", "input.tar") + digests = tar_item_digests(archiver, "dst") + for name, data in contents.items(): + assert digests[name] == {"blake3": blake3(data).digest()} + # the hard link has no content in the tar, it gets the digests of the file it links to + assert digests["dir/hardlink1"] == {"blake3": blake3(contents["dir/file1"]).digest()} + assert digests["dir/subdir"] is None # a directory has no content and thus no digests + + cmd(archiver, "import-tar", "--digests", "sha256", "dst-sha256", "input.tar") + digests = tar_item_digests(archiver, "dst-sha256") + assert digests["dir/file1"] == {"sha256": hashlib.sha256(contents["dir/file1"]).digest()} + + cmd(archiver, "import-tar", "--digests=none", "dst-none", "input.tar") + assert tar_item_digests(archiver, "dst-none")["dir/file1"] is None + + cmd(archiver, "import-tar", "dst-default", "input.tar") # digests are opt-in + assert tar_item_digests(archiver, "dst-default")["dir/file1"] is None + + def test_import_unusual_tar(archivers, request): archiver = request.getfixturevalue(archivers) diff --git a/src/borg/testsuite/cache_test.py b/src/borg/testsuite/cache_test.py index 872499c19..24e64488d 100644 --- a/src/borg/testsuite/cache_test.py +++ b/src/borg/testsuite/cache_test.py @@ -88,7 +88,7 @@ class TestAdHocWithFilesCache: def test_files_cache(self, cache): st = os.stat(".") - assert cache.file_known_and_unchanged(b"foo", bytes(32), st) == (False, None) + assert cache.file_known_and_unchanged(b"foo", bytes(32), st) == (False, None, None) assert cache.cache_mode == "d" assert cache.files == {} @@ -663,14 +663,14 @@ def test_files_cache_save_tolerates_missing_chunk(tmp_path, monkeypatch): cache.memorize_file(os.fspath(realfile), path_hash, st, [(cid, 4)]) # while the chunk is present, the file resolves as known/unchanged - known, chunks = cache.file_known_and_unchanged(os.fspath(realfile), path_hash, st) + known, chunks, digests = cache.file_known_and_unchanged(os.fspath(realfile), path_hash, st) assert known and chunks # simulate the out-of-space rollback: the referenced chunk is gone from the index del cache.chunks[cid] # a lookup must now report the file as unknown (re-chunk it), not raise KeyError - known, chunks = cache.file_known_and_unchanged(os.fspath(realfile), path_hash, st) + known, chunks, digests = cache.file_known_and_unchanged(os.fspath(realfile), path_hash, st) assert known is False and chunks is None # saving the files cache must drop the broken entry rather than raise KeyError diff --git a/src/borg/testsuite/digests_test.py b/src/borg/testsuite/digests_test.py new file mode 100644 index 000000000..d7cae6a8c --- /dev/null +++ b/src/borg/testsuite/digests_test.py @@ -0,0 +1,138 @@ +import hashlib +import os + +import pytest +from blake3 import blake3 + +from ..digests import ContentDigester, new_hasher +from ..constants import HASH_ALGORITHMS + + +def mv(data): + """give data like a chunker does: as a memoryview that gets released after use""" + return memoryview(data) + + +def digest_of(*pieces, digester=None): + digester = digester if digester is not None else ContentDigester(("blake3",)) + digester.start() + for piece in pieces: + digester.update(mv(piece)) + return digester.finish() + + +def test_empty(): + assert digest_of() == {"blake3": blake3(b"").digest()} + + +def test_inline(): + # small pieces are hashed by the calling thread + assert digest_of(b"foo", b"bar") == {"blake3": blake3(b"foobar").digest()} + + +def test_threaded(): + # pieces >= THREAD_MIN_CHUNK_SIZE give the hashing to a background thread + pieces = [os.urandom(ContentDigester.THREAD_MIN_CHUNK_SIZE) for _ in range(5)] + assert digest_of(*pieces) == {"blake3": blake3(b"".join(pieces)).digest()} + + +def test_threaded_more_than_queue_size(): + # the producer must not deadlock when it queues more than QUEUE_SIZE pieces + pieces = [os.urandom(ContentDigester.THREAD_MIN_CHUNK_SIZE) for _ in range(4 * ContentDigester.QUEUE_SIZE)] + assert digest_of(*pieces) == {"blake3": blake3(b"".join(pieces)).digest()} + + +def test_discard(): + digester = ContentDigester(("blake3",)) + digester.start() + digester.update(mv(b"foo")) + digester.discard() + assert digester.finish() is None + + +def test_discard_before_update(): + # --reuse-from discards before the (partial) content is fed + digester = ContentDigester(("blake3",)) + digester.start() + digester.discard() + digester.update(mv(b"foo")) + assert digester.finish() is None + + +def test_discard_threaded(): + digester = ContentDigester(("blake3",)) + digester.start() + big = os.urandom(ContentDigester.THREAD_MIN_CHUNK_SIZE) + digester.update(mv(big)) + digester.update(mv(big)) # from here on, a background thread hashes + digester.discard() + assert digester.finish() is None + + +def test_reuse_after_finish(): + # one digester is used for all the files of a "borg create" run + digester = ContentDigester(("blake3",)) + big = os.urandom(ContentDigester.THREAD_MIN_CHUNK_SIZE) + for pieces in [(b"foo",), (big, big), (), (b"bar", b"baz"), (big, big, b"x")]: + assert digest_of(*pieces, digester=digester) == {"blake3": blake3(b"".join(pieces)).digest()} + + +def test_start_after_incomplete_file(): + # if a file's processing dies (e.g. an OSError while reading), the next start() cleans up + digester = ContentDigester(("blake3",)) + digester.start() + big = os.urandom(ContentDigester.THREAD_MIN_CHUNK_SIZE) + digester.update(mv(big)) + digester.update(mv(big)) # a background thread is running now + assert digest_of(b"foo", digester=digester) == {"blake3": blake3(b"foo").digest()} + + +def test_data_is_released(): + # the digester takes ownership of the data and must release it (see release_chunk_data) + digester = ContentDigester(("blake3",)) + digester.start() + big = os.urandom(ContentDigester.THREAD_MIN_CHUNK_SIZE) + inline, threaded = mv(big), mv(big) + digester.update(inline) # hashed by the calling thread + digester.update(threaded) # hashed by the background thread + digester.finish() + for view in (inline, threaded): + try: + len(view) + except ValueError: + pass # released, as expected + else: + raise AssertionError("memoryview was not released") + + +def test_multiple_algos(): + digester = ContentDigester(("blake3", "sha256")) + assert digest_of(b"foo", b"bar", digester=digester) == { + "blake3": blake3(b"foobar").digest(), + "sha256": hashlib.sha256(b"foobar").digest(), + } + + +def test_no_algos(): + # "borg create --digests=none": the content is still given to us, but we compute nothing + digester = ContentDigester(()) + assert not digester.enabled + assert digest_of(b"foo", os.urandom(ContentDigester.THREAD_MIN_CHUNK_SIZE), digester=digester) is None + + +def test_enabled(): + assert ContentDigester(("sha256",)).enabled + assert not ContentDigester(()).enabled # "--digests=none", borg's default + + +def test_unsupported_algo(): + with pytest.raises(AssertionError): + ContentDigester(("nosuchhash",)) + + +def test_all_supported_algos(): + # all the algorithms the user may give to --digests must work with our hashers + for algo in HASH_ALGORITHMS: + hasher = new_hasher(algo) + hasher.update(b"foo") + assert isinstance(hasher.digest(), bytes) diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index 37fdbb69e..61b7a3936 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -30,6 +30,7 @@ from ...helpers.parseformat import ( swidth_slice, eval_escapes, ChunkerParams, + DigestAlgos, files_cache_mode_no_ctime, get_size_units, normalize_local_path, @@ -975,3 +976,14 @@ def test_json_dump_indent(monkeypatch, env_value, expect_newlines): else: assert "\n" not in result assert json.loads(result) == obj + + +def test_digest_algos(): + assert DigestAlgos("blake3") == ("blake3",) + assert DigestAlgos("sha256,blake3") == ("blake3", "sha256") # sorted and deduplicated + assert DigestAlgos("blake3,blake3") == ("blake3",) + assert DigestAlgos("none") == () + assert DigestAlgos(("blake3",)) == ("blake3",) # the default value comes in as a tuple + for invalid in ("nosuchhash", "blake3,nosuchhash", "shake_128", ""): + with pytest.raises(ArgumentTypeError): + DigestAlgos(invalid)