From 74622607beb137c4340975f2b4086e1ca4843765 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Aug 2026 02:51:18 +0200 Subject: [PATCH] create: add --map to give an input map, see #4363 An input map describes the content ranges of the single input file: data ranges are read and stored, zero ranges are stored as holes without reading them. Primary use case: backing up snapshots of large (esp. thin-provisioned) block devices, where the storage layer knows which ranges are in use. Co-Authored-By: Claude Fable 5 --- docs/man/borg-create.1 | 29 ++++++++++ docs/usage/create.rst.inc | 28 ++++++++++ src/borg/archive.py | 20 ++++++- src/borg/archiver/create_cmd.py | 56 +++++++++++++++++++ src/borg/helpers/__init__.py | 1 + src/borg/helpers/fs.py | 53 ++++++++++++++++++ .../testsuite/archiver/create_cmd_test.py | 54 +++++++++++++++++- src/borg/testsuite/helpers/fs_test.py | 51 +++++++++++++++++ 8 files changed, 289 insertions(+), 3 deletions(-) diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index d0356f3a0..029bba9c3 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -320,6 +320,9 @@ open and read block and char device files as well as FIFOs as if they were regul .TP .BI \-\-read\-special\-timeout \ SECONDS when reading from FIFOs or character devices (see \-\-read\-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO\(aqs writer to connect). Give 0 to wait forever. default: 1800 seconds. +.TP +.BI \-\-map \ MAPFILE +give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the \fIInput maps\fP section below. .UNINDENT .SS Archive options .INDENT 0.0 @@ -581,6 +584,32 @@ creation a bit. .sp By default, the content read from stdin is stored in a file called \(aqstdin\(aq. Use \fB\-\-stdin\-name\fP to change the name. +.SS Input maps +.sp +Usually, borg reads the complete input to determine its contents. If you already +know the contents of parts of the input from an external source of truth, you can +give that information via \fB\-\-map MAPFILE\fP and borg will not read the known parts. +The primary use case is backing up snapshots of large block devices (e.g. LVM thin +volumes), where the storage layer knows which ranges are in use. +.sp +\fB\-\-map\fP requires giving exactly one input path, which must be a regular file or +(with \fB\-\-read\-special\fP) a block device. +.sp +The map file must describe the whole input: one range per line, in the form +\fBSTART LENGTH STATE\fP (byte values, decimal or 0x\-prefixed hexadecimal). The +ranges must be sorted, non\-overlapping and contiguous, starting at offset 0 and +covering the exact input size. \fB#\fP starts a comment, empty lines are ignored. +STATE is one of: +.INDENT 0.0 +.IP \(bu 2 +\fBdata\fP: the range\(aqs contents are read and backed up. +.IP \(bu 2 +\fBzero\fP: the range is known to read as all\-zero bytes. borg stores a hole +(all\-zero range) of that size without reading the range. +.UNINDENT +.sp +\fBThe map is trusted\fP: if it is wrong (e.g. a range marked \fBzero\fP actually +contains data), the archive will not match the input and borg cannot detect that. .SS Feeding all file paths from externally .sp Usually, you give a starting path (recursion root) to borg and then borg diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 70d9ad95a..43403e1a6 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -99,6 +99,8 @@ borg create +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--read-special-timeout SECONDS`` | when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds. | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--map MAPFILE`` | give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the *Input maps* section below. | + +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | **Archive options** | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--comment COMMENT`` | add a comment text to the archive | @@ -173,6 +175,7 @@ borg create --files-changed MODE specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there). --read-special open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. --read-special-timeout SECONDS when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds. + --map MAPFILE give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the *Input maps* section below. Archive options @@ -445,6 +448,31 @@ creation a bit. By default, the content read from stdin is stored in a file called 'stdin'. Use ``--stdin-name`` to change the name. +Input maps +++++++++++ + +Usually, borg reads the complete input to determine its contents. If you already +know the contents of parts of the input from an external source of truth, you can +give that information via ``--map MAPFILE`` and borg will not read the known parts. +The primary use case is backing up snapshots of large block devices (e.g. LVM thin +volumes), where the storage layer knows which ranges are in use. + +``--map`` requires giving exactly one input path, which must be a regular file or +(with ``--read-special``) a block device. + +The map file must describe the whole input: one range per line, in the form +``START LENGTH STATE`` (byte values, decimal or 0x-prefixed hexadecimal). The +ranges must be sorted, non-overlapping and contiguous, starting at offset 0 and +covering the exact input size. ``#`` starts a comment, empty lines are ignored. +STATE is one of: + +- ``data``: the range's contents are read and backed up. +- ``zero``: the range is known to read as all-zero bytes. borg stores a hole + (all-zero range) of that size without reading the range. + +**The map is trusted**: if it is wrong (e.g. a range marked ``zero`` actually +contains data), the archive will not match the input and borg cannot detect that. + Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++ diff --git a/src/borg/archive.py b/src/borg/archive.py index 23dcc7ed5..fec7a21cf 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -43,6 +43,7 @@ from .helpers import bin_to_hex from .helpers import safe_ns from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi, get_progress_dt from .helpers import os_open, flags_normal, flags_dir, O_, SpecialFileReader +from .helpers import MAP_DATA, input_map_check_size from .helpers import os_stat from .helpers import msgpack from .helpers.lrucache import LRUCache @@ -1388,6 +1389,7 @@ class FilesystemObjectProcessors: file_status_printer=None, files_changed="mtime" if is_win32 else "ctime", read_special_timeout=None, + input_map=None, ): self.metadata_collector = metadata_collector self.cache = cache @@ -1398,6 +1400,7 @@ class FilesystemObjectProcessors: self.print_file_status = file_status_printer or (lambda *args: None) self.files_changed = files_changed self.read_special_timeout = read_special_timeout + self.input_map = input_map # --map: content range info for the single input file, see #4363 self.hlm = HardLinkManager(id_type=tuple, info_type=(list, type(None))) # (dev, ino) -> chunks or None self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) @@ -1589,7 +1592,11 @@ class FilesystemObjectProcessors: chunk_entry = cache.reuse_chunk(chunk_id, chunk_size, self.stats) item.chunks.append(chunk_entry) else: # normal case, no "2nd+" hard link - if not is_special_file: + if self.input_map is not None: + # --map: the given map replaces the files cache as content/change information, see #4363. + hashed_path = path_hash = None + known, chunks = False, None + elif not is_special_file: hashed_path = safe_encode(item.path) # path as in archive item! started_hashing = time.monotonic() path_hash = self.key.id_hash(hashed_path) @@ -1630,7 +1637,16 @@ class FilesystemObjectProcessors: # and still commit the archive -- referencing chunks that were never durably # stored. An unwrapped repository OSError is critical and aborts create before # archive.save() runs (see the BackupOSError docstring). - if read_special_timeout is not None: + if self.input_map is not None: + # --map: read only the "data" ranges, store "zero" ranges as holes without + # reading them, see #4363. Non-seekable inputs (fifo/chr) are rejected earlier. + with backup_io("seek"): + input_size = st.st_size if stat.S_ISREG(st.st_mode) else os.lseek(fd, 0, os.SEEK_END) + os.lseek(fd, 0, os.SEEK_SET) + input_map_check_size(self.input_map, input_size) + fmap = [(start, length, state == MAP_DATA) for start, length, state in self.input_map] + chunk_iter = self.chunker.chunkify(None, fd, fmap=fmap, st=st) + elif read_special_timeout is not None: # all reads go through the timeout-enforcing wrapper (fh stays unused). chunk_iter = self.chunker.chunkify(SpecialFileReader(fd, read_special_timeout), st=st) else: diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 1212a739d..248cc5e1f 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -18,6 +18,7 @@ from ..constants import * # NOQA from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec from ..helpers import archivename_validator, FilesCacheMode, files_cache_mode_no_ctime from ..helpers import octal_int, nonnegative_seconds +from ..helpers import read_input_map from ..helpers import eval_escapes from ..helpers import timestamp, archive_ts_now from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify @@ -71,6 +72,27 @@ class CreateMixIn: read_special_timeout = READ_SPECIAL_TIMEOUT_DEFAULT if read_special_timeout == 0: read_special_timeout = None # wait forever + input_map = None + if args.input_map is not None: + # --map only makes sense for a single, seekable input file, see #4363. + if args.paths_from_stdin or args.paths_from_command or args.paths_from_shell_command: + raise CommandError("--map cannot be used with --paths-from-*.") + if args.content_from_command: + raise CommandError("--map cannot be used with --content-from-command.") + if len(args.paths) != 1: + raise CommandError("--map requires exactly one input path.") + if args.paths[0] == "-": + raise CommandError("--map cannot be used with stdin input.") + try: + st_map = os.stat(args.paths[0], follow_symlinks=True) + except OSError as e: + raise CommandError(f"--map input: {args.paths[0]}: {e}") + if stat.S_ISBLK(st_map.st_mode): + if not args.read_special: + raise CommandError("--map with a block device requires --read-special.") + elif not stat.S_ISREG(st_map.st_mode): + raise CommandError("--map input must be a regular file or a block device.") + input_map = read_input_map(args.input_map) if is_win32: # st_ctime is the file *creation* time on Windows, not the "metadata change time", # so a ctime based files cache mode would not detect content changes of a file that @@ -315,6 +337,7 @@ class CreateMixIn: file_status_printer=self.print_file_status, files_changed=args.files_changed, read_special_timeout=read_special_timeout, + input_map=input_map, ) create_inner(archive, cache, fso) args.stats |= args.json @@ -955,6 +978,31 @@ class CreateMixIn: By default, the content read from stdin is stored in a file called 'stdin'. Use ``--stdin-name`` to change the name. + Input maps + ++++++++++ + + Usually, borg reads the complete input to determine its contents. If you already + know the contents of parts of the input from an external source of truth, you can + give that information via ``--map MAPFILE`` and borg will not read the known parts. + The primary use case is backing up snapshots of large block devices (e.g. LVM thin + volumes), where the storage layer knows which ranges are in use. + + ``--map`` requires giving exactly one input path, which must be a regular file or + (with ``--read-special``) a block device. + + The map file must describe the whole input: one range per line, in the form + ``START LENGTH STATE`` (byte values, decimal or 0x-prefixed hexadecimal). The + ranges must be sorted, non-overlapping and contiguous, starting at offset 0 and + covering the exact input size. ``#`` starts a comment, empty lines are ignored. + STATE is one of: + + - ``data``: the range's contents are read and backed up. + - ``zero``: the range is known to read as all-zero bytes. borg stores a hole + (all-zero range) of that size without reading the range. + + **The map is trusted**: if it is wrong (e.g. a range marked ``zero`` actually + contains data), the archive will not match the input and borg cannot detect that. + Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++ @@ -1146,6 +1194,14 @@ class CreateMixIn: "for a FIFO's writer to connect). Give 0 to wait forever. default: %d seconds." % READ_SPECIAL_TIMEOUT_DEFAULT, ) + fs_group.add_argument( + "--map", + metavar="MAPFILE", + dest="input_map", + action=Highlander, + help="give a map file describing the content ranges of the (single) input file, " + "so borg does not need to read all of it. See the *Input maps* section below.", + ) archive_group = subparser.add_argument_group("Archive options") archive_group.add_argument( diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index eb2a97609..2f7420eca 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -24,6 +24,7 @@ from .fs import get_security_dir, get_keys_dir, get_base_dir, get_cache_dir, get from .fs import dir_is_tagged, dir_is_cachedir, remove_dotdot_prefixes, make_path_safe, scandir_inorder from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, get_strip_prefix, umount, slashify from .fs import SpecialFileReader +from .fs import MAP_DATA, MAP_ZERO, MAP_SAME, read_input_map, input_map_check_size from .fs import O_, flags_dir, flags_dir_follow, flags_special_follow, flags_special from .fs import flags_base, flags_normal, flags_normal_follow, flags_noatime from .fs import HardLinkManager diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index 653371a1b..e7a3d44eb 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -640,6 +640,59 @@ class SpecialFileReader: raise OSError(errno.ESPIPE, os.strerror(errno.ESPIPE)) +# input map range states (borg create --map), see #4363. +MAP_DATA = "data" # range contains data, read and store it +MAP_ZERO = "zero" # range is known to read as all-zero, store a hole without reading it +MAP_SAME = "same" # range is known unchanged vs. a reference archive, reuse its chunks (--reuse-from) + + +def read_input_map(path, *, allow_same=False): + """ + Read and validate an input map file (borg create --map), see #4363. + + Format: one range per line: "START LENGTH STATE" (decimal or 0x-prefixed hex), + STATE being data, zero or same. '#' starts a comment, empty lines are ignored. + The ranges must be sorted, non-overlapping and contiguous, starting at offset 0. + + Returns a list of (start, length, state) tuples. + """ + states = {MAP_DATA, MAP_ZERO} | ({MAP_SAME} if allow_same else set()) + input_map = [] + expected_start = 0 + with open(path) as f: + for lineno, line in enumerate(f, start=1): + line = line.split("#", 1)[0].strip() + if not line: + continue + err = f"{path}:{lineno}: invalid input map line" + try: + start_s, length_s, state = line.split() + start, length = int(start_s, 0), int(length_s, 0) + except ValueError: + raise Error(f"{err}: expected 'START LENGTH STATE', got: {line!r}") from None + if state not in states: + raise Error(f"{err}: invalid state {state!r} (expected: {', '.join(sorted(states))})") + if length <= 0: + raise Error(f"{err}: LENGTH must be positive") + if start != expected_start: + raise Error( + f"{err}: ranges must be sorted, contiguous and start at offset 0 " + f"(expected START {expected_start}, got {start})" + ) + input_map.append((start, length, state)) + expected_start = start + length + if not input_map: + raise Error(f"{path}: empty input map") + return input_map + + +def input_map_check_size(input_map, size): + """Check that *input_map* covers [0, size) exactly - a mismatch means the map does not belong to this input.""" + covered = input_map[-1][0] + input_map[-1][1] + if covered != size: + raise Error(f"input map covers {covered} bytes, but the input has {size} bytes") + + def umount(mountpoint): from . import set_ec diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index 30edb17d4..fd26e818b 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -17,7 +17,7 @@ from ...platform import is_win32, is_cygwin from ...platformflags import is_msystem from ...repository import Repository from ...helpers import CommandError, BackupPermissionError, BackupTimeoutError, BackupBrokenSymlinkError -from ...helpers import BackupWarning +from ...helpers import BackupWarning, Error from .. import has_lchflags, has_mknod from .. import changedir from .. import ( @@ -1554,3 +1554,55 @@ def test_exclude_nodump_dir_with_file(archivers, request): list_output = cmd(archiver, "list", "test", "--short") assert "input/nd\n" not in list_output assert "input/nd/file_in_ndir\n" not in list_output + + +def test_create_map(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + block = 4096 + data_a, data_b, data_c = os.urandom(block), os.urandom(block), os.urandom(block) + # data_b is real data, but the map claims that range reads as zero: + # borg must not read "zero" ranges, so the archive must contain zeros there. + create_regular_file(archiver.input_path, "file", contents=data_a + data_b + data_c + b"\0" * block) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write("# test input map\n") + f.write(f"0 {block} data\n") + f.write(f"0x1000 {block} zero\n") + f.write(f"{2 * block} {block} data\n") + f.write(f"{3 * block} {block} zero\n") + expected = data_a + b"\0" * block + data_c + b"\0" * block + for name, chunker_args in [("test-fixed", ("--chunker-params", "fixed,4096")), ("test-default", ())]: + cmd(archiver, "create", *chunker_args, "--map", map_path, name, "input/file") + with changedir("output"): + cmd(archiver, "extract", name) + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == expected + shutil.rmtree("output") + os.mkdir("output") + + +def test_create_map_errors(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + create_regular_file(archiver.input_path, "file", size=2 * 4096) + create_regular_file(archiver.input_path, "file2", size=100) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write("0 4096 data\n") + + def expect_error(exc_class, *args): + if archiver.FORK_DEFAULT: + cmd(archiver, *args, exit_code=exc_class().exit_code) + else: + with pytest.raises(exc_class): + cmd(archiver, *args) + + # --map requires exactly one input path + expect_error(CommandError, "create", "--map", map_path, "test", "input/file", "input/file2") + # --map input must not be a directory + expect_error(CommandError, "create", "--map", map_path, "test", "input") + # --map cannot be used with stdin input + expect_error(CommandError, "create", "--map", map_path, "test", "-") + # the map covers 4096 bytes, but the input file has 8192 bytes + expect_error(Error, "create", "--map", map_path, "test", "input/file") diff --git a/src/borg/testsuite/helpers/fs_test.py b/src/borg/testsuite/helpers/fs_test.py index e67a23001..d93ddb1c6 100644 --- a/src/borg/testsuite/helpers/fs_test.py +++ b/src/borg/testsuite/helpers/fs_test.py @@ -24,7 +24,13 @@ from ...helpers.fs import ( make_path_safe, map_chars, SpecialFileReader, + MAP_DATA, + MAP_ZERO, + MAP_SAME, + read_input_map, + input_map_check_size, ) +from ...helpers.errors import Error from ...platform import is_win32, is_darwin from .. import are_fifos_supported, are_hardlinks_supported from .. import rejected_dotdot_paths @@ -606,3 +612,48 @@ def test_special_file_reader_not_seekable(): with pytest.raises(OSError) as exc_info: reader.seek(0) assert exc_info.value.errno == errno.ESPIPE + + +def test_read_input_map(tmp_path): + p = tmp_path / "input.map" + p.write_text("# a comment\n\n0 0x1000 data\n0x1000 4096 zero # trailing comment\n8192 100 data\n") + assert read_input_map(os.fspath(p)) == [(0, 4096, MAP_DATA), (4096, 4096, MAP_ZERO), (8192, 100, MAP_DATA)] + + +def test_read_input_map_same_state(tmp_path): + p = tmp_path / "input.map" + p.write_text("0 100 same\n") + with pytest.raises(Error, match="invalid state"): + read_input_map(os.fspath(p)) + assert read_input_map(os.fspath(p), allow_same=True) == [(0, 100, MAP_SAME)] + + +@pytest.mark.parametrize( + "content, match", + [ + ("", "empty input map"), + ("0 100\n", "expected 'START LENGTH STATE'"), + ("0 100 data extra\n", "expected 'START LENGTH STATE'"), + ("x 100 data\n", "expected 'START LENGTH STATE'"), + ("0 100 nodata\n", "invalid state"), + ("0 0 data\n", "LENGTH must be positive"), + ("0 -5 data\n", "LENGTH must be positive"), + ("100 100 data\n", "expected START 0"), + ("0 100 data\n50 100 zero\n", "expected START 100"), + ("0 100 data\n200 100 zero\n", "expected START 100"), + ], +) +def test_read_input_map_invalid(tmp_path, content, match): + p = tmp_path / "input.map" + p.write_text(content) + with pytest.raises(Error, match=match): + read_input_map(os.fspath(p)) + + +def test_input_map_check_size(): + input_map = [(0, 100, MAP_DATA), (100, 50, MAP_ZERO)] + input_map_check_size(input_map, 150) # exact coverage - ok + with pytest.raises(Error, match="covers 150 bytes, but the input has 151"): + input_map_check_size(input_map, 151) + with pytest.raises(Error, match="covers 150 bytes, but the input has 149"): + input_map_check_size(input_map, 149)