From 669b43980dd51652d395dfe6b3b8675baa9904ae Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 24 Sep 2026 03:24:47 +0200 Subject: [PATCH 1/3] use ssh:// URLs for current remote repositories, fixes #9765 ssh:// now addresses a current repository via REST over stdio over ssh (served by a remote "borg serve --rest"); with --from-borg1, it still addresses a legacy borg 1.x repository via the legacy RPC protocol. The testsuite uses ssh://__testsuite__/path (like the legacy client) to run "borg serve --rest" locally instead of via ssh. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- src/borg/archiver/_common.py | 22 ++-- src/borg/archiver/check_cmd.py | 2 +- src/borg/archiver/serve_cmd.py | 15 +-- src/borg/archiver/version_cmd.py | 4 +- src/borg/conftest.py | 4 +- src/borg/helpers/parseformat.py | 46 ++----- src/borg/legacy/remote.py | 9 +- src/borg/repository.py | 22 ++-- src/borg/testsuite/archiver/__init__.py | 2 +- .../testsuite/archiver/remote_repo_test.py | 2 +- .../testsuite/helpers/parseformat_test.py | 115 ++++-------------- src/borg/testsuite/repository_test.py | 33 +++-- 13 files changed, 100 insertions(+), 178 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d483e398..a59e3f6ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -282,7 +282,7 @@ jobs: sudo chmod o+x "$HOME" # Export SFTP test URL for tox via GITHUB_ENV echo "BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo" >> $GITHUB_ENV - echo "BORG_TEST_REST_REPO=rest://sftpuser@localhost:22/borg/rest-repo" >> $GITHUB_ENV + echo "BORG_TEST_REST_REPO=ssh://sftpuser@localhost:22/borg/rest-repo" >> $GITHUB_ENV - name: Install and start moto S3 server (test only) # The S3 tests run against moto in server mode: it comes from PyPI (pinned in diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index fa63b03e8..0eac26d4c 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -32,23 +32,17 @@ logger = create_logger(__name__) def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_legacy, allow_incomplete=False): # create_config=False: when creating, the command (repo-create) writes the repository config itself, # once the key exists, see Repository.create(). For an existing repository, the flag is irrelevant. - if location.proto == "ssh": - if v1_legacy: - from ..legacy.remote import LegacyRemoteRepository + if location.proto == "ssh" and v1_legacy: + # legacy borg 1.x repository, served by a remote "borg serve" via the legacy RPC protocol + from ..legacy.remote import LegacyRemoteRepository - repository = LegacyRemoteRepository( - location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock, args=args - ) - else: - raise Error( - "ssh:// is no longer supported for current repositories; use rest:// instead " - "(it can tunnel over ssh). ssh:// remains available only for legacy v1 repositories " - "via --from-borg1." - ) + repository = LegacyRemoteRepository( + location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock, args=args + ) elif ( - location.proto in ("rest", "sftp", "file", "http", "https", "rclone", "s3", "b2") and not v1_legacy - ): # stuff directly supported by borgstore + location.proto in ("ssh", "sftp", "file", "http", "https", "rclone", "s3", "b2") and not v1_legacy + ): # stuff directly supported by borgstore (ssh: REST via a remote "borg serve --rest") repository = Repository( location, create=create, diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index 6d3a5aef4..953838b22 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -130,7 +130,7 @@ class CheckMixIn: A corrupt index ends the check after this step, as the archives check needs it, unless ``--repair`` is given (see below). Running the repository check can be split into multiple partial checks using ``--max-duration``. - For rest:// repositories, the server computes the hashes, so the pack contents do + For ssh:// repositories, the server computes the hashes, so the pack contents do not have to travel over the network. For other remote backends, borg usually has to read (download) the objects to hash them. diff --git a/src/borg/archiver/serve_cmd.py b/src/borg/archiver/serve_cmd.py index f221dcf5e..1a42174fd 100644 --- a/src/borg/archiver/serve_cmd.py +++ b/src/borg/archiver/serve_cmd.py @@ -23,7 +23,7 @@ class ServeMixIn: ).serve() def do_serve_rest(self, args): - """Serve a current (non-legacy) rest:// repository on stdio (borgstore REST server).""" + """Serve a current (non-legacy) repository on stdio (borgstore REST server).""" from borgstore.server.rest import serve as rest_serve from ..repository import borg_permissions @@ -68,13 +68,14 @@ class ServeMixIn: It operates in one of two modes: - default (no option): serve a **legacy** (borg 1.x / v1) repository using the legacy - RPC protocol. This is used e.g. for ``borg transfer --from-borg1`` and is command-line - compatible with borg 1.x ``borg serve``. + RPC protocol. A borg client using an ``ssh://`` repository together with ``--from-borg1`` + (e.g. ``borg transfer --from-borg1``) starts this automatically via SSH. It is + command-line compatible with borg 1.x ``borg serve``. - ``--rest``: serve a **current** (non-legacy) repository as the server-side component of - a ``rest://`` repository, talking HTTP over stdio. The repository to serve is given via - ``--backend FILE:``. A borg client using a ``rest://`` repository starts this - automatically (over SSH if a host is given). + an ``ssh://`` repository, talking HTTP over stdio. The repository to serve is given via + ``--backend FILE:``. A borg client using an ``ssh://`` repository starts this + automatically via SSH. Please note that, in legacy mode, `borg serve` does not support providing a specific repository via the `--repo` option or the `BORG_REPO` environment variable - it is the @@ -101,7 +102,7 @@ class ServeMixIn: "--rest", dest="rest", action="store_true", - help="serve a current (non-legacy) repository as a rest:// server (HTTP over stdio). " + help="serve a current (non-legacy) repository as an ssh:// server (HTTP over stdio). " "Requires --backend. Without this option, a legacy (borg 1.x) repository is served.", ) subparser.add_argument( diff --git a/src/borg/archiver/version_cmd.py b/src/borg/archiver/version_cmd.py index ddce30a59..01f9e6348 100644 --- a/src/borg/archiver/version_cmd.py +++ b/src/borg/archiver/version_cmd.py @@ -38,9 +38,9 @@ class VersionMixIn: This command displays the Borg client and server versions. For current repositories the client code directly accesses the repository (also for - rest:// repositories), so the client version is shown as the server version, too. + ssh:// repositories), so the client version is shown as the server version, too. - If a legacy (borg 1.x / v1) repository is given via ssh: together with --from-borg1, + If a legacy (borg 1.x / v1) repository is given via ssh:// together with --from-borg1, the remote Borg is queried, and its version is displayed as the server version. Examples:: diff --git a/src/borg/conftest.py b/src/borg/conftest.py index 9925ee9d2..e0a5dce9b 100644 --- a/src/borg/conftest.py +++ b/src/borg/conftest.py @@ -162,7 +162,7 @@ class ArchiverSetup: self.patterns_file_path: str | None = None def get_kind(self) -> str: - if self.repository_location.startswith("rest://"): + if self.repository_location.startswith("ssh://"): return "remote" elif self.EXE == "borg.exe": return "binary" @@ -218,7 +218,7 @@ def archiver(tmp_path, set_env_variables): @pytest.fixture() def remote_archiver(archiver, monkeypatch): - archiver.repository_location = "rest://" + "/" + str(archiver.repository_path) + archiver.repository_location = "ssh://__testsuite__/" + str(archiver.repository_path) # don't measure coverage in the "borg serve --rest" children, see #9470 monkeypatch.delenv("COVERAGE_PROCESS_CONFIG", raising=False) monkeypatch.delenv("COVERAGE_PROCESS_START", raising=False) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 5530fc90b..96d8c8f73 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -767,15 +767,15 @@ class Location: # passes the raw URL through. covers both "scheme://..." and opaque "scheme:..." forms. BORGSTORE_SCHEMES = ("sftp", "http", "https", "s3", "b2", "rclone") - # locations that borg parses itself, see ssh_re / rest_re / file_re below. - BORG_SCHEMES = ("ssh", "rest", "file") + # locations that borg parses itself, see ssh_re / file_re below. + BORG_SCHEMES = ("ssh", "file") # path may contain any chars, but to avoid ambiguities with the other regexes it must not start # with any of the scheme specifiers above (all of which are matched before local_re in _parse). # Rejecting them here makes a malformed URL fail with a helpful error instead of being silently - # taken for a local path - e.g. "rest://host/" used to end up as the local directory - # "./rest:/host", see #10215. A local path that really starts with such a prefix can still be - # used by prefixing it with "./" (or by giving it as an absolute path). + # taken for a local path - e.g. "ssh://host/" would end up as the local directory "./ssh:/host", + # see #10215. A local path that really starts with such a prefix can still be used by prefixing + # it with "./" (or by giving it as an absolute path). local_path_re = r"(?!(?:" + "|".join(BORG_SCHEMES + BORGSTORE_SCHEMES) + r"):)" r"(?P.+)" # abs_path must start with a slash (or drive letter on Windows). @@ -785,11 +785,13 @@ class Location: abs_or_rel_path_re = r"(?P.+)" # We only parse out individual fields (user/host/port/path) for the protocols where borg - # itself needs them: legacy "ssh" (v1 repositories) and "rest" (for the ssh tunnel + FILE - # backend), plus local "file" paths. Everything else (see BORGSTORE_SCHEMES) is handed to - # borgstore as the raw URL and parsed/validated there - we only detect the scheme. + # itself needs them: "ssh" (for the ssh command line and the FILE backend path) plus local + # "file" paths. Everything else (see BORGSTORE_SCHEMES) is handed to borgstore as the raw URL + # and parsed/validated there - we only detect the scheme. - # ssh:// is only used for legacy borg 1.x repositories nowadays. + # ssh:// reaches a remote "borg serve" via ssh: for current repositories, borg talks HTTP (REST) + # over the ssh connection's stdio; for legacy borg 1.x repositories (--from-borg1), it uses the + # legacy RPC protocol. ssh_re = re.compile( r"(?Pssh)://" + optional_user_re @@ -800,19 +802,6 @@ class Location: re.VERBOSE, ) - # REST http via stdio (via ssh, if host given): - rest_re = re.compile( - r"(?P(rest))://" - + r"(" - + optional_user_re - + host_re - + optional_port_re - + r")?" - + r"/" # this is the separator, not part of the path! - + abs_or_rel_path_re, - re.VERBOSE, - ) - # scheme detector for the borgstore-handled locations listed in BORGSTORE_SCHEMES above. scheme_re = re.compile(r"(?P[a-zA-Z][a-zA-Z0-9+.\-]*):") @@ -823,8 +812,6 @@ class Location: # accepted forms per scheme borg parses itself, used to explain a URL we could not parse. scheme_hints = { - "rest": "rest://[user@]host[:port]/path/to/repo (path relative to the remote directory ssh " - "logs into) or rest://[user@]host[:port]//path/to/repo (absolute path)", "ssh": "ssh://[user@]host[:port]/path/to/repo (path relative to the remote directory ssh " "logs into) or ssh://[user@]host[:port]//path/to/repo (absolute path)", "file": "file:///C:/path/to/repo" if is_win32 else "file:///path/to/repo", @@ -873,15 +860,6 @@ class Location: def _parse(self, text): m = self.ssh_re.match(text) - if m: - self.proto = m.group("proto") - self.user = m.group("user") - self._host = m.group("host") - self.port = m.group("port") and int(m.group("port")) or None - # remote path: normalize with posixpath, not with the client's os.path, see #10199. - self.path = posixpath.normpath(m.group("path")) - return True - m = self.rest_re.match(text) if m: self.proto = m.group("proto") self.user = m.group("user") @@ -932,7 +910,7 @@ class Location: def canonical_path(self): if self.proto == "file": return normalize_local_path(self.path) - if self.proto in ("rest", "ssh"): + if self.proto == "ssh": return ( f"{self.proto}://" f"{(self.user + '@') if self.user else ''}" diff --git a/src/borg/legacy/remote.py b/src/borg/legacy/remote.py index cef1be02c..e05392026 100644 --- a/src/borg/legacy/remote.py +++ b/src/borg/legacy/remote.py @@ -633,9 +633,10 @@ class LegacyRemoteRepository: """actual remoting is done via self.call in the @api decorator""" -# borg serve: borg only serves legacy (borg 1.x / v1) repositories over ssh:// now (current -# repositories use rest://). The legacy client above (LegacyRemoteRepository) spawns "borg serve" -# on the remote host; this server keeps the legacy RPC method allowlist and opens LegacyRepository. +# borg serve (without --rest): serves legacy (borg 1.x / v1) repositories via the legacy RPC protocol +# (current repositories are served via REST by "borg serve --rest"). The legacy client above +# (LegacyRemoteRepository) spawns "borg serve" on the remote host; this server keeps the legacy RPC +# method allowlist and opens LegacyRepository. class RepositoryServer: # pragma: no cover @@ -817,7 +818,7 @@ class RepositoryServer: # pragma: no cover return path def open(self, path, create=False, lock_wait=None, lock=True, exclusive=None, v1_legacy=False): - # borg only serves legacy (v1) repositories now; current repositories are accessed via rest://. + # this server only serves legacy (v1) repositories; current repositories are served via REST. self.RepoCls = LegacyRepository self.rpc_methods = self._legacy_rpc_methods logging.debug("Resolving repository path %r", path) diff --git a/src/borg/repository.py b/src/borg/repository.py index 4539255fc..800e1bc2c 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -105,13 +105,13 @@ def borg_permissions(permissions): def rest_serve_command(location): - """Build the command line that serves a rest:// *location* via "borg serve --rest". + """Build the command line that serves an ssh:// *location* via "borg serve --rest". - For a local rest:// (no host) we run this borg directly (over stdio); if a host is - given, we prefix an ssh command (reusing borgstore's ssh_cmd / BORGSTORE_RSH). + We prefix an ssh command (reusing borgstore's ssh_cmd / BORGSTORE_RSH) to reach the remote borg. + For the special host "__testsuite__", we run this borg locally instead (over stdio, no ssh). """ backend_arg = f"FILE:{location.path}" - if not location.host: + if location.host == "__testsuite__": # run this borg locally, talking over stdio borg_cmd = [sys.executable] if getattr(sys, "frozen", False) else [sys.executable, "-m", "borg"] return borg_cmd + ["serve", "--rest", "--backend", backend_arg] @@ -138,7 +138,7 @@ def propagate_rsh(): def build_rest_backend(location): - """Return a borgstore REST backend for a rest:// *location*, served by "borg serve --rest".""" + """Return a borgstore REST backend for an ssh:// *location*, served by "borg serve --rest".""" return REST(base_url="http://stdio-backend", command=rest_serve_command(location)) @@ -950,9 +950,9 @@ class Repository: propagate_rsh() # borgstore shall use the same remote shell command as borg try: - if location.proto == "rest": - # rest:// is served by "borg serve --rest" (reachable via ssh if a host is given), - # talking HTTP over stdio - rather than borgstore's own "borgstore-server-rest" command. + if location.proto == "ssh": + # ssh:// is served by a remote "borg serve --rest" (reached via ssh), talking HTTP over stdio - + # rather than borgstore's own "borgstore-server-rest" command. # permissions are not given to the (remote) backend here; they are enforced on the # server side by "borg serve --rest --permissions ...". backend = build_rest_backend(location) @@ -963,9 +963,9 @@ class Repository: self.store = Store(url, config=ns_config, permissions=permissions, cache_url=cache_url) except StoreBackendError as e: raise Error(str(e)) - # None means "all" (no restrictions); for rest:// the backend enforces permissions + # None means "all" (no restrictions); for ssh:// the backend enforces permissions # server-side, so the client does not check them (see above). - self.permissions = None if location.proto == "rest" else permissions + self.permissions = None if location.proto == "ssh" else permissions self.store_opened = False self.version = None self.id = None @@ -1127,7 +1127,7 @@ class Repository: try: text = self.store.load("config/config").decode() except StoreBackendDoesNotExist: - # A rest:// store's open() does not contact the server, so for rest:// a missing repository + # An ssh:// store's open() does not contact the server, so for ssh:// a missing repository # only shows up here, when the first request fails with BackendDoesNotExist (#10365). raise self.DoesNotExist(str(self._location)) from None except StoreObjectNotFound: diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index ca3151662..b831e5617 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -140,7 +140,7 @@ def generate_archiver_tests(metafunc, kinds: str): # # Picking "kinds" (see #9324, testsuite speedup): # - "local" should always be included. - # - "remote" exercises the rest:// transport (it spawns a borgstore-server-rest subprocess and does + # - "remote" exercises the ssh:// transport (it spawns a "borg serve --rest" subprocess and does # synchronous HTTP-over-stdio round trips per object), so a remote run costs several times its local # twin. Only add "remote" where a backend genuinely branches its implementation per transport, and # that branch is not already exercised by another remote-tested file: diff --git a/src/borg/testsuite/archiver/remote_repo_test.py b/src/borg/testsuite/archiver/remote_repo_test.py index 0174ec4c4..e422906e8 100644 --- a/src/borg/testsuite/archiver/remote_repo_test.py +++ b/src/borg/testsuite/archiver/remote_repo_test.py @@ -63,7 +63,7 @@ def test_rclone_repo_basics(archiver, tmp_path): def test_rest_repo_basics(archiver, monkeypatch): create_regular_file(archiver.input_path, "file1", size=100 * 1024) create_regular_file(archiver.input_path, "file2", size=10 * 1024) - # A rest:// repo over ssh starts "borg serve --rest" on the remote. For this test the remote is + # An ssh:// repo starts "borg serve --rest" on the remote. For this test the remote is # localhost (see CI BORG_TEST_REST_REPO), so point BORG_REMOTE_PATH at the borg under test # (an absolute path that is valid locally) unless the caller already set it. if not os.environ.get("BORG_REMOTE_PATH"): diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index 1c82020e1..03e7576d5 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -149,73 +149,8 @@ class TestLocationWithoutEnv: "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')" ) - def test_rest(self, monkeypatch): - monkeypatch.delenv("BORG_REPO", raising=False) - assert ( - repr(Location("rest://user@host:1234//absolute/path")) - == "Location(proto='rest', user='user', pass=None, host='host', port=1234, path='/absolute/path')" - ) - assert ( - repr(Location("rest://user@host:1234/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='host', port=1234, path='relative/path')" - ) - assert ( - repr(Location("rest://user@host/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='host', port=None, path='relative/path')" - ) - assert ( - repr(Location("rest://user@[::]:1234/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='::', port=1234, path='relative/path')" - ) - assert ( - repr(Location("rest://user@[::]/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='::', port=None, path='relative/path')" - ) - assert ( - repr(Location("rest://user@[2001:db8::]:1234/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='2001:db8::', port=1234, path='relative/path')" - ) - assert ( - repr(Location("rest://user@[2001:db8::]/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='2001:db8::', port=None, path='relative/path')" - ) - assert ( - repr(Location("rest://user@[2001:db8::c0:ffee]:1234/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='2001:db8::c0:ffee', port=1234, path='relative/path')" # noqa: E501 - ) - assert ( - repr(Location("rest://user@[2001:db8::c0:ffee]/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='2001:db8::c0:ffee', port=None, path='relative/path')" # noqa: E501 - ) - assert ( - repr(Location("rest://user@[2001:db8::192.0.2.1]:1234/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='2001:db8::192.0.2.1', port=1234, path='relative/path')" # noqa: E501 - ) - assert ( - repr(Location("rest://user@[2001:db8::192.0.2.1]/relative/path")) - == "Location(proto='rest', user='user', pass=None, host='2001:db8::192.0.2.1', port=None, path='relative/path')" # noqa: E501 - ) - assert ( - repr(Location("rest://user@[2a02:0001:0002:0003:0004:0005:0006:0007]/relative/path")) - == "Location(proto='rest', user='user', pass=None, " - "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=None, path='relative/path')" - ) - assert ( - repr(Location("rest://user@[2a02:0001:0002:0003:0004:0005:0006:0007]:1234/relative/path")) - == "Location(proto='rest', user='user', pass=None, " - "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')" - ) - assert ( - repr(Location("rest:///relative/path")) - == "Location(proto='rest', user=None, pass=None, host=None, port=None, path='relative/path')" - ) - assert ( - repr(Location("rest:////absolute/path")) - == "Location(proto='rest', user=None, pass=None, host=None, port=None, path='/absolute/path')" - ) - def test_server_side_path_is_posix(self, monkeypatch): - # ssh:// and rest:// paths live on the *server*, so they must be normalized as POSIX + # ssh:// paths live on the *server*, so they must be normalized as POSIX # paths, no matter what the client runs on. A Windows client used to normalize them # with ntpath, turning "/path/to/repo" into "\path\to\repo"; ssh passes the command # through the remote shell, which ate the backslashes and left "pathtorepo", see #10199. @@ -226,31 +161,29 @@ class TestLocationWithoutEnv: # Faking a Windows client by swapping in ntpath tests this on any platform instead. monkeypatch.delenv("BORG_REPO", raising=False) monkeypatch.setattr(parseformat.os, "path", ntpath) - for proto in ("ssh", "rest"): - assert Location(f"{proto}://user@host/relative/path").path == "relative/path" - assert Location(f"{proto}://user@host//absolute/path").path == "/absolute/path" - assert Location(f"{proto}://user@host:1234//absolute/path").path == "/absolute/path" - assert Location(f"{proto}://user@host//absolute/./x/../path").path == "/absolute/path" - assert ( - Location(f"{proto}://user@host//absolute/path").canonical_path() - == f"{proto}://user@host//absolute/path" - ) + assert Location("ssh://user@host/relative/path").path == "relative/path" + assert Location("ssh://user@host//absolute/path").path == "/absolute/path" + assert Location("ssh://user@host:1234//absolute/path").path == "/absolute/path" + assert Location("ssh://user@host//absolute/./x/../path").path == "/absolute/path" + assert Location("ssh://user@host//absolute/path").canonical_path() == "ssh://user@host//absolute/path" @pytest.mark.parametrize( "location", [ - "rest://host", # no path - "rest://host/", # empty path - "rest://user@host/", # empty path - "rest://user@host:1234/", # empty path - "rest://", # nothing at all - "rest:/host/path", # only one slash after the scheme - "rest:host/path", # no slash after the scheme + "ssh://host", # no path + "ssh://host/", # empty path + "ssh://user@host/", # empty path + "ssh://user@host:1234/", # empty path + "ssh:///path", # no host + "ssh:////absolute/path", # no host + "ssh://", # nothing at all + "ssh:/host/path", # only one slash after the scheme + "ssh:host/path", # no slash after the scheme ], ) - def test_rest_invalid(self, monkeypatch, location): - # a malformed rest:// URL must be rejected - it used to be taken for a local path - # (relative to the cwd) and then failed with a confusing error, see #10215. + def test_ssh_invalid(self, monkeypatch, location): + # a malformed ssh:// URL must be rejected - it must not be taken for a local path + # (relative to the cwd) and then fail with a confusing error, see #10215. monkeypatch.delenv("BORG_REPO", raising=False) with pytest.raises(ValueError, match="Invalid location format"): Location(location) @@ -258,7 +191,7 @@ class TestLocationWithoutEnv: def test_invalid_location_hint(self, monkeypatch): # an unparsable URL of a scheme we parse ourselves tells the user the accepted forms. monkeypatch.delenv("BORG_REPO", raising=False) - for location, scheme in [("rest://host/", "rest"), ("ssh://host/", "ssh"), ("file://rel", "file")]: + for location, scheme in [("ssh://host/", "ssh"), ("file://rel", "file")]: with pytest.raises(ValueError, match=f"Expected: {scheme}:"): Location(location) @@ -267,9 +200,9 @@ class TestLocationWithoutEnv: # a local path starting with a scheme name is not mistaken for a URL if it is made # unambiguous with "./" or by giving it as an absolute path. monkeypatch.delenv("BORG_REPO", raising=False) - assert Location("./rest:/host").proto == "file" - assert Location("./rest:/host").path == os.path.abspath("rest:/host") - assert Location("/abs/rest:/host").path == "/abs/rest:/host" + assert Location("./ssh:/host").proto == "file" + assert Location("./ssh:/host").path == os.path.abspath("ssh:/host") + assert Location("/abs/ssh:/host").path == "/abs/ssh:/host" # For the protocols handled (parsed + validated) by borgstore itself, borg only detects # the scheme and passes the raw URL through; it no longer extracts user/host/port/path. @@ -395,10 +328,6 @@ class TestLocationWithoutEnv: "ssh://host/relative/path", "ssh://host//absolute/path", "ssh://user@host:1234/relative/path", - "rest://host/relative/path", - "rest://host//absolute/path", - "rest://user@host:1234/relative/path", - "rest:///relative/path", "sftp://host/relative/path", "sftp://host//absolute/path", "sftp://user@host:1234/relative/path", diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py index b10b0bb58..b593ed7fc 100644 --- a/src/borg/testsuite/repository_test.py +++ b/src/borg/testsuite/repository_test.py @@ -32,20 +32,20 @@ def use_test_key_on_open(monkeypatch): def test_rest_serve_command_local(): - # rest:// without a host runs "borg serve --rest" locally, talking over stdio. - cmd = rest_serve_command(Location("rest:////tmp/repo")) + # ssh:// with the special host "__testsuite__" runs "borg serve --rest" locally, talking over stdio. + cmd = rest_serve_command(Location("ssh://__testsuite__//tmp/repo")) assert "ssh" not in cmd assert cmd[0] == sys.executable assert cmd[-4:] == ["serve", "--rest", "--backend", "FILE:/tmp/repo"] def test_rest_serve_command_ssh(monkeypatch): - # rest:// with a host is reached via ssh, running "borg serve --rest" remotely. + # ssh:// with a host is reached via ssh, running "borg serve --rest" remotely. # we override BORGSTORE_RSH to a simple "ssh" here to simplify testing. # without that, borgstore 0.5.5+ would also set some ssh options via cmdline. monkeypatch.setenv("BORGSTORE_RSH", "ssh") monkeypatch.delenv("BORG_REMOTE_PATH", raising=False) - cmd = rest_serve_command(Location("rest://user@host/repo/path")) + cmd = rest_serve_command(Location("ssh://user@host/repo/path")) assert cmd[:2] == ["ssh", "user@host"] assert cmd[-5:] == ["borg", "serve", "--rest", "--backend", "FILE:repo/path"] @@ -67,11 +67,30 @@ def test_propagate_rsh(monkeypatch): assert "BORGSTORE_RSH" not in os.environ -@pytest.mark.parametrize("proto", ["file", "rest"]) +@pytest.mark.parametrize("v1_legacy", [False, True]) +def test_get_repository_ssh(monkeypatch, v1_legacy): + # ssh:// is a current repository served via REST, or a legacy repository with --from-borg1 (#9765). + from ..archiver._common import get_repository + from ..legacy import remote as legacy_remote + + class FakeLegacyRemoteRepository: + def __init__(self, location, **kw): + self._location = location + + monkeypatch.setattr(legacy_remote, "LegacyRemoteRepository", FakeLegacyRemoteRepository) + location = Location("ssh://user@host/repo/path") + repository = get_repository( + location, create=False, exclusive=False, lock_wait=1, lock=True, args=None, v1_legacy=v1_legacy + ) + assert type(repository) is (FakeLegacyRemoteRepository if v1_legacy else Repository) + assert repository._location is location + + +@pytest.mark.parametrize("proto", ["file", "ssh"]) def test_open_nonexistent_repository(tmp_path, proto): - # A missing repository raises Repository.DoesNotExist, also via the rest:// transport (#10365). + # A missing repository raises Repository.DoesNotExist, also via the ssh:// transport (#10365). path = os.fspath(tmp_path / "nonexistent") - location = Location(path if proto == "file" else f"rest:///{path}") + location = Location(path if proto == "file" else f"ssh://__testsuite__/{path}") with pytest.raises(Repository.DoesNotExist): with Repository(location, exclusive=True): pass From 93481e0fb830f1778a2a67dc332a8b3350e96cee Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 24 Sep 2026 03:43:59 +0200 Subject: [PATCH 2/3] ssh://: explain a remote borg serve that fails to start If the first request to the remote "borg serve" fails (e.g. a borg 1.x borg serve does not know --rest, or borg is not installed there), raise an Error showing the server's stderr and a hint (BORG_REMOTE_PATH, --from-borg1) instead of a "Local Exception" traceback. Repository.close() does not raise from closing the store while unwinding an error anymore (like the lock release before it): a dead store fails to close, too, which masked the original error. Co-Authored-By: Claude Opus 5.5 --- src/borg/repository.py | 40 +++++++++++++++++++-------- src/borg/testsuite/repository_test.py | 21 +++++++++++++- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index 800e1bc2c..c08fa4451 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1007,18 +1007,28 @@ class Repository: return f"<{self.__class__.__name__} {self._location}>" def __enter__(self): - if self.do_create: - self.do_create = False - self.create() - self.created = True try: - self.open(exclusive=bool(self.exclusive), lock_wait=self.lock_wait, lock=self.do_lock) - except Exception: - self.close(aborting=True) - if self.created: - # we just created the store, but could not open it: do not leave it behind (see create()). - self.store.destroy() - raise + if self.do_create: + self.do_create = False + self.create() + self.created = True + try: + self.open(exclusive=bool(self.exclusive), lock_wait=self.lock_wait, lock=self.do_lock) + except Exception: + self.close(aborting=True) + if self.created: + # we just created the store, but could not open it: do not leave it behind (see create()). + self.store.destroy() + raise + except StoreBackendError as e: + if self._location.proto != "ssh": + raise + # the first request to the remote "borg serve" failed, usually because it did not start. + raise Error( + f"Could not access the repository via borg serve on the remote host: {e}\n" + "Is borg 2 installed there (see BORG_REMOTE_PATH)? " + "For a borg 1.x repository, use --from-borg1." + ) from None return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -1387,7 +1397,13 @@ class Repository: logger.warning("failed to release the lock during close: %s", exc) self.lock = None if self.store_opened: - self.store.close() + try: + self.store.close() + except Exception as exc: + if not unwinding: + raise + # when the store is dead (e.g. the remote borg serve exited), closing it fails, too. + logger.debug("failed to close the store during close: %s", exc) self.store_opened = False self.opened = False self._pack_cache.clear() diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py index b593ed7fc..6a85a5ab1 100644 --- a/src/borg/testsuite/repository_test.py +++ b/src/borg/testsuite/repository_test.py @@ -8,6 +8,7 @@ from collections import namedtuple import pytest from borghash import HashTableNT +from borgstore.backends.rest import REST from ..crypto.key import store_hash from .. import repository as repository_module @@ -15,7 +16,7 @@ from ..cache import chunkindex_is_invalid, delete_chunkindex_from_repo, write_ch from ..compress import CNONE from ..constants import MAX_CLOCK_SKEW, ROBJ_FILE_STREAM from ..crypto.key import AESOCBKey, AuthenticatedKey, Blake3AuthenticatedKey, CHPOKey -from ..helpers import IntegrityError, Location, bin_to_hex +from ..helpers import Error, IntegrityError, Location, bin_to_hex from ..hashindex import ChunkIndex, ChunkIndexEntry from ..repository import Repository, MAX_DATA_SIZE, MAX_VALIDATED_META_SIZE, propagate_rsh, rest_serve_command from ..repository import PackWriter, PackReader, PackTracker, superseded_gap_ranges @@ -97,6 +98,24 @@ def test_open_nonexistent_repository(tmp_path, proto): assert not os.path.exists(path) +@pytest.mark.parametrize("create", [False, True]) +def test_remote_serve_fails_to_start(tmp_path, monkeypatch, create): + # e.g. a borg 1.x "borg serve" on the remote host does not know --rest: the user gets an error + # showing the server's stderr and a hint, not a traceback of the dead store's close(). + def failing_rest_backend(location): + error = "borg serve: error: unrecognized arguments: --rest" + command = [sys.executable, "-c", f"import sys; sys.stderr.write({error!r} + '\\n'); sys.exit(2)"] + # no waiting between the reconnect attempts, to keep the test fast. + return REST(base_url="http://stdio-backend", command=command, reconnect_wait=0) + + monkeypatch.setattr(repository_module, "build_rest_backend", failing_rest_backend) + location = Location("ssh://__testsuite__/" + os.fspath(tmp_path / "repository")) + with pytest.raises(Error, match="use --from-borg1") as exc_info: + with Repository(location, exclusive=True, create=create): + pass + assert "unrecognized arguments: --rest" in str(exc_info.value) + + @pytest.fixture() def repository(tmp_path): repository_location = os.fspath(tmp_path / "repository") From db0c4ac2c1f3527f8749f77a100c9e7d4414fd22 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 24 Sep 2026 15:52:04 +0200 Subject: [PATCH 3/3] docs: ssh:// repository URLs, #9765 Describe ssh:// repository URLs for current repositories (served by a remote "borg serve --rest") and for legacy borg 1.x repositories (with --from-borg1, served by a remote "borg serve") and add a change log entry. Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 14 ++++++------ README.rst | 2 +- docs/changes.rst | 5 ++++- docs/deployment/central-backup-server.rst | 9 ++++---- docs/deployment/hosting-repositories.rst | 16 +++++++------- docs/deployment/pull-backup.rst | 18 ++++++++-------- docs/faq.rst | 8 +++---- docs/internals/security.rst | 19 ++++++++-------- docs/misc/asciinema/demo.tcl | 2 +- docs/quickstart.rst | 25 +++++++++++----------- docs/usage/general/file-systems.rst.inc | 4 ++-- docs/usage/general/repository-urls.rst.inc | 24 ++++++++------------- docs/usage/serve.rst | 7 +++--- docs/usage/transfer.rst | 6 +++--- scripts/rest_perf_test.sh | 10 ++++----- 15 files changed, 82 insertions(+), 87 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index a09001a15..69dbf7796 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -131,13 +131,13 @@ so the repository storage layer never sees plaintext or has any notion of files/ - **`repository.py`** — `Repository`, a key→value object store built on top of **borgstore** (`borgstore.store.Store`). Keys are 32-byte object IDs; the repo doesn't know what an "archive" or "file" is. borgstore provides the actual storage backends and transport: local filesystem, - `ssh://` (borgstore ssh backend, optionally reusing `BORGSTORE_RSH`), and `rest://` (borgstore REST - server, served by `borg serve --rest`). This replaces the borg 1.x segment/journal format — read + `ssh://` (borgstore REST backend talking HTTP over stdio over ssh to a remote `borg serve`, + optionally reusing `BORGSTORE_RSH`), `sftp://`, `http(s)://`, `rclone:`, `s3:`/`b2:`. This replaces the borg 1.x segment/journal format — read `docs/internals/data-structures.rst` before touching repository/object/format code. -- **`archiver/serve_cmd.py`** — `borg serve` provides the server side for remote repositories: the - default mode serves a repository over stdio to a connecting borgstore ssh backend, and - `borg serve --rest` runs the borgstore REST server. (There is no separate `remote.py` / custom RPC - layer anymore — borgstore owns the client/server transport.) +- **`archiver/serve_cmd.py`** — `borg serve` provides the server side for `ssh://` repositories: + `borg serve --rest` runs the borgstore REST server on stdio for a current repository, plain + `borg serve` serves a legacy borg 1.x repository via the legacy RPC protocol (`--from-borg1`, + `legacy/remote.py`). - **`crypto/key.py`, `crypto/low_level.pyx`** — encryption/authentication (AEAD: AES-256-OCB or ChaCha20-Poly1305; ids via HMAC-SHA256 or BLAKE3) and key file handling (keyfile vs repokey storage). The `id_hash` used for deduplication depends on the key/encryption mode. @@ -168,7 +168,7 @@ so the repository storage layer never sees plaintext or has any notion of files/ progress reporting, filesystem helpers); check here before writing a new utility, it may exist. **Client/server split:** most `do_*` archiver commands operate against a local or remote `Repository` -transparently — the choice of local vs. `ssh://`/`rest://` is resolved by `Location` parsing in +transparently — the choice of local vs. `ssh://` etc. is resolved by `Location` parsing in `helpers/` and borgstore backend selection, and doesn't otherwise change command logic. `borg serve` on the remote end provides the server side. diff --git a/README.rst b/README.rst index 1f23ae0d8..952aef2f8 100644 --- a/README.rst +++ b/README.rst @@ -102,7 +102,7 @@ Main features **Off-site backups** Borg can store data on any remote host accessible via misc. protocols: - * ``rest://`` (REST-http-over-stdio-over-ssh) and ``http(s)://`` + * ``ssh://`` (REST-http-over-stdio-over-ssh) and ``http(s)://`` (REST-http-over-tcp). Significant performance gains can be achieved with these by having a remote agent (borg must be installed on the repo server). * ``sftp://`` diff --git a/docs/changes.rst b/docs/changes.rst index 4c2bd4cc0..7444ccc2b 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -180,6 +180,9 @@ Breaking changes (you must create new repos for b25): the authenticated modes, #9819, #10235. - drop OBJ_VERSION_NO_HEADER_AAD (pack object format v1) support, #9973 - KeyType: renumber the authenticated key types to 0x50 / 0x60 +- ssh:// repository URLs access current repositories via REST over ssh + (served by a remote "borg serve"), #9765. With --from-borg1, they access + legacy borg 1.x repositories via the legacy protocol (as before). New features: @@ -197,7 +200,7 @@ Fixes: - crypto: the AEAD ciphers (AES-OCB, ChaCha20-Poly1305) feed their input to OpenSSL in <= 1GB chunks, to overcome the 32bit size limit of the OpenSSL API. -- repository: raise DoesNotExist for a missing rest:// repo, #10365 +- repository: raise DoesNotExist for a missing ssh:// repo, #10365 - compact: - build the chunk index once, not three times diff --git a/docs/deployment/central-backup-server.rst b/docs/deployment/central-backup-server.rst index b1b0860cf..cbef937cd 100644 --- a/docs/deployment/central-backup-server.rst +++ b/docs/deployment/central-backup-server.rst @@ -10,11 +10,10 @@ clients. .. note:: - This example predates Borg 2 and uses the legacy ``ssh://`` transport (served - by ``borg serve``) and ``borg init``. With Borg 2, the ``ssh://`` transport is - only used for legacy borg 1.x (v1) repositories; for current repositories use a - ``rest://`` repository instead (Borg connects via ssh and runs a borgstore REST - server on the remote host), and use ``borg repo-create`` instead of ``borg init``. + This example predates Borg 2 and uses ``borg init`` and ``borg serve``. With + Borg 2, use ``borg repo-create`` instead of ``borg init``, and ``borg serve --rest`` + instead of ``borg serve`` in the forced command (for ``ssh://`` repositories, + Borg connects via ssh and runs ``borg serve --rest`` on the remote host). Machines -------- diff --git a/docs/deployment/hosting-repositories.rst b/docs/deployment/hosting-repositories.rst index 1dc56e411..30ac948ca 100644 --- a/docs/deployment/hosting-repositories.rst +++ b/docs/deployment/hosting-repositories.rst @@ -49,11 +49,11 @@ files they will be included in this set. The ``command`` keyword forces execution of the specified command upon login. This must be ``borg serve --rest``: ``--rest`` serves a current -repository (the server side of a ``rest://`` repository URL), while ``borg serve`` -without it serves a legacy borg 1.x repository using the legacy RPC protocol. -The client cannot supply ``--rest`` itself - the mode is deliberately pinned by -the forced command - so a forced command without ``--rest`` locks out all clients -using current (``rest://``) repositories. +repository (the server side of an ``ssh://`` repository URL), while ``borg serve`` +without it serves a legacy borg 1.x repository using the legacy RPC protocol +(used by ``--from-borg1``). The client cannot supply ``--rest`` itself - the mode +is deliberately pinned by the forced command - so a forced command without +``--rest`` locks out all clients using current repositories. The ``--restrict-to-repository`` option permits access to exactly **one** repository. It can be given multiple times to permit access to more than @@ -77,12 +77,12 @@ come from the forced command. The user therefore selects the repository through the path in the repository URL:: - borg -r rest://@/repository repo-list + borg -r ssh://@/repository repo-list A path with a single leading slash is relative to the directory ssh logs into (the user's home directory), so with the forced command shown above, -``rest://@/repository`` and -``rest://@//home//repository`` address the same repository. +``ssh://@/repository`` and +``ssh://@//home//repository`` address the same repository. Refer to the `sshd(8) `_ man page for more details on SSH options. diff --git a/docs/deployment/pull-backup.rst b/docs/deployment/pull-backup.rst index 11e9fb4c7..6558eb6c8 100644 --- a/docs/deployment/pull-backup.rst +++ b/docs/deployment/pull-backup.rst @@ -262,7 +262,7 @@ standard input and output available to a unix socket:: Socat will wait until a connection is opened. Then socat will execute the command given, redirecting Standard Input and Output to the unix socket. ``--rest`` and ``--backend`` are required here: ``--rest`` serves a current -repository (talking HTTP over stdio, the server side of a ``rest://`` repository +repository (talking HTTP over stdio, the server side of an ``ssh://`` repository URL), and ``--backend`` selects the repository to serve. ``--restrict-to-path`` is not necessary but a sane default. @@ -271,7 +271,7 @@ is not necessary but a sane default. (see ``BORG_RSH`` below) is discarded - including the ``--backend`` option the client would normally use to tell the server which repository it wants. The repository therefore is the one given by ``--backend`` in the socat command - above, no matter which path the client uses in its ``rest://`` URL. Use the + above, no matter which path the client uses in its ``ssh://`` URL. Use the same path in both places to avoid confusion. .. note:: @@ -322,7 +322,7 @@ SSH command. Those arguments can not be handled by socat. We wrap the command wi All Borg commands can now be executed on *borg-client*. For example to create a backup execute the ``borg create`` command:: - borg-client:~$ borg create --repo rest://borg-server//path/to/repo archive /path_to_backup + borg-client:~$ borg create --repo ssh://borg-server//path/to/repo archive /path_to_backup When automating backup creation, the interactive ssh session may seem inappropriate. An alternative way of creating @@ -333,7 +333,7 @@ a backup may be the following command:: borgc@borg-client \ BORG_RSH="sh -c 'exec socat STDIO UNIX-CONNECT:/run/borg/reponame.sock'" \ borg create \ - --repo rest://borg-server//path/to/repo archive /path_to_backup \ + --repo ssh://borg-server//path/to/repo archive /path_to_backup \ ';' rm /run/borg/reponame.sock This command also automatically removes the socket file after the ``borg @@ -406,7 +406,7 @@ Initiating borg command execution from *borg-server* (e.g. repo-create):: eval $(ssh-agent) > /dev/null ssh-add -q ~/.ssh/borg-client_key echo 'your secure borg key passphrase' | \ - ssh -A -o StrictHostKeyChecking=no borgc@borg-client "BORG_PASSPHRASE=\$(cat) BORG_RSH='ssh -o StrictHostKeyChecking=no' borg repo-create --encryption aes256-ocb --key-location repokey -r rest://borgs@borg-server/repo" + ssh -A -o StrictHostKeyChecking=no borgc@borg-client "BORG_PASSPHRASE=\$(cat) BORG_RSH='ssh -o StrictHostKeyChecking=no' borg repo-create --encryption aes256-ocb --key-location repokey -r ssh://borgs@borg-server/repo" kill "${SSH_AGENT_PID}" ) @@ -432,13 +432,13 @@ Parentheses are not needed when using a dedicated bash process. * The keys meant to be loaded into the agent must be specified explicitly, not from default locations. * The *borg-client*'s entry in *borgs@borg-server:~/.ssh/authorized_keys* must be as restrictive as possible. -``echo 'your secure borg key passphrase' | ssh -A -o StrictHostKeyChecking=no borgc@borg-client "BORG_PASSPHRASE=\$(cat) BORG_RSH='ssh -o StrictHostKeyChecking=no' borg repo-create --encryption aes256-ocb --key-location repokey -r rest://borgs@borg-server/repo"`` +``echo 'your secure borg key passphrase' | ssh -A -o StrictHostKeyChecking=no borgc@borg-client "BORG_PASSPHRASE=\$(cat) BORG_RSH='ssh -o StrictHostKeyChecking=no' borg repo-create --encryption aes256-ocb --key-location repokey -r ssh://borgs@borg-server/repo"`` Run the *borg repo-create* command on *borg-client*. - *rest://borgs@borg-server/repo* refers to the repository *repo* within borgs's home directory on *borg-server* + *ssh://borgs@borg-server/repo* refers to the repository *repo* within borgs's home directory on *borg-server* (a path given with a single leading slash is relative to the directory ssh logs into; use - *rest://borgs@borg-server//path/to/repo* for an absolute path). + *ssh://borgs@borg-server//path/to/repo* for an absolute path). Borg on *borg-client* connects via ssh (using *BORG_RSH*) and runs ``borg serve --rest`` on *borg-server*, which the forced command in *borgs@borg-server:~/.ssh/authorized_keys* pins down. @@ -490,7 +490,7 @@ using ``localhost`` instead of ``mybackup`` 2. On machine ``myclient`` -``borg create -v --progress --stats -r rest://backup@localhost:8022//home/backup/repos/myclient system /`` +``borg create -v --progress --stats -r ssh://backup@localhost:8022//home/backup/repos/myclient system /`` Make sure to use port ``8022`` and ``localhost`` for the repository as this instructs borg on ``myclient`` to use the remote forwarded ssh connection. diff --git a/docs/faq.rst b/docs/faq.rst index fcd18de52..41f514b60 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -860,7 +860,7 @@ Please see the next question. Why does Borg disconnect or hang when backing up to a remote server? -------------------------------------------------------------------- -Communication with the remote server (e.g. using a ``rest://user@host/path`` repo URL) +Communication with the remote server (e.g. using an ``ssh://user@host/path`` repo URL) is tunneled through an SSH connection. This can lead to some issues that would not occur during a local backup: @@ -1142,7 +1142,7 @@ borgstore, which Borg uses for repository access, can limit the transfer rate:: Pros: -- works for all backends (``sftp://``, ``rest://``, ``rclone:``, ``s3://``, ...). +- works for all backends (``sftp://``, ``ssh://``, ``rclone:``, ``s3://``, ...). - limits both directions. - needs no additional software. @@ -1163,7 +1163,7 @@ backend call. Using pv on the ssh connection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -For ``rest://`` repositories, Borg connects via ssh, so the transfer can be +For ``ssh://`` repositories, Borg connects via ssh, so the transfer can be limited with pipeviewer_. Put a ``pv`` on each side of the connection using an ssh ``ProxyCommand`` (this needs ``nc``), e.g. in ``~/.ssh/config``:: @@ -1189,7 +1189,7 @@ Pros: Cons: -- only works for ``rest://`` repositories. ``sftp://`` does not run an external +- only works for ``ssh://`` repositories. ``sftp://`` does not run an external ssh command (borgstore uses paramiko and creates the connection itself) and a ``ProxyCommand`` in ``~/.ssh/config`` is not used there. - needs ``pv`` and ``nc``. diff --git a/docs/internals/security.rst b/docs/internals/security.rst index a9c8c2d6a..e5cfc21a1 100644 --- a/docs/internals/security.rst +++ b/docs/internals/security.rst @@ -402,10 +402,10 @@ used: Authorization and transport security come from the transport, not from borg. -Remote repositories over SSH: ``rest://`` +Remote repositories over SSH: ``ssh://`` ----------------------------------------- -For a ``rest://`` repository, borg runs ``borg serve --rest --backend FILE:`` +For an ``ssh://`` repository, borg runs ``borg serve --rest --backend FILE:`` on the remote machine and speaks HTTP to it over that process' *stdin/stdout* -- not over a socket. If the URL contains a host, the process is started through the system's SSH client (honouring ``BORG_RSH`` / ``BORGSTORE_RSH`` for the ssh command @@ -452,7 +452,7 @@ its dependencies, so their security properties are those of the respective libra - ``http(s)://`` talks to a borgstore REST server over plain HTTP, authenticating with HTTP Basic auth taken from the URL or from ``BORGSTORE_REST_USERNAME`` / ``BORGSTORE_REST_PASSWORD``. Basic auth sends the credentials to the server on - every request, so use ``https`` if you use this at all. ``rest://`` over SSH needs + every request, so use ``https`` if you use this at all. ``ssh://`` needs no such credentials. In every case the repository never receives the borg key or the passphrase, so a @@ -465,11 +465,10 @@ Legacy borg 1.x RPC protocol ``borg serve`` *without* ``--rest`` still speaks the borg 1.x RPC protocol: msgpack'd messages exchanged over stdio (``borg.legacy.remote``). It exists only so -that borg 2 can *read* a borg 1.x repository through an ``ssh://`` URL, e.g. for -``borg transfer --from-borg1``; ``ssh://`` is rejected for current repositories, and -this protocol cannot serve one. Its transport is the system's SSH client, so the same -"authorization and transport security are SSH's" reasoning as for ``rest://`` -applies. +that borg 2 can *read* a borg 1.x repository through an ``ssh://`` URL with +``--from-borg1``, e.g. for ``borg transfer --from-borg1``; it cannot serve a current +repository. Its transport is the system's SSH client, so the same +"authorization and transport security are SSH's" reasoning as for REST applies. Within that protocol, critical vulnerabilities such as remote code execution are inhibited by its design: @@ -526,8 +525,8 @@ while libssl implements TLS and related protocols. The latter historically contained most vulnerabilities, especially critical ones, and Borg's own extension modules do not use it: they link ``libcrypto`` only. Note that libssl can still be reached through Python's ``ssl`` module by the transports that do networking inside the borg -process, i.e. ``s3:``/``b2:`` and ``http(s)://`` (see :ref:`remote_access_security`); ``rest://`` -and the legacy ``ssh://`` protocol do not need it, as they only talk to a subprocess over pipes. +process, i.e. ``s3:``/``b2:`` and ``http(s)://`` (see :ref:`remote_access_security`); ``ssh://`` +(REST and the legacy protocol) does not need it, as it only talks to a subprocess over pipes. Accordingly, the binaries released by the project do include Python's ``ssl``/``_ssl`` modules (they are needed by pyfuse3/trio as well). diff --git a/docs/misc/asciinema/demo.tcl b/docs/misc/asciinema/demo.tcl index 32bdc8bab..30160db13 100644 --- a/docs/misc/asciinema/demo.tcl +++ b/docs/misc/asciinema/demo.tcl @@ -78,7 +78,7 @@ ls ~/Documents # Backups are stored in a "repository". Let's tell borg where ours is, # so we do not have to repeat it in every command: export BORG_REPO=/media/backup/borgdemo -# It could also live on another machine, e.g. rest://user@server/backup. +# It could also live on another machine, e.g. ssh://user@server/backup. # Creating the repository - encrypted and authenticated, with the key stored in the repo itself: borg repo-create --encryption=aes256-ocb --key-location=repokey diff --git a/docs/quickstart.rst b/docs/quickstart.rst index e901ad9c3..d561a0474 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -65,7 +65,7 @@ If you only back up your own files, run it as your normal user (i.e. not root). For a local repository always use the same user to invoke borg. -For a remote repository: always use e.g., rest://borg@remote_host/path/to/repo +For a remote repository: always use e.g., ssh://borg@remote_host/path/to/repo (Borg connects via ssh and runs ``borg serve --rest`` on the remote). You can use this from different local users; the remote user running borg and accessing the repo will always be `borg`. @@ -147,7 +147,7 @@ backed up and that the ``prune`` command keeps and deletes the correct backups. # Setting this, so the repo does not need to be given on the commandline. # One slash after the host means: relative to the remote login directory, # so this is ~/backup/main of the remote "username" user: - export BORG_REPO=rest://username@example.com:2022/backup/main + export BORG_REPO=ssh://username@example.com:2022/backup/main # See the section "Passphrase notes" for more infos. export BORG_PASSPHRASE='XYZl0ngandsecurepa_55_phrasea&&123' @@ -394,21 +394,22 @@ Remote repositories Borg can initialize and access repositories on remote hosts if the host is accessible using SSH. This is fastest and easiest when Borg -is installed on the remote host, in which case a ``rest://`` repository URL is +is installed on the remote host, in which case an ``ssh://`` repository URL is used. Borg connects via SSH and runs ``borg serve --rest`` on the remote host, which serves the repository talking HTTP over stdio:: - $ borg -r rest://user@hostname:port/path/to/repo repo-create ... + $ borg -r ssh://user@hostname:port/path/to/repo repo-create ... Note: Please see the usage chapter for a full documentation of repo URLs. Also see :ref:`ssh_configuration` for recommended settings to avoid disconnects and hangs. .. note:: - The legacy ``ssh://`` transport, served by ``borg serve`` on the remote host, - is now only used to access legacy borg 1.x (v1) repositories (e.g. via - ``borg transfer --from-borg1 --other-repo ssh://...``). For current - repositories, use a ``rest://`` repository as shown above. + An ``ssh://`` repository URL is also used to access legacy borg 1.x (v1) + repositories on a remote host, e.g. via + ``borg transfer --from-borg1 --other-repo ssh://...``. Due to ``--from-borg1``, + Borg then talks the legacy borg 1.x protocol to ``borg serve`` on the remote + host, which may also be a borg 1.x ``borg serve``. If it is not possible to install Borg on the remote host, it is still possible to use the remote host to store a repository by @@ -426,7 +427,7 @@ Other kinds of repositories --------------------------- Due to using the `borgstore` project, borg also supports other kinds of -(remote) repositories besides `file:` and `rest:`: +(remote) repositories besides `file:` and `ssh:`: - sftp: the borg client will directly talk to an sftp server. This does not require borg being installed on the sftp server. @@ -572,7 +573,7 @@ Example with **borg extract**: Difference when using a **remote borg backup server**: It is basically all the same as with the local repository, but you need to -refer to the repo using a ``rest://`` URL (Borg connects via ssh and runs +refer to the repo using an ``ssh://`` URL (Borg connects via ssh and runs ``borg serve --rest`` on the remote host). In the given example, ``borg`` is the user name used to log into the machine @@ -587,6 +588,6 @@ case if unattended, automated backups were done). :: - borg -r rest://borg@backup.example.org:2222/path/to/repo mount /mnt/borg + borg -r ssh://borg@backup.example.org:2222/path/to/repo mount /mnt/borg # or - borg -r rest://borg@backup.example.org:2222/path/to/repo extract archive + borg -r ssh://borg@backup.example.org:2222/path/to/repo extract archive diff --git a/docs/usage/general/file-systems.rst.inc b/docs/usage/general/file-systems.rst.inc index 1b750f70b..68661fb39 100644 --- a/docs/usage/general/file-systems.rst.inc +++ b/docs/usage/general/file-systems.rst.inc @@ -8,8 +8,8 @@ Borg now uses the ``borgstore`` package to implement the key/value store it uses for the repository. For a local repository (or a locally mounted network filesystem) it uses the -``file:`` store (posixfs backend). For a remote repository it uses the ``rest:`` -store, talking to a ``borg serve --rest`` process on the remote side (which in +``file:`` store (posixfs backend). For a remote ``ssh://`` repository it uses the +REST store, talking to a ``borg serve --rest`` process on the remote side (which in turn uses borgstore there). Other backends (``sftp:``, ``rclone:``, ``s3:``/ ``b2:``) do not need a filesystem on the repository side at all. diff --git a/docs/usage/general/repository-urls.rst.inc b/docs/usage/general/repository-urls.rst.inc index ae070cecc..6a7078abc 100644 --- a/docs/usage/general/repository-urls.rst.inc +++ b/docs/usage/general/repository-urls.rst.inc @@ -18,22 +18,16 @@ Note: UNC paths (``//server/share/path``, ``\\server\share\path``) are not supported — mount the share (on Windows: map it to a drive letter, e.g. ``net use X: \\server\share``) and use the mounted path instead. -**Remote repositories** accessed via SSH user@host (REST http over stdio): - -``rest://user@host:port//abs/path/to/repo`` — absolute path - -``rest://user@host:port/rel/path/to/repo`` — path relative to the remote login directory - -**Remote repositories** accessed via SSH user@host (legacy borg RPC protocol): +**Remote repositories** accessed via SSH user@host (borg serve on the remote host): ``ssh://user@host:port//abs/path/to/repo`` — absolute path ``ssh://user@host:port/rel/path/to/repo`` — path relative to the remote login directory -For current (non-legacy) repositories, ``ssh://`` is rejected; use ``rest://`` -instead, which can also tunnel over ssh (see above). ``ssh://`` remains available -only for legacy borg 1.x repositories, e.g. via -``borg transfer --from-borg1 --other-repo ssh://...``. +Borg talks HTTP (REST) over the ssh connection's stdio to ``borg serve --rest`` on the +remote host. Legacy borg 1.x repositories are also accessed via ``ssh://`` URLs, +together with ``--from-borg1`` (e.g. ``borg transfer --from-borg1 --other-repo ssh://...``): +Borg then talks the legacy borg 1.x protocol to ``borg serve`` on the remote host. **Remote repositories** accessed via SFTP: @@ -41,8 +35,8 @@ only for legacy borg 1.x repositories, e.g. via ``sftp://user@host:port/rel/path/to/repo`` — path relative to the remote login directory -For REST, SSH and SFTP URLs, the ``user@`` and ``:port`` parts are optional, but the -path is required: a URL without one, e.g. ``rest://host`` or ``rest://host/``, is +For SSH and SFTP URLs, the ``user@`` and ``:port`` parts are optional, but the +path is required: a URL without one, e.g. ``ssh://host`` or ``ssh://host/``, is rejected. Mind the difference between one and two slashes after the host: one slash means a path relative to the directory the remote login lands in (usually the remote user's home directory), two slashes mean an absolute path. @@ -59,7 +53,7 @@ Authentication is HTTP Basic auth: credentials come from the URL if given, other from the ``BORGSTORE_REST_USERNAME`` / ``BORGSTORE_REST_PASSWORD`` environment variables; prefer ``https://`` over ``http://`` whenever credentials are used, since Basic auth sends them on every request. A URL path after the host is optional and, -unlike ``rest://``, is not a remote filesystem path — it does not follow the +unlike ``ssh://``, is not a remote filesystem path — it does not follow the one-vs-two-slash rule above and is only needed to reach the server through a reverse proxy mounted below a sub-path. @@ -82,7 +76,7 @@ If you frequently need the same repository URL, it is a good idea to set the :: - export BORG_REPO='rest://user@host:port/rel/path/to/repo' + export BORG_REPO='ssh://user@host:port/rel/path/to/repo' Then simply omit the ``--repo`` option when you want to use the default — it will be read from BORG_REPO. diff --git a/docs/usage/serve.rst b/docs/usage/serve.rst index 127dec5bd..145cad65f 100644 --- a/docs/usage/serve.rst +++ b/docs/usage/serve.rst @@ -19,9 +19,8 @@ options. This list currently contains: ``--restrict-to-path`` / ``--restrict-to-repository``. Notably, ``--rest`` is *not* in the allowlist, so the forced command pins the -mode (current ``rest://`` repositories vs. legacy Borg 1.x repositories), and -neither are ``--restrict-to-path``, ``--restrict-to-repository``, ``--umask`` -and ``--permissions``. +mode (current repositories vs. legacy Borg 1.x repositories), and neither are +``--restrict-to-path``, ``--restrict-to-repository``, ``--umask`` and ``--permissions``. Environment variables (such as BORG_XXX) contained in the original command sent by the client are *not* interpreted, but ignored. If BORG_XXX environment @@ -33,7 +32,7 @@ locations like ``/etc/environment`` or in the forced command itself (example bel # Allow an SSH keypair to run only borg, and only have access to /path/to/repo. # Use key options to disable unneeded and potentially dangerous SSH functionality. # This will help to secure an automated remote backup system. - # --rest serves a current (rest://) repository; without it, a legacy Borg 1.x + # --rest serves a current (ssh://) repository; without it, a legacy Borg 1.x # repository would be served. The client supplies --backend FILE:, # which is validated against --restrict-to-path. $ cat ~/.ssh/authorized_keys diff --git a/docs/usage/transfer.rst b/docs/usage/transfer.rst index 705a4fc2e..0875157f1 100644 --- a/docs/usage/transfer.rst +++ b/docs/usage/transfer.rst @@ -8,9 +8,9 @@ locations and passphrases first: :: - # The destination is a Borg 2 repository: use a rest:// (or local) location, - # ssh:// is only supported for the legacy Borg 1.x source repository. - export BORG_REPO=rest://borg2@borgbackup/tests/b20 + # The destination is a Borg 2 repository, the source is a legacy Borg 1.x repository + # (accessed via the legacy protocol due to --from-borg1). + export BORG_REPO=ssh://borg2@borgbackup/tests/b20 export BORG_PASSPHRASE='your-borg2-repo-passphrase' export BORG_OTHER_REPO=ssh://borg2@borgbackup/./tests/b1x export BORG_OTHER_PASSPHRASE='your-borg1-repo-passphrase' diff --git a/scripts/rest_perf_test.sh b/scripts/rest_perf_test.sh index 69cb14d3f..7c0d445e5 100755 --- a/scripts/rest_perf_test.sh +++ b/scripts/rest_perf_test.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # -# rest_perf_test.sh - borg2 performance harness for rest:// repositories. +# rest_perf_test.sh - borg2 performance harness for ssh:// repositories. # # Runs a fixed workload (repo-create -> backup -> backup again -> dry-run extract -# -> check) against a rest:// repo served over a localhost sshd, under several +# -> check) against an ssh:// repo served over a localhost sshd, under several # network conditions emulated by borgstore (BORGSTORE_LATENCY / BORGSTORE_BANDWIDTH). # LAN and WAN use the SAME bandwidth (1 Gbit/s) and differ only in latency (0.3ms vs # 30ms), to isolate latency's effect; raw is a no-emulation baseline. Prints a table. @@ -35,8 +35,8 @@ set -euo pipefail # Default ~7.4GB: 20000x20KiB + 1000x5MiB + 2x1GiB. Override GROUPS_SPEC for smoke tests. : "${GROUPS_SPEC:=tiny:20000:20480 med:1000:5242880 big:2:1073741824}" -# rest://user@host//abs/path (the double slash is intentional: //tmp/REPO is absolute) -REPO_URL="rest://${REPO_TARGET}/${REPO_PATH}" +# ssh://user@host//abs/path (the double slash is intentional: //tmp/REPO is absolute) +REPO_URL="ssh://${REPO_TARGET}/${REPO_PATH}" # Per-profile emulation values: LATENCY in microseconds, BANDWIDTH in bits/s (0 = off). # lan and wan share the SAME bandwidth (1 Gbit/s); only latency differs, to isolate how @@ -52,7 +52,7 @@ profile_params() { # Suppress borg's interactive prompts (unencrypted repo access, repo deletion) and # point borg at the venv borg on the *remote* side (a bare 'borg' would not resolve -# in a non-interactive ssh session's minimal PATH). rest:// is served by 'borg serve +# in a non-interactive ssh session's minimal PATH). ssh:// is served by 'borg serve # --rest', so BORG_REMOTE_PATH is the command the server runs. export BORG_REMOTE_PATH="$VENV/bin/borg" export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes