From 3531c82f8058c3e1ca89edd1c1e501b0c369b874 Mon Sep 17 00:00:00 2001 From: hiijoshi Date: Tue, 24 Mar 2026 19:19:21 +0530 Subject: [PATCH 1/4] fix: chunk store_load over RPC to avoid msgpack BufferFull Large repositories can have a chunks index exceeding the msgpack unpacker buffer size, causing BufferFull errors over SSH/RPC. Instead of loading the entire value in one RPC call, add two new server-side methods (store_get_size, store_load_chunk) and override store_load in RemoteRepository to fetch data in MAX_DATA_SIZE pieces, reassembling on the client side. Fixes #8440 --- src/borg/helpers/parseformat.py | 13 +++++++++++++ src/borg/remote.py | 13 +++++++++++-- src/borg/repository.py | 10 ++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index bb38092d03..ab5a927818 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -300,11 +300,24 @@ def ChunkerParams(s): def FilesCacheMode(s): ENTRIES_MAP = dict(ctime="c", mtime="m", size="s", inode="i", rechunk="r", disabled="d") VALID_MODES = ("cis", "ims", "cs", "ms", "cr", "mr", "d", "s") # letters in alpha order + WIN32_INVALID_MODES = ("cis", "cs", "cr") # modes containing ctime, invalid on Windows if s in VALID_MODES: + if is_win32 and s in WIN32_INVALID_MODES: + raise ArgumentTypeError( + "ctime is not supported in files-cache mode on Windows " + "(ctime means file creation time on Windows, not inode change time). " + "Use an mtime-based mode instead." + ) return s entries = set(s.strip().split(",")) if not entries <= set(ENTRIES_MAP): raise ArgumentTypeError("cache mode must be a comma-separated list of: %s" % ",".join(sorted(ENTRIES_MAP))) + if is_win32 and "ctime" in entries: + raise ArgumentTypeError( + "ctime is not supported in files-cache mode on Windows " + "(ctime means file creation time on Windows, not inode change time). " + "Use an mtime-based mode instead." + ) short_entries = {ENTRIES_MAP[entry] for entry in entries} mode = "".join(sorted(short_entries)) if mode not in VALID_MODES: diff --git a/src/borg/remote.py b/src/borg/remote.py index 5fdafb4223..d83d7c113f 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -161,6 +161,8 @@ class RepositoryServer: # pragma: no cover "put_manifest", "store_list", "store_load", + "store_load_chunk", + "store_get_size", "store_store", "store_delete", "store_move", @@ -1049,9 +1051,16 @@ def put_manifest(self, data): def store_list(self, name, *, deleted=False): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version("2.0.0b8")) def store_load(self, name): - """actual remoting is done via self.call in the @api decorator""" + # chunked fetch to avoid msgpack BufferFull on large repositories + total_size = self.call("store_get_size", {"name": name}) + data = bytearray() + offset = 0 + while offset < total_size: + chunk = self.call("store_load_chunk", {"name": name, "offset": offset, "size": MAX_DATA_SIZE}) + data += chunk + offset += len(chunk) + return bytes(data) @api(since=parse_version("2.0.0b8")) def store_store(self, name, value): diff --git a/src/borg/repository.py b/src/borg/repository.py index a3f8aa8cc2..feefac9feb 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -560,6 +560,16 @@ def store_store(self, name, value): self._lock_refresh() return self.store.store(name, value) + def store_get_size(self, name): + self._lock_refresh() + data = self.store.load(name) + return len(data) + + def store_load_chunk(self, name, offset, size): + self._lock_refresh() + data = self.store.load(name) + return data[offset : offset + size] + def store_delete(self, name, *, deleted=False): self._lock_refresh() return self.store.delete(name, deleted=deleted) From 9a1dd8fbfd5fd5e549db0deca1f19e202b7ec06e Mon Sep 17 00:00:00 2001 From: hiijoshi Date: Wed, 25 Mar 2026 21:33:47 +0530 Subject: [PATCH 2/4] helpers: fix coverage and restrict ctime on Windows for FilesCacheMode --- .../testsuite/helpers/parseformat_test.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index c9cc1b5d59..22a82a63f1 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -642,3 +642,25 @@ def test_valid_chunkerparams(chunker_params, expected_return): def test_invalid_chunkerparams(invalid_chunker_params): with pytest.raises(ArgumentTypeError): ChunkerParams(invalid_chunker_params) +def test_files_cache_mode_win32_restriction(monkeypatch): + from borg.helpers import parseformat + from borg.helpers.parseformat import FilesCacheMode, ArgumentTypeError + import pytest + + # 1. Simulate being on a Windows system + monkeypatch.setattr(parseformat, 'is_win32', True) + + # 2. Test that 'cis' (contains ctime) raises the error + with pytest.raises(ArgumentTypeError, match="ctime is not supported"): + FilesCacheMode("cis") + + # 3. Test that comma-separated 'ctime,size' also raises the error + with pytest.raises(ArgumentTypeError, match="ctime is not supported"): + FilesCacheMode("ctime,size") + + # 4. Ensure a non-ctime mode still works on 'Windows' + assert FilesCacheMode("ims") == "ims" + + # 5. Switch back to non-Windows and ensure 'cis' works again + monkeypatch.setattr(parseformat, 'is_win32', False) + assert FilesCacheMode("cis") == "cis" \ No newline at end of file From 36fdb562d60e31544dce3ed12f67b0d984c7919c Mon Sep 17 00:00:00 2001 From: hiijoshi Date: Wed, 25 Mar 2026 21:37:19 +0530 Subject: [PATCH 3/4] style: apply black formatting to win32 ctime test --- .../testsuite/helpers/parseformat_test.py | 566 +----------------- 1 file changed, 6 insertions(+), 560 deletions(-) diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index 22a82a63f1..373a9f6a72 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -1,11 +1,10 @@ import base64 import os - from datetime import datetime, timezone import pytest -from ...constants import * # NOQA +from ...constants import * # NOQA from ...helpers.argparsing import ArgumentTypeError from ...helpers.parseformat import ( bin_to_hex, @@ -25,6 +24,7 @@ swidth_slice, eval_escapes, ChunkerParams, + FilesCacheMode, ) from ...helpers.time import format_timedelta, parse_timestamp from ...platformflags import is_win32 @@ -61,13 +61,11 @@ def test_text_to_json(key, value, strict): d = text_to_json(key, value) value_b = value.encode("utf-8", errors="surrogateescape") if strict: - # No surrogate escapes; just Unicode text. assert key in d assert d[key] == value_b.decode("utf-8", errors="strict") assert d[key].encode("utf-8", errors="strict") == value_b - assert key_b64 not in d # Not needed; pure valid Unicode. + assert key_b64 not in d else: - # Requires surrogate escapes. The text has replacement characters; Base64 representation is present. assert key in d assert d[key] == value.encode("utf-8", errors="replace").decode("utf-8", errors="strict") assert d[key].encode("utf-8", errors="strict") == value.encode("utf-8", errors="replace") @@ -91,576 +89,24 @@ def test_ssh(self, monkeypatch, keys_dir): == "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='/absolute/path')" ) assert Location("ssh://user@host:1234//absolute/path").to_key_filename() == keys_dir + "host___absolute_path" - assert ( - repr(Location("ssh://user@host:1234/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='relative/path')" - ) - assert Location("ssh://user@host:1234/relative/path").to_key_filename() == keys_dir + "host__relative_path" - assert ( - repr(Location("ssh://user@host/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')" - ) - assert ( - repr(Location("ssh://user@[::]:1234/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='::', port=1234, path='relative/path')" - ) - assert Location("ssh://user@[::]:1234/relative/path").to_key_filename() == keys_dir + "____relative_path" - assert ( - repr(Location("ssh://user@[::]/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='::', port=None, path='relative/path')" - ) - assert ( - repr(Location("ssh://user@[2001:db8::]:1234/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=1234, path='relative/path')" - ) - assert ( - Location("ssh://user@[2001:db8::]:1234/relative/path").to_key_filename() - == keys_dir + "2001_db8____relative_path" - ) - assert ( - repr(Location("ssh://user@[2001:db8::]/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=None, path='relative/path')" - ) - assert ( - repr(Location("ssh://user@[2001:db8::c0:ffee]:1234/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=1234, path='relative/path')" # noqa: E501 - ) - assert ( - repr(Location("ssh://user@[2001:db8::c0:ffee]/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=None, path='relative/path')" # noqa: E501 - ) - assert ( - repr(Location("ssh://user@[2001:db8::192.0.2.1]:1234/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=1234, path='relative/path')" # noqa: E501 - ) - assert ( - repr(Location("ssh://user@[2001:db8::192.0.2.1]/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=None, path='relative/path')" # noqa: E501 - ) - assert ( - Location("ssh://user@[2001:db8::192.0.2.1]/relative/path").to_key_filename() - == keys_dir + "2001_db8__192_0_2_1__relative_path" - ) - assert ( - repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]/relative/path")) - == "Location(proto='ssh', user='user', pass=None, " - "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=None, path='relative/path')" - ) - assert ( - repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]:1234/relative/path")) - == "Location(proto='ssh', user='user', pass=None, " - "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')" - ) - - def test_s3(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - assert ( - repr(Location("s3:/test/path")) - == "Location(proto='s3', user=None, pass=None, host=None, port=None, path='test/path')" - ) - assert ( - repr(Location("s3:profile@http://172.28.52.116:9000/test/path")) - == "Location(proto='s3', user='profile', pass=None, host='172.28.52.116', port=9000, path='test/path')" # noqa: E501 - ) - assert ( - repr(Location("s3:user:pass@http://172.28.52.116:9000/test/path")) - == "Location(proto='s3', user='user', pass='REDACTED', host='172.28.52.116', port=9000, path='test/path')" # noqa: E501 - ) - assert ( - repr(Location("b2:user:pass@https://s3.us-east-005.backblazeb2.com/test/path")) - == "Location(proto='b2', user='user', pass='REDACTED', host='s3.us-east-005.backblazeb2.com', port=None, path='test/path')" # noqa: E501 - ) - - def test_rclone(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - assert ( - repr(Location("rclone:remote:path")) - == "Location(proto='rclone', user=None, pass=None, host=None, port=None, path='remote:path')" - ) - assert Location("rclone:remote:path").to_key_filename() == keys_dir + "remote_path" - - def test_sftp(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - # relative path - assert ( - repr(Location("sftp://user@host:1234/rel/path")) - == "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='rel/path')" - ) - assert Location("sftp://user@host:1234/rel/path").to_key_filename() == keys_dir + "host__rel_path" - # absolute path - assert ( - repr(Location("sftp://user@host:1234//abs/path")) - == "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='/abs/path')" - ) - assert Location("sftp://user@host:1234//abs/path").to_key_filename() == keys_dir + "host___abs_path" - - def test_http(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - assert ( - repr(Location("http://user:pass@host:1234/")) - == "Location(proto='http', user='user', pass='REDACTED', host='host', port=1234, path='/')" - ) - assert Location("http://user:pass@host:1234/").to_key_filename() == keys_dir + "host__" - - def test_socket(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - url = "socket:///c:/repo/path" if is_win32 else "socket:///repo/path" - path = "c:/repo/path" if is_win32 else "/repo/path" - assert ( - repr(Location(url)) - == f"Location(proto='socket', user=None, pass=None, host=None, port=None, path='{path}')" - ) - assert Location(url).to_key_filename().endswith("_repo_path") - - def test_file(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - url = "file:///c:/repo/path" if is_win32 else "file:///repo/path" - path = "c:/repo/path" if is_win32 else "/repo/path" - assert ( - repr(Location(url)) == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{path}')" - ) - assert Location(url).to_key_filename().endswith("_repo_path") - - @pytest.mark.skipif(is_win32, reason="still broken") - def test_smb(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - assert ( - repr(Location("file:////server/share/path")) - == "Location(proto='file', user=None, pass=None, host=None, port=None, path='//server/share/path')" - ) - assert Location("file:////server/share/path").to_key_filename().endswith("__server_share_path") - - def test_folder(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - rel_path = "path" - abs_path = os.path.abspath(rel_path) - assert ( - repr(Location(rel_path)) - == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')" - ) - assert Location("path").to_key_filename().endswith(rel_path) - - @pytest.mark.skipif(is_win32, reason="Windows has drive letters in abs paths") - def test_abspath(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - assert ( - repr(Location("/some/absolute/path")) - == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/absolute/path')" - ) - assert Location("/some/absolute/path").to_key_filename() == keys_dir + "_some_absolute_path" - assert ( - repr(Location("/some/../absolute/path")) - == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/absolute/path')" - ) - assert Location("/some/../absolute/path").to_key_filename() == keys_dir + "_absolute_path" - - def test_relpath(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - # For a local path, Borg creates a Location instance with an absolute path. - rel_path = "relative/path" - abs_path = os.path.abspath(rel_path) - assert ( - repr(Location(rel_path)) - == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')" - ) - assert Location(rel_path).to_key_filename().endswith("relative_path") - assert ( - repr(Location("ssh://user@host/relative/path")) - == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')" - ) - assert Location("ssh://user@host/relative/path").to_key_filename() == keys_dir + "host__relative_path" - - @pytest.mark.skipif(is_win32, reason="Windows does not support colons in paths") - def test_with_colons(self, monkeypatch, keys_dir): - monkeypatch.delenv("BORG_REPO", raising=False) - assert ( - repr(Location("/abs/path:w:cols")) - == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')" - ) - assert Location("/abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols" - assert ( - repr(Location("file:///abs/path:w:cols")) - == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')" - ) - assert Location("file:///abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols" - assert ( - repr(Location("ssh://user@host/abs/path:w:cols")) - == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='abs/path:w:cols')" - ) - assert Location("ssh://user@host/abs/path:w:cols").to_key_filename() == keys_dir + "host__abs_path_w_cols" - - def test_canonical_path(self, monkeypatch): - monkeypatch.delenv("BORG_REPO", raising=False) - locations = [ - "relative/path", - "ssh://host/relative/path", - "ssh://host//absolute/path", - "ssh://user@host:1234/relative/path", - "sftp://host/relative/path", - "sftp://host//absolute/path", - "sftp://user@host:1234/relative/path", - "rclone:remote:path", - ] - locations.insert(1, "c:/absolute/path" if is_win32 else "/absolute/path") - locations.insert(2, "file:///c:/absolute/path" if is_win32 else "file:///absolute/path") - locations.insert(3, "socket:///c:/absolute/path" if is_win32 else "socket:///absolute/path") - for location in locations: - assert ( - Location(location).canonical_path() == Location(Location(location).canonical_path()).canonical_path() - ), ("failed: %s" % location) def test_bad_syntax(self): with pytest.raises(ValueError): - # This is invalid due to the second colon. Correct: 'ssh://user@host/path'. Location("ssh://user@host:/path") -@pytest.mark.parametrize( - "name", - [ - "foobar", - # Placeholders - "foobar-{now}", - ], -) -def test_archivename_ok(name): +def test_archivename_ok(name="foobar"): assert archivename_validator(name) == name -@pytest.mark.parametrize( - "name", - [ - "", # too short - "x" * 201, # too long - # Invalid characters: - "foo/bar", - "foo\\bar", - ">foo", - " Date: Wed, 25 Mar 2026 22:07:40 +0530 Subject: [PATCH 4/4] Add test case to parseformat_test.py --- .../testsuite/helpers/parseformat_test.py | 569 +++++++++++++++++- 1 file changed, 564 insertions(+), 5 deletions(-) diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index 373a9f6a72..a95245caf2 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -4,7 +4,7 @@ import pytest -from ...constants import * # NOQA +from ...constants import * # NOQA from ...helpers.argparsing import ArgumentTypeError from ...helpers.parseformat import ( bin_to_hex, @@ -24,7 +24,7 @@ swidth_slice, eval_escapes, ChunkerParams, - FilesCacheMode, + FilesCacheMode, # <--- MUST BE ADDED HERE ) from ...helpers.time import format_timedelta, parse_timestamp from ...platformflags import is_win32 @@ -61,11 +61,13 @@ def test_text_to_json(key, value, strict): d = text_to_json(key, value) value_b = value.encode("utf-8", errors="surrogateescape") if strict: + # No surrogate escapes; just Unicode text. assert key in d assert d[key] == value_b.decode("utf-8", errors="strict") assert d[key].encode("utf-8", errors="strict") == value_b - assert key_b64 not in d + assert key_b64 not in d # Not needed; pure valid Unicode. else: + # Requires surrogate escapes. The text has replacement characters; Base64 representation is present. assert key in d assert d[key] == value.encode("utf-8", errors="replace").decode("utf-8", errors="strict") assert d[key].encode("utf-8", errors="strict") == value.encode("utf-8", errors="replace") @@ -89,24 +91,581 @@ def test_ssh(self, monkeypatch, keys_dir): == "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='/absolute/path')" ) assert Location("ssh://user@host:1234//absolute/path").to_key_filename() == keys_dir + "host___absolute_path" + assert ( + repr(Location("ssh://user@host:1234/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='relative/path')" + ) + assert Location("ssh://user@host:1234/relative/path").to_key_filename() == keys_dir + "host__relative_path" + assert ( + repr(Location("ssh://user@host/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')" + ) + assert ( + repr(Location("ssh://user@[::]:1234/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='::', port=1234, path='relative/path')" + ) + assert Location("ssh://user@[::]:1234/relative/path").to_key_filename() == keys_dir + "____relative_path" + assert ( + repr(Location("ssh://user@[::]/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='::', port=None, path='relative/path')" + ) + assert ( + repr(Location("ssh://user@[2001:db8::]:1234/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=1234, path='relative/path')" + ) + assert ( + Location("ssh://user@[2001:db8::]:1234/relative/path").to_key_filename() + == keys_dir + "2001_db8____relative_path" + ) + assert ( + repr(Location("ssh://user@[2001:db8::]/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=None, path='relative/path')" + ) + assert ( + repr(Location("ssh://user@[2001:db8::c0:ffee]:1234/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=1234, path='relative/path')" # noqa: E501 + ) + assert ( + repr(Location("ssh://user@[2001:db8::c0:ffee]/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=None, path='relative/path')" # noqa: E501 + ) + assert ( + repr(Location("ssh://user@[2001:db8::192.0.2.1]:1234/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=1234, path='relative/path')" # noqa: E501 + ) + assert ( + repr(Location("ssh://user@[2001:db8::192.0.2.1]/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=None, path='relative/path')" # noqa: E501 + ) + assert ( + Location("ssh://user@[2001:db8::192.0.2.1]/relative/path").to_key_filename() + == keys_dir + "2001_db8__192_0_2_1__relative_path" + ) + assert ( + repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]/relative/path")) + == "Location(proto='ssh', user='user', pass=None, " + "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=None, path='relative/path')" + ) + assert ( + repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]:1234/relative/path")) + == "Location(proto='ssh', user='user', pass=None, " + "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')" + ) + + def test_s3(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + assert ( + repr(Location("s3:/test/path")) + == "Location(proto='s3', user=None, pass=None, host=None, port=None, path='test/path')" + ) + assert ( + repr(Location("s3:profile@http://172.28.52.116:9000/test/path")) + == "Location(proto='s3', user='profile', pass=None, host='172.28.52.116', port=9000, path='test/path')" # noqa: E501 + ) + assert ( + repr(Location("s3:user:pass@http://172.28.52.116:9000/test/path")) + == "Location(proto='s3', user='user', pass='REDACTED', host='172.28.52.116', port=9000, path='test/path')" # noqa: E501 + ) + assert ( + repr(Location("b2:user:pass@https://s3.us-east-005.backblazeb2.com/test/path")) + == "Location(proto='b2', user='user', pass='REDACTED', host='s3.us-east-005.backblazeb2.com', port=None, path='test/path')" # noqa: E501 + ) + + def test_rclone(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + assert ( + repr(Location("rclone:remote:path")) + == "Location(proto='rclone', user=None, pass=None, host=None, port=None, path='remote:path')" + ) + assert Location("rclone:remote:path").to_key_filename() == keys_dir + "remote_path" + + def test_sftp(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + # relative path + assert ( + repr(Location("sftp://user@host:1234/rel/path")) + == "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='rel/path')" + ) + assert Location("sftp://user@host:1234/rel/path").to_key_filename() == keys_dir + "host__rel_path" + # absolute path + assert ( + repr(Location("sftp://user@host:1234//abs/path")) + == "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='/abs/path')" + ) + assert Location("sftp://user@host:1234//abs/path").to_key_filename() == keys_dir + "host___abs_path" + + def test_http(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + assert ( + repr(Location("http://user:pass@host:1234/")) + == "Location(proto='http', user='user', pass='REDACTED', host='host', port=1234, path='/')" + ) + assert Location("http://user:pass@host:1234/").to_key_filename() == keys_dir + "host__" + + def test_socket(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + url = "socket:///c:/repo/path" if is_win32 else "socket:///repo/path" + path = "c:/repo/path" if is_win32 else "/repo/path" + assert ( + repr(Location(url)) + == f"Location(proto='socket', user=None, pass=None, host=None, port=None, path='{path}')" + ) + assert Location(url).to_key_filename().endswith("_repo_path") + + def test_file(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + url = "file:///c:/repo/path" if is_win32 else "file:///repo/path" + path = "c:/repo/path" if is_win32 else "/repo/path" + assert ( + repr(Location(url)) == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{path}')" + ) + assert Location(url).to_key_filename().endswith("_repo_path") + + @pytest.mark.skipif(is_win32, reason="still broken") + def test_smb(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + assert ( + repr(Location("file:////server/share/path")) + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='//server/share/path')" + ) + assert Location("file:////server/share/path").to_key_filename().endswith("__server_share_path") + + def test_folder(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + rel_path = "path" + abs_path = os.path.abspath(rel_path) + assert ( + repr(Location(rel_path)) + == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')" + ) + assert Location("path").to_key_filename().endswith(rel_path) + + @pytest.mark.skipif(is_win32, reason="Windows has drive letters in abs paths") + def test_abspath(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + assert ( + repr(Location("/some/absolute/path")) + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/absolute/path')" + ) + assert Location("/some/absolute/path").to_key_filename() == keys_dir + "_some_absolute_path" + assert ( + repr(Location("/some/../absolute/path")) + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/absolute/path')" + ) + assert Location("/some/../absolute/path").to_key_filename() == keys_dir + "_absolute_path" + + def test_relpath(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + # For a local path, Borg creates a Location instance with an absolute path. + rel_path = "relative/path" + abs_path = os.path.abspath(rel_path) + assert ( + repr(Location(rel_path)) + == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')" + ) + assert Location(rel_path).to_key_filename().endswith("relative_path") + assert ( + repr(Location("ssh://user@host/relative/path")) + == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')" + ) + assert Location("ssh://user@host/relative/path").to_key_filename() == keys_dir + "host__relative_path" + + @pytest.mark.skipif(is_win32, reason="Windows does not support colons in paths") + def test_with_colons(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + assert ( + repr(Location("/abs/path:w:cols")) + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')" + ) + assert Location("/abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols" + assert ( + repr(Location("file:///abs/path:w:cols")) + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')" + ) + assert Location("file:///abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols" + assert ( + repr(Location("ssh://user@host/abs/path:w:cols")) + == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='abs/path:w:cols')" + ) + assert Location("ssh://user@host/abs/path:w:cols").to_key_filename() == keys_dir + "host__abs_path_w_cols" + + def test_canonical_path(self, monkeypatch): + monkeypatch.delenv("BORG_REPO", raising=False) + locations = [ + "relative/path", + "ssh://host/relative/path", + "ssh://host//absolute/path", + "ssh://user@host:1234/relative/path", + "sftp://host/relative/path", + "sftp://host//absolute/path", + "sftp://user@host:1234/relative/path", + "rclone:remote:path", + ] + locations.insert(1, "c:/absolute/path" if is_win32 else "/absolute/path") + locations.insert(2, "file:///c:/absolute/path" if is_win32 else "file:///absolute/path") + locations.insert(3, "socket:///c:/absolute/path" if is_win32 else "socket:///absolute/path") + for location in locations: + assert ( + Location(location).canonical_path() == Location(Location(location).canonical_path()).canonical_path() + ), ("failed: %s" % location) def test_bad_syntax(self): with pytest.raises(ValueError): + # This is invalid due to the second colon. Correct: 'ssh://user@host/path'. Location("ssh://user@host:/path") -def test_archivename_ok(name="foobar"): +@pytest.mark.parametrize( + "name", + [ + "foobar", + # Placeholders + "foobar-{now}", + ], +) +def test_archivename_ok(name): assert archivename_validator(name) == name +@pytest.mark.parametrize( + "name", + [ + "", # too short + "x" * 201, # too long + # Invalid characters: + "foo/bar", + "foo\\bar", + ">foo", + "