From ade20d57d7ffac13f9053fa91f61a97f987639d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 16:46:51 +0100 Subject: [PATCH 01/11] Add a propper error message if using s3/b2 urls but not have boto3 installed. Also add boto3 to dev dependencies --- README.rst | 4 ++++ src/borgstore/backends/errors.py | 3 +++ src/borgstore/backends/s3.py | 5 ++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 51b456d..8de2229 100644 --- a/README.rst +++ b/README.rst @@ -221,6 +221,10 @@ Use storage on an S3-compliant cloud service: There is a known issue with some S3-compatible services (e.g., **Backblaze B2**). If you encounter problems, try using ``b2:`` instead of ``s3:`` in the URL. + .. note:: + + You need to install ``boto3`` to be able to use s3/b2 backends. Run ``pip install boto3`` in your venv. + - Namespaces: directories - Values: in key-named files diff --git a/src/borgstore/backends/errors.py b/src/borgstore/backends/errors.py index c8b8356..291c74f 100644 --- a/src/borgstore/backends/errors.py +++ b/src/borgstore/backends/errors.py @@ -37,3 +37,6 @@ class ObjectNotFound(BackendError): class PermissionDenied(BackendError): """Permission denied for the requested operation.""" + +class DependencyMissing(BackendError): + """Permission denied for the requested operation.""" diff --git a/src/borgstore/backends/s3.py b/src/borgstore/backends/s3.py index c4fa602..8b7383a 100644 --- a/src/borgstore/backends/s3.py +++ b/src/borgstore/backends/s3.py @@ -14,7 +14,7 @@ from borgstore.constants import TMP_SUFFIX from ._base import BackendBase, ItemInfo, validate_name -from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists +from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists, DependencyMissing from .errors import ObjectNotFound @@ -25,6 +25,9 @@ def get_s3_backend(url: str): (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path """ if boto3 is None: + if url.startswith("s3:") or url.startswith("b2"): + raise DependencyMissing("Backend url seems to be an s3/b2 url but 'boto3' lib is not installed.") + return None # (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path From 2df3cbbe44300cac5559116203a4acb1b78fc5b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 16:47:35 +0100 Subject: [PATCH 02/11] Add boto3 to dev dependencies --- requirements.d/dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.d/dev.txt b/requirements.d/dev.txt index 8507eb3..f4acb23 100644 --- a/requirements.d/dev.txt +++ b/requirements.d/dev.txt @@ -3,3 +3,4 @@ tox pytest build twine +boto3 From d14ceccc3b0c164098bb8c36f912d5a37d59fc7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 19:25:32 +0100 Subject: [PATCH 03/11] Rearrange code and update error message. Also Check requirements in SFTP backend --- src/borgstore/backends/s3.py | 7 ++++--- src/borgstore/backends/sftp.py | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/borgstore/backends/s3.py b/src/borgstore/backends/s3.py index 8b7383a..7ac66ea 100644 --- a/src/borgstore/backends/s3.py +++ b/src/borgstore/backends/s3.py @@ -24,12 +24,13 @@ def get_s3_backend(url: str): Supports URLs of the form: (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path """ - if boto3 is None: - if url.startswith("s3:") or url.startswith("b2"): - raise DependencyMissing("Backend url seems to be an s3/b2 url but 'boto3' lib is not installed.") + if not url.startswith("s3:") and not url.startswith("b2:"): return None + if boto3 is None: + raise DependencyMissing("The S3 backend requires dependencies. Install it with 'pip install borgstore[s3]'") + # (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path s3_regex = r""" (?P(s3|b2)): diff --git a/src/borgstore/backends/sftp.py b/src/borgstore/backends/sftp.py index 6906358..1a3834f 100644 --- a/src/borgstore/backends/sftp.py +++ b/src/borgstore/backends/sftp.py @@ -14,13 +14,22 @@ paramiko = None from ._base import BackendBase, ItemInfo, validate_name -from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists +from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists, \ + DependencyMissing from .errors import ObjectNotFound from ..constants import TMP_SUFFIX def get_sftp_backend(url): """Get SFTP backend from URL.""" + + if not url.startswith("sftp://"): + return None + + if paramiko is not None: + raise DependencyMissing("The SFTP backend requires dependencies. Install it with 'pip install borgstore[sftp]'") + + # sftp://username@hostname:22/path # Notes: # - username and port are optional @@ -34,10 +43,9 @@ def get_sftp_backend(url): (?P([^:/]+))(?::(?P\d+))?/ # slash as separator, not part of the path (?P(.+)) # path may or may not start with a slash, must not be empty """ - if paramiko is not None: - m = re.match(sftp_regex, url, re.VERBOSE) - if m: - return Sftp(username=m["username"], hostname=m["hostname"], port=int(m["port"] or "0"), path=m["path"]) + m = re.match(sftp_regex, url, re.VERBOSE) + if m: + return Sftp(username=m["username"], hostname=m["hostname"], port=int(m["port"] or "0"), path=m["path"]) class Sftp(BackendBase): From d12f0b0057b8302e2ad996e00a8774d98d386803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 19:44:40 +0100 Subject: [PATCH 04/11] update startswith --- src/borgstore/backends/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borgstore/backends/s3.py b/src/borgstore/backends/s3.py index 7ac66ea..80deee3 100644 --- a/src/borgstore/backends/s3.py +++ b/src/borgstore/backends/s3.py @@ -25,7 +25,7 @@ def get_s3_backend(url: str): (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path """ - if not url.startswith("s3:") and not url.startswith("b2:"): + if not url.startswith(("s3:", "b2:")): return None if boto3 is None: From 47899cce2f15a2738b05033350a820396c375a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 19:50:45 +0100 Subject: [PATCH 05/11] Update error message --- src/borgstore/backends/s3.py | 2 +- src/borgstore/backends/sftp.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/borgstore/backends/s3.py b/src/borgstore/backends/s3.py index 80deee3..9116357 100644 --- a/src/borgstore/backends/s3.py +++ b/src/borgstore/backends/s3.py @@ -29,7 +29,7 @@ def get_s3_backend(url: str): return None if boto3 is None: - raise DependencyMissing("The S3 backend requires dependencies. Install it with 'pip install borgstore[s3]'") + raise DependencyMissing("The S3 backend requires dependencies. Install them with 'pip install borgstore[s3]'") # (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path s3_regex = r""" diff --git a/src/borgstore/backends/sftp.py b/src/borgstore/backends/sftp.py index 1a3834f..c334b65 100644 --- a/src/borgstore/backends/sftp.py +++ b/src/borgstore/backends/sftp.py @@ -26,8 +26,8 @@ def get_sftp_backend(url): if not url.startswith("sftp://"): return None - if paramiko is not None: - raise DependencyMissing("The SFTP backend requires dependencies. Install it with 'pip install borgstore[sftp]'") + if paramiko is None: + raise DependencyMissing("The SFTP backend requires dependencies. Install them with: 'pip install borgstore[sftp]'") # sftp://username@hostname:22/path From 30cb36828231b962da4c4b411505829eef3796bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 19:59:47 +0100 Subject: [PATCH 06/11] Remove dev dependency and note in readme --- README.rst | 3 --- requirements.d/dev.txt | 1 - 2 files changed, 4 deletions(-) diff --git a/README.rst b/README.rst index 8de2229..d1d8548 100644 --- a/README.rst +++ b/README.rst @@ -221,9 +221,6 @@ Use storage on an S3-compliant cloud service: There is a known issue with some S3-compatible services (e.g., **Backblaze B2**). If you encounter problems, try using ``b2:`` instead of ``s3:`` in the URL. - .. note:: - - You need to install ``boto3`` to be able to use s3/b2 backends. Run ``pip install boto3`` in your venv. - Namespaces: directories - Values: in key-named files diff --git a/requirements.d/dev.txt b/requirements.d/dev.txt index f4acb23..8507eb3 100644 --- a/requirements.d/dev.txt +++ b/requirements.d/dev.txt @@ -3,4 +3,3 @@ tox pytest build twine -boto3 From b8a78b83e33d356b4ccb3372fe9eaa200d9d2e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 20:01:55 +0100 Subject: [PATCH 07/11] Remove // in sftp check --- src/borgstore/backends/sftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borgstore/backends/sftp.py b/src/borgstore/backends/sftp.py index c334b65..52c739d 100644 --- a/src/borgstore/backends/sftp.py +++ b/src/borgstore/backends/sftp.py @@ -23,7 +23,7 @@ def get_sftp_backend(url): """Get SFTP backend from URL.""" - if not url.startswith("sftp://"): + if not url.startswith("sftp:"): return None if paramiko is None: From ba673b1164fcf4009c5bd5255b62dcdf42e1221e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 20:06:27 +0100 Subject: [PATCH 08/11] Update rclone control flow DependencyMissing --> BackendDoesNotExist --- src/borgstore/backends/errors.py | 2 -- src/borgstore/backends/rclone.py | 19 ++++++++++++------- src/borgstore/backends/s3.py | 4 ++-- src/borgstore/backends/sftp.py | 5 ++--- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/borgstore/backends/errors.py b/src/borgstore/backends/errors.py index 291c74f..7fd811f 100644 --- a/src/borgstore/backends/errors.py +++ b/src/borgstore/backends/errors.py @@ -38,5 +38,3 @@ class ObjectNotFound(BackendError): class PermissionDenied(BackendError): """Permission denied for the requested operation.""" -class DependencyMissing(BackendError): - """Permission denied for the requested operation.""" diff --git a/src/borgstore/backends/rclone.py b/src/borgstore/backends/rclone.py index 904e3ea..a54a3f9 100644 --- a/src/borgstore/backends/rclone.py +++ b/src/borgstore/backends/rclone.py @@ -44,19 +44,24 @@ def get_rclone_backend(url): rclone:remote: rclone:remote:path """ + + if not url.startswith("rclone:"): + return None + + try: + # Check rclone is on the path + info = json.loads(subprocess.check_output([RCLONE, "rc", "--loopback", "core/version"])) + except Exception: + raise BackendDoesNotExist("rclone binary not found on the path or not working properly") + if info["decomposed"] < [1, 57, 0]: + raise BackendDoesNotExist(f"rclone version must be at least v1.57.0 - found {info['version']}") + rclone_regex = r""" rclone: (?P(.*)) """ m = re.match(rclone_regex, url, re.VERBOSE) if m: - # Check rclone is on the path - try: - info = json.loads(subprocess.check_output([RCLONE, "rc", "--loopback", "core/version"])) - except Exception: - raise BackendDoesNotExist("rclone binary not found on the path or not working properly") - if info["decomposed"] < [1, 57, 0]: - raise BackendDoesNotExist(f"rclone version must be at least v1.57.0 - found {info['version']}") return Rclone(path=m["path"]) diff --git a/src/borgstore/backends/s3.py b/src/borgstore/backends/s3.py index 9116357..ff78446 100644 --- a/src/borgstore/backends/s3.py +++ b/src/borgstore/backends/s3.py @@ -14,7 +14,7 @@ from borgstore.constants import TMP_SUFFIX from ._base import BackendBase, ItemInfo, validate_name -from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists, DependencyMissing +from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists from .errors import ObjectNotFound @@ -29,7 +29,7 @@ def get_s3_backend(url: str): return None if boto3 is None: - raise DependencyMissing("The S3 backend requires dependencies. Install them with 'pip install borgstore[s3]'") + raise BackendDoesNotExist("The S3 backend requires dependencies. Install them with 'pip install borgstore[s3]'") # (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path s3_regex = r""" diff --git a/src/borgstore/backends/sftp.py b/src/borgstore/backends/sftp.py index 52c739d..99c1ab4 100644 --- a/src/borgstore/backends/sftp.py +++ b/src/borgstore/backends/sftp.py @@ -14,8 +14,7 @@ paramiko = None from ._base import BackendBase, ItemInfo, validate_name -from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists, \ - DependencyMissing +from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists from .errors import ObjectNotFound from ..constants import TMP_SUFFIX @@ -27,7 +26,7 @@ def get_sftp_backend(url): return None if paramiko is None: - raise DependencyMissing("The SFTP backend requires dependencies. Install them with: 'pip install borgstore[sftp]'") + raise BackendDoesNotExist("The SFTP backend requires dependencies. Install them with: 'pip install borgstore[sftp]'") # sftp://username@hostname:22/path From fb3b74e4997dc8aab0efa03d36f4de91561dccb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 20:10:29 +0100 Subject: [PATCH 09/11] Remove line --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index d1d8548..51b456d 100644 --- a/README.rst +++ b/README.rst @@ -221,7 +221,6 @@ Use storage on an S3-compliant cloud service: There is a known issue with some S3-compatible services (e.g., **Backblaze B2**). If you encounter problems, try using ``b2:`` instead of ``s3:`` in the URL. - - Namespaces: directories - Values: in key-named files From 35c4ce3c2f1ceb5f7fd9d02fd050bda986909ee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 20:11:29 +0100 Subject: [PATCH 10/11] Remove line --- src/borgstore/backends/errors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/borgstore/backends/errors.py b/src/borgstore/backends/errors.py index 7fd811f..c8b8356 100644 --- a/src/borgstore/backends/errors.py +++ b/src/borgstore/backends/errors.py @@ -37,4 +37,3 @@ class ObjectNotFound(BackendError): class PermissionDenied(BackendError): """Permission denied for the requested operation.""" - From 38f84d5687fe021b0281225295ed0a939c913266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Fr=C3=B6hlich?= Date: Sun, 25 Jan 2026 22:05:04 +0100 Subject: [PATCH 11/11] Add colon --- src/borgstore/backends/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borgstore/backends/s3.py b/src/borgstore/backends/s3.py index ff78446..f29982d 100644 --- a/src/borgstore/backends/s3.py +++ b/src/borgstore/backends/s3.py @@ -29,7 +29,7 @@ def get_s3_backend(url: str): return None if boto3 is None: - raise BackendDoesNotExist("The S3 backend requires dependencies. Install them with 'pip install borgstore[s3]'") + raise BackendDoesNotExist("The S3 backend requires dependencies. Install them with: 'pip install borgstore[s3]'") # (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path s3_regex = r"""