From 688afbf6fd3b037b6bda10b36b1b519150592f76 Mon Sep 17 00:00:00 2001 From: hiijoshi Date: Tue, 24 Mar 2026 19:19:21 +0530 Subject: [PATCH] fix: chunk store_load over RPC to avoid msgpack BufferFull Large repositories can have a chunks index exceeding the msgpack unpacker buffer size, causing BufferFull errors over SSH/RPC. Instead of loading the entire value in one RPC call, add two new server-side methods (store_get_size, store_load_chunk) and override store_load in RemoteRepository to fetch data in MAX_DATA_SIZE pieces, reassembling on the client side. Fixes #8440 --- src/borg/remote.py | 13 +++++++++++-- src/borg/repository.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index 5fdafb4223..d83d7c113f 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -161,6 +161,8 @@ class RepositoryServer: # pragma: no cover "put_manifest", "store_list", "store_load", + "store_load_chunk", + "store_get_size", "store_store", "store_delete", "store_move", @@ -1049,9 +1051,16 @@ def put_manifest(self, data): def store_list(self, name, *, deleted=False): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version("2.0.0b8")) def store_load(self, name): - """actual remoting is done via self.call in the @api decorator""" + # chunked fetch to avoid msgpack BufferFull on large repositories + total_size = self.call("store_get_size", {"name": name}) + data = bytearray() + offset = 0 + while offset < total_size: + chunk = self.call("store_load_chunk", {"name": name, "offset": offset, "size": MAX_DATA_SIZE}) + data += chunk + offset += len(chunk) + return bytes(data) @api(since=parse_version("2.0.0b8")) def store_store(self, name, value): diff --git a/src/borg/repository.py b/src/borg/repository.py index a3f8aa8cc2..feefac9feb 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -560,6 +560,16 @@ def store_store(self, name, value): self._lock_refresh() return self.store.store(name, value) + def store_get_size(self, name): + self._lock_refresh() + data = self.store.load(name) + return len(data) + + def store_load_chunk(self, name, offset, size): + self._lock_refresh() + data = self.store.load(name) + return data[offset : offset + size] + def store_delete(self, name, *, deleted=False): self._lock_refresh() return self.store.delete(name, deleted=deleted)