From d01db1785819b72e947548a3d2aeb4e25b40d998 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 28 Sep 2026 21:27:01 +0200 Subject: [PATCH] remote: use deques for the RPC call queues, fixes #10450 RemoteRepository.call_many took the next call from the front of a list (calls.pop(0), self.preload_ids.pop(0)), which moves all the remaining entries each time. For a single item with millions of chunks (preload + get_many over all of its chunk ids), that made extract from a remote repository quadratic in the chunk count of the item. Use collections.deque with popleft() for calls, waiting_for and preload_ids. Backport of #10454. Co-Authored-By: Claude Opus 5.5 --- src/borg/remote.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index ec9d490c4..156476c19 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -13,6 +13,7 @@ import tempfile import textwrap import time import traceback +from collections import deque from subprocess import Popen, PIPE, TimeoutExpired from . import __version__ @@ -538,7 +539,7 @@ class RemoteRepository: def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock=True, append_only=False, make_parent_dirs=False, args=None): self.location = self._location = location - self.preload_ids = [] + self.preload_ids = deque() self.msgid = 0 self.rx_bytes = 0 self.tx_bytes = 0 @@ -781,8 +782,8 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+. else: raise self.RPCError(unpacked) - calls = list(calls) - waiting_for = [] + calls = deque(calls) + waiting_for = deque() maximum_to_send = 0 if wait else self.upload_buffer_size_limit send_buffer() # Try to send data, as some cases (async_response) will never try to send data otherwise. while wait or calls: @@ -795,7 +796,7 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+. while waiting_for: try: unpacked = self.responses.pop(waiting_for[0]) - waiting_for.pop(0) + waiting_for.popleft() if b'exception_class' in unpacked: handle_error(unpacked) else: @@ -870,9 +871,9 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+. if is_preloaded: assert cmd == 'get', "is_preload is only supported for 'get'" if calls[0]['id'] in self.chunkid_to_msgids: - waiting_for.append(pop_preload_msgid(calls.pop(0)['id'])) + waiting_for.append(pop_preload_msgid(calls.popleft()['id'])) else: - args = calls.pop(0) + args = calls.popleft() if cmd == 'get' and args['id'] in self.chunkid_to_msgids: waiting_for.append(pop_preload_msgid(args['id'])) else: @@ -883,7 +884,7 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+. else: self.to_send.push_back(msgpack.packb((1, self.msgid, cmd, self.named_to_positional(cmd, args)))) if not self.to_send and self.preload_ids: - chunk_id = self.preload_ids.pop(0) + chunk_id = self.preload_ids.popleft() args = {'id': chunk_id} self.msgid += 1 self.chunkid_to_msgids.setdefault(chunk_id, []).append(self.msgid) @@ -1026,7 +1027,7 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+. return resp def preload(self, ids): - self.preload_ids += ids + self.preload_ids.extend(ids) def handle_remote_line(line):