bench: adapters that run Mem0, Hindsight and Letta as their own benchmarks do (memory_<name>.py, pinned in requirements-<name>.txt), a pilot limit for LongMemEval runs, and Hindsight's recall sized to the 25,000 characters of MEMORY.md

tests/bench/letta-server.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# TEST HARNESS ONLY: starts the Letta server that tests/bench/memory_letta.py talks to (letta 0.16.8, Apache-2.0,
+# https://github.com/letta-ai/letta), the way Letta's own image does it (letta/server/startup.sh and init.sql at tag
+# 0.16.8): a PostgreSQL with pgvector, the release's `alembic upgrade head`, then `letta server`. Letta 0.16.8 needs
+# Postgres (its ORM imports asyncpg; SQLite is no longer wired in), so here it is a local cluster in $DATA run as
+# the calling user, instead of the one bundled in Letta's image. Redis is optional in Letta and not started.
+#
+#   bash tests/bench/letta-server.sh   # http://127.0.0.1:8283; Postgres on 127.0.0.1:5432; data in /work/letta
+#
+# Once, as root:  apt-get install postgresql-16 postgresql-16-pgvector && python3 -m venv /opt/mem-letta &&
+#                 /opt/mem-letta/bin/pip install -r tests/bench/requirements-letta.txt
+# No model key is needed here: memory_letta.py points every agent at the metering proxy, which adds the key.
+set -euo pipefail
+
+VENV=/opt/mem-letta
+DATA=${LETTA_BENCH_DIR:-/work/letta}
+PORT=${LETTA_PORT:-8283}
+PG_PORT=${LETTA_PG_PORT:-5432}
+PG=/usr/lib/postgresql/16/bin
+SRC=$VENV/src/letta-0.16.8
+
+# The migrations ship in the release's sdist, not in the wheel.
+if [ ! -f "$SRC/alembic.ini" ]; then
+    mkdir -p "$VENV/src"
+    "$VENV/bin/pip" download -q --no-deps --no-binary :all: letta==0.16.8 -d "$VENV/src"
+    tar xzf "$VENV/src/letta-0.16.8.tar.gz" -C "$VENV/src" letta-0.16.8/alembic.ini letta-0.16.8/alembic
+fi
+
+mkdir -p "$DATA"
+if [ ! -s "$DATA/pg/PG_VERSION" ]; then
+    "$PG/initdb" -D "$DATA/pg" -U letta --auth=trust >/dev/null
+fi
+if ! "$PG/pg_ctl" -D "$DATA/pg" status >/dev/null 2>&1; then
+    "$PG/pg_ctl" -D "$DATA/pg" -l "$DATA/postgres.log" -w \
+        -o "-p $PG_PORT -k $DATA -c listen_addresses=127.0.0.1 -c max_connections=200" start >/dev/null
+fi
+psql() { "$PG/psql" -h 127.0.0.1 -p "$PG_PORT" -U letta -v ON_ERROR_STOP=1 -qtA "$@"; }
+if [ -z "$(psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'letta'")" ]; then
+    psql -d postgres -c "CREATE DATABASE letta"
+    # init.sql
+    psql -d letta -c "CREATE SCHEMA letta AUTHORIZATION letta" \
+        -c "ALTER DATABASE letta SET search_path TO letta" \
+        -c "CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA letta" \
+        -c "DROP SCHEMA IF EXISTS public CASCADE"
+fi
+
+export LETTA_PG_URI="postgresql://letta:[email protected]:$PG_PORT/letta"
+export LETTA_DIR="$DATA"
+(cd "$SRC" && "$VENV/bin/alembic" upgrade head)
+exec "$VENV/bin/letta" server --host 127.0.0.1 --port "$PORT"

tests/bench/longmemeval.py

@@ -200,6 +200,13 @@ async def external_memory(dot_id: str, variant: str, question: dict[str, Any]) -
         "model": os.environ.get("BENCH_MODEL", "z-ai/glm-5.3-flash"),
         "embedding_model": "openai/text-embedding-3-small",
         "letta_url": LETTA_URL,
+        "question_id": question["question_id"],
+        # Mem0 per session rather than per pair (its own runner's choice), at about a fifth of the calls.
+        "add_per": "session",
+        # What Hindsight recalls, kept within the 25,000 characters a Dot carries of MEMORY.md (its benchmark asks
+        # for 32768 + 16384 tokens): its own ranking chooses, not a cut.
+        "recall_max_tokens": 4000,
+        "chunk_max_tokens": 2000,
     }
     process = await asyncio.create_subprocess_exec(
         f"/opt/mem-{variant}/bin/python",
@@ -311,6 +318,8 @@ async def run(args: argparse.Namespace) -> None:
         questions = sorted((q for q in every if q["question_type"] == args.kind), key=lambda q: q["question_id"])
     else:
         questions = subset(every, args.questions)
+    if args.limit:
+        questions = questions[: args.limit]
     done = {row["question_id"] for row in read_jsonl(out / "hypotheses.jsonl")}
     todo = [q for q in questions if q["question_id"] not in done]
     print(f"{len(questions)} questions, {len(done)} answered, {len(todo)} to go, {args.n} at once", flush=True)
@@ -425,6 +434,7 @@ def main() -> None:
     )
     commands.choices["run"].add_argument("--questions", type=int, default=116)
     commands.choices["run"].add_argument("--kind", help="every question of this question_type instead of the subset")
+    commands.choices["run"].add_argument("--limit", type=int, help="only the first N of the questions (a pilot)")
     commands.choices["run"].add_argument("-n", type=int, default=4)
     args = parser.parse_args()
     if args.command == "run":

tests/bench/memory_hindsight.py

@@ -0,0 +1,259 @@
+"""TEST HARNESS ONLY: Hindsight writes MEMORY.md for one LongMemEval question, so the bench can weigh it against our
+own memory methods. Nothing here ships.
+
+Hindsight, MIT, https://github.com/vectorize-io/hindsight (hindsight-api / hindsight-client 0.10.3, pinned in
+requirements-hindsight.txt), run the way its own LongMemEval runs run it. Hindsight's benchmarks live in AMB,
+https://github.com/vectorize-io/agent-memory-benchmark (MIT; hindsight-system-evals/README.md: "AMB is now the only
+copy"), main @ 8ea4c837cb00903453515929df4f80bcc793530c, provider `hindsight-http`:
+
+  src/memory_bench/dataset/longmemeval.py  load_documents: one document a session, content = json.dumps(the
+      session's turns), context "Session <id> - you are the assistant in this conversation - happened on
+      <YYYY-mm-dd HH:MM:SS> UTC.", timestamp = the session's date (UTC ISO); load_queries: query_timestamp = the
+      question's date (UTC ISO). One bank per question (isolation_unit = "question").
+  src/memory_bench/memory/hindsight.py     _bank_kwargs (enable_observations=False), async_ingest (retain_batch
+      of 20 items, retain_async=True, then _await_bank_ingest: wait until no operation is pending or processing,
+      never going on past a failed one), _doc_to_items, _recall_kwargs (budget "high", max_tokens 32768,
+      include_chunks with max_chunk_tokens 16384, include_entities False, query cut at 1900 characters,
+      query_timestamp), _deduplicate_results and _format_result(s).
+  src/memory_bench/modes/rag.py            the context the answer model is given: "## Memory <n>\\n<result>" joined
+      by blank lines. That text is MEMORY.md.
+
+ADAPTED: one Hindsight server per question, started here as its README's bare-metal way (`hindsight-api`, embedded
+Postgres "pg0" under a name of its own, Hindsight's default local embeddings BAAI/bge-small-en-v1.5 and reranker
+cross-encoder/ms-marco-MiniLM-L-6-v2), and dropped afterwards; AMB runs one long-lived server. Only the LLM is
+configured: Hindsight's "openrouter" provider (OpenAI-compatible; it adds OpenRouter's require_parameters when it
+asks for JSON) with its base URL set to the bench's metering proxy, which forwards to OpenRouter. Session ids are
+"<question id>_<index>" because the input carries no session ids. An operation that still fails after Hindsight's
+own retries (3 a task, each LLM call retried with backoff) ends the run; AMB re-queues it up to 5 more times, 180
+seconds apart, to ride out outages of a shared server. AMB answers with an LLM over the raw recall JSON
+(dataset.build_rag_prompt); here the Dot answers with the rendered text in MEMORY.md.
+
+"observations": true in the input turns on Hindsight's own default instead of AMB's setting: observation
+consolidation after each retain, waited for like the retains (its operations are in the same queue).
+
+    /opt/mem-hindsight/bin/python memory_hindsight.py < input.json > output.json
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import signal
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+import httpx
+from hindsight_client import Hindsight
+from pg0 import Pg0
+
+HINDSIGHT_API = str(Path(sys.executable).parent / "hindsight-api")
+BATCH_SIZE = 20  # AMB async_ingest
+STARTUP_TIMEOUT_S = 900  # the first start downloads the embedding and reranker models
+INGEST_TIMEOUT_S = 8 * 3600  # AMB_INGEST_MAX_WAIT_S
+
+
+def parse_date(text: str) -> datetime:
+    """LongMemEval's "2023/05/20 (Sat) 02:21" as UTC, as AMB's LongMemEvalDataset._parse_date reads it."""
+    return datetime.strptime(text.split("(")[0].strip() + " " + text.split(")")[-1].strip(), "%Y/%m/%d %H:%M").replace(
+        tzinfo=timezone.utc
+    )
+
+
+def items_for(sessions: list[dict], question_id: str) -> list[dict]:
+    """AMB LongMemEvalDataset.load_documents + _HindsightBase._doc_to_items, one item a session."""
+    items = []
+    for index, session in enumerate(sessions):
+        doc_id = f"{question_id}_{index}"
+        date = parse_date(session["date"])
+        turns = [{"role": t["role"], "content": t["content"]} for t in session["turns"]]
+        items.append(
+            {
+                "content": json.dumps(turns).replace("\x00", ""),
+                "document_id": doc_id,
+                "metadata": {"doc_id": doc_id},
+                "timestamp": date.isoformat(),
+                "context": f"Session {doc_id} - you are the assistant in this conversation - happened on "
+                f"{date.strftime('%Y-%m-%d %H:%M:%S')} UTC.",
+            }
+        )
+    return items
+
+
+def format_result(result: dict, chunks: dict, seen_chunks: set) -> str:
+    """AMB _format_result."""
+    lines = [f"**[{result['type']}]** {result['text']}" if result.get("type") else result["text"]]
+    meta = []
+    start, end = result.get("occurred_start"), result.get("occurred_end")
+    if start and end and start != end:
+        meta.append(f"occurred: {start} – {end}")
+    elif start:
+        meta.append(f"occurred: {start}")
+    if result.get("mentioned_at"):
+        meta.append(f"mentioned: {result['mentioned_at']}")
+    if result.get("chunk_id"):
+        meta.append(f"chunk: {result['chunk_id']}")
+    if meta:
+        lines.append("_" + " · ".join(meta) + "_")
+    chunk_id = result.get("chunk_id")
+    if chunk_id and chunk_id in chunks and chunk_id not in seen_chunks:
+        lines.append(f"> {chunks[chunk_id]['text']}")
+        seen_chunks.add(chunk_id)
+    return "\n".join(lines)
+
+
+def render(recalled: dict) -> str:
+    """AMB _deduplicate_results + _format_results, joined as modes/rag.py joins them for the answer model."""
+    seen_ids: set = set()
+    results = [r for r in recalled.get("results") or [] if not (r["id"] in seen_ids or seen_ids.add(r["id"]))]
+    chunks = {k: v or {} for k, v in (recalled.get("chunks") or {}).items()}
+    seen_chunks: set = set()
+    return "\n\n".join(f"## Memory {i + 1}\n{format_result(r, chunks, seen_chunks)}" for i, r in enumerate(results))
+
+
+def free_port() -> int:
+    with socket.socket() as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+def operations(http: httpx.Client, bank_id: str, status: str | None = None, limit: int = 1, offset: int = 0) -> dict:
+    params = {"limit": limit, "offset": offset, **({"status": status} if status else {})}
+    response = http.get(f"/v1/default/banks/{bank_id}/operations", params=params, timeout=60)
+    response.raise_for_status()
+    return response.json()
+
+
+def await_bank(http: httpx.Client, bank_id: str) -> dict:
+    """AMB _await_bank_ingest: until nothing is pending or processing, refusing to go on past a failed operation."""
+    start = time.monotonic()
+    while time.monotonic() - start < INGEST_TIMEOUT_S:
+        total = operations(http, bank_id)["total"]
+        completed = operations(http, bank_id, "completed")["total"]
+        failed = operations(http, bank_id, "failed")
+        if failed["total"]:
+            op = failed["operations"][0]
+            raise RuntimeError(
+                f"{failed['total']} operation(s) failed after Hindsight's own retries, e.g. {op['task_type']}: "
+                f"{op.get('error_message')}"
+            )
+        if total - completed == 0:
+            kinds: dict[str, int] = {}
+            offset = 0
+            while offset < total:
+                page = operations(http, bank_id, "completed", limit=100, offset=offset)
+                for op in page["operations"]:
+                    kind = op.get("task_type") or op.get("operation_type") or "?"
+                    kinds[kind] = kinds.get(kind, 0) + 1
+                offset += len(page["operations"]) or total
+            return {"seconds": round(time.monotonic() - start), "operations": kinds}
+        time.sleep(5)
+    raise RuntimeError(f"bank {bank_id}: operations still running after {INGEST_TIMEOUT_S}s")
+
+
+def main() -> None:
+    # A bench that stops this process still gets the server stopped and its database dropped (the finally below).
+    signal.signal(signal.SIGTERM, lambda *_: sys.exit(143))
+    request = json.load(sys.stdin)
+    question_id = request.get("question_id") or "q"
+    bank_id = f"longmemeval-{question_id}"
+    observations = bool(request.get("observations"))
+    port = free_port()
+    instance = f"lme-{uuid.uuid4().hex[:12]}"
+    workdir = Path(tempfile.mkdtemp(prefix="hindsight-"))
+    log_path = workdir / "server.log"
+    env = {
+        **os.environ,
+        "HINDSIGHT_API_DATABASE_URL": f"pg0://{instance}",
+        "HINDSIGHT_API_LLM_PROVIDER": "openrouter",
+        "HINDSIGHT_API_LLM_BASE_URL": request["base_url"],
+        "HINDSIGHT_API_LLM_API_KEY": request["api_key"],
+        "HINDSIGHT_API_LLM_MODEL": request["model"],
+    }
+    started = time.monotonic()
+    with log_path.open("wb") as log:
+        server = subprocess.Popen(
+            [HINDSIGHT_API, "--host", "127.0.0.1", "--port", str(port)],
+            env=env,
+            stdout=log,
+            stderr=subprocess.STDOUT,
+            cwd=workdir,
+            start_new_session=True,
+        )
+    try:
+        base = f"http://127.0.0.1:{port}"
+        with httpx.Client(base_url=base) as http:
+            while True:
+                if server.poll() is not None:
+                    raise RuntimeError(f"hindsight-api exited with {server.returncode} while starting")
+                if time.monotonic() - started > STARTUP_TIMEOUT_S:
+                    raise RuntimeError(f"hindsight-api not ready after {STARTUP_TIMEOUT_S}s")
+                try:
+                    if http.get("/health", timeout=5).status_code == 200:
+                        break
+                except httpx.HTTPError:
+                    pass
+                time.sleep(1)
+            startup_s = round(time.monotonic() - started)
+
+            client = Hindsight(base_url=base, timeout=600)
+            client.create_bank(bank_id=bank_id, name=f"Benchmark Bank ({bank_id})", enable_observations=observations)
+            items = items_for(request["sessions"], question_id)
+            for i in range(0, len(items), BATCH_SIZE):
+                client.retain_batch(bank_id=bank_id, items=items[i : i + BATCH_SIZE], retain_async=True)
+            waited = await_bank(http, bank_id)
+
+            # AMB's 32768 and 16384 unless the bench asks for less (ADAPTED: a Dot keeps 25,000 characters of MEMORY.md).
+            recall_tokens = int(request.get("recall_max_tokens") or 32768)
+            chunk_tokens = int(request.get("chunk_max_tokens") or 16384)
+            recall = {
+                "query": request["question"][:1900],
+                "budget": "high",
+                "max_tokens": recall_tokens,
+                "include": {"entities": None, "chunks": {"max_tokens": chunk_tokens}},
+                "query_timestamp": parse_date(request["question_date"]).isoformat(),
+            }
+            response = http.post(f"/v1/default/banks/{bank_id}/memories/recall", json=recall, timeout=600)
+            response.raise_for_status()
+            recalled = response.json()
+            stats = http.get(f"/v1/default/banks/{bank_id}/stats", timeout=60)
+            client.close()
+        memory_md = render(recalled)
+        kinds: dict[str, int] = {}
+        for r in recalled.get("results") or []:
+            kinds[r.get("type") or "?"] = kinds.get(r.get("type") or "?", 0) + 1
+        notes = (
+            f"Hindsight 0.10.3, bank {bank_id}, observations {'on' if observations else 'off (AMB)'}: "
+            f"{len(items)} sessions ({sum(len(i['content']) for i in items)} characters) retained; "
+            f"server ready in {startup_s}s; operations done in {waited['seconds']}s {waited['operations']}; "
+            f"bank stats {stats.json() if stats.status_code == 200 else stats.status_code}; "
+            f"recall (budget high, max_tokens {recall_tokens}, chunks {chunk_tokens}, query_timestamp) returned {kinds} and "
+            f"{len(recalled.get('chunks') or {})} chunks, {len(memory_md)} characters"
+        )
+        json.dump({"memory_md": memory_md, "notes": notes}, sys.stdout, ensure_ascii=False)
+    except Exception as error:
+        tail = log_path.read_text(encoding="utf-8", errors="replace")[-4000:] if log_path.exists() else ""
+        print(f"memory_hindsight: {type(error).__name__}: {error}\n--- hindsight-api log (end) ---\n{tail}", file=sys.stderr)
+        sys.exit(1)
+    finally:
+        os.killpg(server.pid, signal.SIGTERM)
+        try:
+            server.wait(timeout=60)
+        except subprocess.TimeoutExpired:
+            os.killpg(server.pid, signal.SIGKILL)
+            server.wait()
+        try:
+            Pg0(name=instance).drop()
+        except Exception as error:
+            print(f"memory_hindsight: dropping pg0 instance {instance}: {error}", file=sys.stderr)
+        shutil.rmtree(workdir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()

tests/bench/memory_letta.py

@@ -0,0 +1,193 @@
+"""TEST HARNESS ONLY: MEMORY.md written by Letta's sleep-time agent, for the LongMemEval bench. Nothing here ships.
+
+Letta, Apache-2.0, https://github.com/letta-ai/letta, release 0.16.8 (tag 0.16.8, commit 1131535716e8, the last
+release of the server; the PyPI name `letta` now ships Letta Code). Letta keeps memory blocks ("human", "persona") in
+the agent's prompt; with enable_sleeptime=True a background sleep-time agent (agent_type sleeptime_agent, its own
+prompt prompts/system_prompts/sleeptime_v2.py and tools memory_insert / memory_replace / memory_rethink /
+memory_finish_edits) rewrites them from the conversation, every `sleeptime_agent_frequency` turns (5, as
+server.create_sleeptime_agent_async sets it); the main agent then has no memory tools of its own.
+
+Per question this creates one agent with enable_sleeptime=True and Letta's default blocks (schemas/block.py
+DEFAULT_BLOCKS: an empty "human" and "persona"), records the whole history into it oldest first, waits for every
+sleep-time run, renders the blocks the main agent sees as MEMORY.md, and deletes the agents and their blocks.
+Letta's prompts, tools, tool rules, frequency and compaction are left as they are; only the model is configured.
+
+    /opt/mem-letta/bin/python memory_letta.py < input.json > output.json      (server: tests/bench/letta-server.sh)
+
+ADAPTED:
+- The history is recorded, not replayed: each user message(s) + assistant reply of a session is one turn sent to
+  POST /v1/agents/{id}/messages/capture, Letta's own endpoint for persisting a turn that was answered elsewhere; it
+  stores the dataset's assistant reply as the agent's and runs the sleep-time group exactly as after a real turn.
+  The main agent therefore makes no model call. Consecutive assistant messages are one reply.
+- Dates: capture stamps messages with the current time and the sleep-time transcript (groups/helpers.py
+  stringify_message) carries no time, so each user message is prefixed with its session's date, "[2023/05/20 (Sat)
+  02:21] ...".
+- Each sleep-time run is waited for before the next turn is recorded (a person's next message comes after it ends),
+  and the last turn is recorded with the frequency set to 1, so the turns after the last multiple of 5 are processed
+  too (each run gets every message since the previous run).
+- Model: the LLMConfig Letta's OpenRouterProvider.list_llm_models_async builds (model_endpoint_type "openrouter",
+  context_window from the endpoint's /models, 128000 if absent; max_tokens 16384), with model_endpoint the
+  per-question base_url, so the sleep-time agent (which copies the main agent's config) also goes through it. No
+  handle is set: compaction then summarizes with this same config instead of resolving a server-wide provider.
+  The key is not per agent in Letta (env or a BYOK provider); the metering proxy adds it, so `api_key` is unused.
+- Embeddings (required for a sleep-time agent to be created; none of its tools embed): `embedding_model` at the same
+  base_url, 1536 dimensions.
+- MEMORY.md: every block of the main agent with text, as "## <label>", Letta's description of the block, then its
+  value (Letta shows the same three in the prompt, as XML).
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+import uuid
+
+import httpx
+from letta_client import Letta
+
+RUN_TIMEOUT_S = 3600
+FINAL_FREQUENCY = 1
+
+
+def turns(sessions: list[dict]) -> list[tuple[list[str], str]]:
+    """(user messages, assistant reply) per turn, oldest first; a turn never spans two sessions."""
+    out: list[tuple[list[str], str]] = []
+    for session in sessions:
+        users: list[str] = []
+        replies: list[str] = []
+        for turn in session["turns"]:
+            if turn["role"] == "user":
+                if replies:
+                    out.append((users, "\n\n".join(replies)))
+                    users, replies = [], []
+                users.append(f"[{session['date']}] {turn['content']}")
+            else:
+                replies.append(turn["content"])
+        if users or replies:
+            out.append((users, "\n\n".join(replies)))
+    return out
+
+
+def context_window(http: httpx.Client, base_url: str, model: str) -> int:
+    response = http.get(base_url.rstrip("/") + "/models", params={"supported_parameters": "tools"})
+    response.raise_for_status()
+    for entry in response.json().get("data", []):
+        if entry.get("id") == model and entry.get("context_length"):
+            return int(entry["context_length"])
+    return 128000
+
+
+def wait_for_run(client: Letta, run_id: str) -> dict:
+    deadline = time.monotonic() + RUN_TIMEOUT_S
+    while True:
+        run = client.runs.retrieve(run_id)
+        if run.status in ("completed", "failed", "cancelled"):
+            return run.model_dump()
+        if time.monotonic() > deadline:
+            raise RuntimeError(f"sleep-time run {run_id} still {run.status} after {RUN_TIMEOUT_S}s")
+        time.sleep(1)
+
+
+def render(blocks: list) -> str:
+    parts = []
+    for block in sorted(blocks, key=lambda b: b.label):
+        if (block.value or "").strip():
+            parts += [f"## {block.label}", (block.description or "").strip(), block.value.strip()]
+    return "\n\n".join(part for part in parts if part)
+
+
+def main() -> None:
+    job = json.load(sys.stdin)
+    base_url, model = job["base_url"], job["model"]
+    client = Letta(base_url=job["letta_url"], timeout=600)
+    http = httpx.Client(timeout=600)
+    llm_config = {
+        "model": model,
+        "model_endpoint_type": "openrouter",
+        "model_endpoint": base_url,
+        "provider_name": "openrouter",
+        "provider_category": "base",
+        "context_window": context_window(http, base_url, model),
+        "max_tokens": 16384,
+    }
+    embedding_config = {
+        "embedding_endpoint_type": "openai",
+        "embedding_endpoint": base_url,
+        "embedding_model": job["embedding_model"],
+        "embedding_dim": 1536,
+    }
+    agent = client.agents.create(
+        name=f"longmemeval-{uuid.uuid4().hex[:12]}",
+        agent_type="letta_v1_agent",
+        enable_sleeptime=True,
+        memory_blocks=[{"label": "human", "value": ""}, {"label": "persona", "value": ""}],
+        llm_config=llm_config,
+        embedding_config=embedding_config,
+    )
+    block_ids: set[str] = set()
+    try:
+        group = agent.multi_agent_group
+        if group is None or not group.agent_ids:
+            raise RuntimeError("Letta created no sleep-time agent")
+        sleeptime_id = group.agent_ids[0]
+        block_ids |= {b.id for b in client.agents.blocks.list(agent.id)}
+        block_ids |= {b.id for b in client.agents.blocks.list(sleeptime_id)}
+
+        history = turns(job["sessions"])
+        runs: list[dict] = []
+        for i, (users, reply) in enumerate(history):
+            if i == len(history) - 1:
+                http.patch(
+                    f"{job['letta_url']}/v1/groups/{group.id}",
+                    json={"manager_config": {"manager_type": "sleeptime", "sleeptime_agent_frequency": FINAL_FREQUENCY}},
+                ).raise_for_status()
+            response = http.post(
+                f"{job['letta_url']}/v1/agents/{agent.id}/messages/capture",
+                json={
+                    "provider": "openrouter",
+                    "model": model,
+                    "request_messages": [{"role": "user", "content": text} for text in users],
+                    "response_dict": {"content": reply},
+                },
+            )
+            response.raise_for_status()
+            for run_id in response.json().get("run_ids") or []:
+                runs.append(wait_for_run(client, run_id))
+
+        group_state = http.get(f"{job['letta_url']}/v1/groups/{group.id}").json()
+        newest = http.get(f"{job['letta_url']}/v1/agents/{agent.id}/messages", params={"limit": 1, "order": "desc"}).json()
+        if not newest or group_state.get("last_processed_message_id") != newest[0]["id"]:
+            raise RuntimeError("the sleep-time agent was not handed the last message")
+        failed = [r for r in runs if r["status"] != "completed"]
+        if runs and len(failed) == len(runs):
+            raise RuntimeError(f"every sleep-time run failed: {failed[0].get('metadata')}")
+
+        blocks = list(client.agents.blocks.list(agent.id))
+        steps = sum(((r.get("metadata") or {}).get("result") or {}).get("usage", {}).get("step_count", 0) for r in runs)
+        notes = {
+            "turns_recorded": len(history),
+            "user_messages": sum(len(u) for u, _ in history),
+            "assistant_replies": sum(1 for _, r in history if r),
+            "sleeptime_runs": len(runs),
+            "sleeptime_runs_failed": len(failed),
+            "sleeptime_steps": steps,
+            "block_chars": {b.label: len(b.value or "") for b in blocks},
+            "context_window": llm_config["context_window"],
+        }
+        json.dump({"memory_md": render(blocks), "notes": json.dumps(notes)}, sys.stdout, ensure_ascii=False)
+    finally:
+        client.agents.delete(agent.id)  # also deletes its sleep-time agent and their group
+        for block_id in block_ids:
+            try:
+                client.blocks.delete(block_id)
+            except Exception:  # already gone with the agent
+                pass
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as error:
+        print(f"memory_letta: {type(error).__name__}: {error}", file=sys.stderr)
+        sys.exit(1)

tests/bench/memory_mem0.py

@@ -0,0 +1,150 @@
+"""TEST HARNESS ONLY: Mem0 open source (https://github.com/mem0ai/mem0, Apache-2.0, `mem0ai` 2.2.1, the venv
+/opt/mem-mem0 from requirements-mem0.txt) writing MEMORY.md for one LongMemEval question, so the bench can weigh it
+against our own memory methods. Nothing here ships.
+
+    /opt/mem-mem0/bin/python tests/bench/memory_mem0.py < input.json > output.json
+
+input:  {"sessions": [{"date": "2023/05/20 (Sat) 02:21", "turns": [{"role", "content"}]}] oldest first, "question",
+         "question_date", "base_url" (OpenAI-compatible, ending in /api/v1), "api_key", "model", "embedding_model",
+         optional "top_k" (50) and "add_per" ("pair" or "session")}
+output: {"memory_md", "notes"}; on failure the reason goes to stderr and the exit status is 1.
+
+Used as Mem0 is, with its own prompts and its v3 pipeline (one additive extraction call per add(), hybrid search:
+semantic, BM25 by fastembed, entity boost by spaCy), in a fresh local Qdrant and history db per question, telemetry
+off. Configured: the LLM and the embedder (OpenAI-compatible, through `base_url`); everything else is Mem0's default.
+The history is fed and searched the way Mem0's own LongMemEval runner does it (github.com/mem0ai/memory-benchmarks
+@ 4b61c5d, benchmarks/longmemeval/run.py): one add() per user/assistant pair, oldest session first, pairs with an
+empty message skipped; search() with the question as it is; the top k shown oldest first, grouped under their date
+(prompts.get_answer_generation_prompt). That runner uses k = 200 and also reports k = 50; 50 is the default here
+because 200 memories pass the 25,000 characters of MEMORY.md the Dot is given.
+
+ADAPTED: the session's date. Mem0's platform takes it as add(timestamp=...), which the open-source add() refuses, and
+the open-source pipeline gives its extraction prompt today's date as "Observation Date", the only date it resolves
+"last week" against. So each add() gets, as the platform's timestamp would set them, the session's date as the
+memory's created_at (through metadata, which Mem0 stores as given) and as the prompt's Observation Date (the
+`timestamp` argument of Mem0's own generate_additive_extraction_prompt, which the open-source add() leaves unset).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import sys
+import tempfile
+from datetime import datetime
+from importlib.metadata import version
+
+TOP_K = 50
+
+
+def session_time(date: str) -> datetime:
+    """LongMemEval's "2023/05/20 (Sat) 02:21"."""
+    day, _, clock = date.split(" ")
+    return datetime.strptime(f"{day} {clock}", "%Y/%m/%d %H:%M")
+
+
+def chunks(turns: list[dict], add_per: str) -> list[list[dict]]:
+    messages = [{"role": t["role"], "content": t["content"]} for t in turns]
+    if add_per == "session":
+        return [messages] if messages else []
+    pairs = [messages[i : i + 2] for i in range(0, len(messages), 2)]
+    return [p for p in pairs if all(m["content"].strip() for m in p)]
+
+
+def render(results: list[dict], top_k: int) -> str:
+    """As Mem0's LongMemEval answerer is shown them: oldest first, under a line per date."""
+    lines = [f"Memories Mem0 retrieved from past conversations for the current question ({len(results)} of at most {top_k}, oldest first):"]
+    last = None
+    for result in sorted(results, key=lambda r: r.get("created_at") or ""):
+        created = result.get("created_at")
+        if created:
+            day = datetime.fromisoformat(created).strftime("%A, %B %d, %Y")
+            if day != last:
+                lines.append(f"\n--- {day} ---")
+                last = day
+        lines.append(f"- {result['memory']}")
+    return "\n".join(lines)
+
+
+def run(job: dict, store: str) -> dict:
+    os.environ["MEM0_TELEMETRY"] = "False"
+    os.environ["MEM0_DIR"] = store
+    os.environ.pop("OPENROUTER_API_KEY", None)  # Mem0's OpenAI LLM would go to OpenRouter directly with it
+    os.environ.setdefault("FASTEMBED_CACHE_PATH", os.path.join(sys.prefix, "fastembed_cache"))
+    import mem0.memory.main as mem0_main
+    from mem0 import Memory
+
+    observation = {"date": None}
+    extraction_prompt = mem0_main.generate_additive_extraction_prompt
+    mem0_main.generate_additive_extraction_prompt = lambda *a, **k: extraction_prompt(*a, **{"timestamp": observation["date"], **k})
+
+    llm_calls: list[str | None] = []
+    memory = Memory.from_config(
+        {
+            "llm": {
+                "provider": "openai",
+                "config": {
+                    "model": job["model"],
+                    "api_key": job["api_key"],
+                    "openai_base_url": job["base_url"],
+                    "response_callback": lambda llm, response, params: llm_calls.append(response.choices[0].finish_reason),
+                },
+            },
+            "embedder": {
+                "provider": "openai",
+                "config": {"model": job["embedding_model"], "api_key": job["api_key"], "openai_base_url": job["base_url"]},
+            },
+            "vector_store": {"provider": "qdrant", "config": {"path": os.path.join(store, "qdrant"), "on_disk": True}},
+            "history_db_path": os.path.join(store, "history.db"),
+        }
+    )
+    embed_calls = [0]
+    embeddings = memory.embedding_model.client.embeddings
+    create = embeddings.create
+
+    def counted(*a, **k):
+        embed_calls[0] += 1
+        return create(*a, **k)
+
+    embeddings.create = counted
+
+    user = "longmemeval"
+    add_per = job.get("add_per", "pair")
+    adds = extracted = 0
+    for session in job["sessions"]:
+        when = session_time(session["date"])
+        observation["date"] = when.date().isoformat()
+        for messages in chunks(session["turns"], add_per):
+            result = memory.add(messages, user_id=user, metadata={"created_at": when.isoformat() + "+00:00"})
+            adds += 1
+            extracted += len(result.get("results", []))
+    add_llm, add_embed = len(llm_calls), embed_calls[0]
+
+    top_k = int(job.get("top_k", TOP_K))
+    found = memory.search(job["question"], filters={"user_id": user}, top_k=top_k)["results"]
+    memory_md = render(found, top_k)
+    truncated = sum(1 for f in llm_calls if f == "length")
+    notes = (
+        f"mem0ai {version('mem0ai')}; {len(job['sessions'])} sessions, {adds} add() calls (per {add_per}), {extracted} memories extracted; "
+        f"adding: {add_llm} chat completions ({truncated} cut at max_tokens), {add_embed} embedding requests; "
+        f"search: {len(llm_calls) - add_llm} chat completions, {embed_calls[0] - add_embed} embedding requests, "
+        f"{len(found)} memories of top_k {top_k}, {len(memory_md)} characters"
+    )
+    return {"memory_md": memory_md, "notes": notes}
+
+
+def main() -> None:
+    job = json.load(sys.stdin)
+    with tempfile.TemporaryDirectory(prefix="mem0-") as store:
+        try:
+            with contextlib.redirect_stdout(sys.stderr):  # stdout carries only the answer
+                out = run(job, store)
+        except Exception as error:
+            print(f"memory_mem0: {type(error).__name__}: {error}", file=sys.stderr)
+            sys.exit(1)
+    json.dump(out, sys.stdout, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()

tests/bench/requirements-hindsight.txt

@@ -0,0 +1,241 @@
+# TEST HARNESS ONLY: the venv tests/bench/memory_hindsight.py runs in, inside the bench container (Python 3.12.3,
+# Ubuntu 24.04, no system package beyond python3-venv). Hindsight (MIT, github.com/vectorize-io/hindsight) installed
+# the way its README installs it bare metal, `pip install hindsight-api`, plus its Python client:
+#
+#   hindsight-api==0.10.3
+#   hindsight-client==0.10.3
+#
+# It brings its embedded Postgres (pg0-embedded) and its local embedding and reranker models (sentence-transformers,
+# torch with CUDA wheels, unused on CPU): about 6.8 GB installed. Below, everything pip resolved (pip freeze):
+#
+#   python3 -m venv /opt/mem-hindsight
+#   /opt/mem-hindsight/bin/pip install -r tests/bench/requirements-hindsight.txt
+agent-detector==2.0.0
+aiofile==3.12.3
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.4
+aiohttp-retry==2.9.1
+aiosignal==1.4.0
+alembic==1.20.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anthropic==1.12.1
+anyio==4.15.1
+asgiref==3.12.1
+asyncpg==0.32.0
+attrs==26.1.0
+Authlib==1.8.0
+beartype==0.22.9
+beautifulsoup4==4.15.0
+boto3==1.43.110
+botocore==1.43.110
+cachetools==7.2.1
+caio==0.12.9
+certifi==2026.7.22
+cffi==2.1.1
+charset-normalizer==3.5.2
+claude-agent-sdk==0.2.165
+click==8.5.0
+cloudpickle==3.1.2
+cobble==0.1.4
+cohere==7.2.0
+croniter==6.2.4
+cryptography==50.0.2
+cuda-bindings==13.4.3
+cuda-pathfinder==1.8.3
+cuda-toolkit==13.0.3.0
+cyclopts==5.2.0
+dateparser==1.4.3
+defusedxml==0.7.1
+detect-installer==0.2.1
+distro==1.9.0
+dnspython==2.9.0
+docstring_parser==0.18.0
+einops==0.8.2
+email-validator==2.3.0
+et_xmlfile==2.0.0
+exceptiongroup==1.3.1
+fastapi==0.143.0
+fastapi-cli==0.0.32
+fastapi-cloud-cli==0.26.0
+fastar==0.12.0
+fastavro==1.13.1
+fastmcp==4.1.0
+fastmcp-slim==4.1.0
+fastuuid==0.14.0
+filelock==3.32.7
+FlashRank==0.2.10
+flatbuffers==25.12.19
+frozenlist==1.8.0
+fsspec==2026.9.0
+github-copilot-sdk==1.0.18
+google-auth==2.61.0
+google-genai==2.29.0
+googleapis-common-protos==1.75.5
+greenlet==3.3.2
+griffelib==2.3.2
+h11==0.16.0
+h2==4.4.1
+hf-xet==1.7.0
+hindsight-api==0.10.3
+hindsight-api-slim==0.10.3
+hindsight-client==0.10.3
+hpack==4.2.0
+httpcore==1.0.9
+httpcore2==2.13.1
+httptools==0.8.0
+httpx==0.28.1
+httpx2==2.13.1
+huggingface_hub==1.33.0
+hyperframe==6.1.0
+idna==3.20
+importlib_metadata==8.9.0
+jaraco.classes==3.4.0
+jaraco.context==6.1.2
+jaraco.functools==4.6.0
+jeepney==0.9.0
+Jinja2==3.1.6
+jiter==0.17.0
+jmespath==1.1.0
+joblib==1.6.0
+joserfc==1.7.5
+json_repair==0.63.5
+jsonref==1.1.0
+jsonschema==4.26.0
+jsonschema-path==0.5.0
+jsonschema-specifications==2025.9.1
+keyring==25.7.0
+litellm==1.104.2
+lxml==6.1.3
+magika==0.6.3
+Mako==1.4.3
+mammoth==1.11.0
+markdown-it-py==4.2.0
+markdownify==1.2.3
+markitdown==0.1.8
+MarkupSafe==3.0.4
+mcp==2.3.0
+mcp-types==2.3.0
+mdurl==0.1.2
+more-itertools==11.1.0
+mpmath==1.3.0
+multidict==6.9.1
+narwhals==2.26.0
+networkx==3.7
+numpy==2.5.3
+nvidia-cublas==13.1.1.3
+nvidia-cuda-cupti==13.0.85
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-runtime==13.0.96
+nvidia-cudnn-cu13==9.24.0.43
+nvidia-cufft==12.0.0.61
+nvidia-cufile==1.15.1.6
+nvidia-curand==10.4.0.35
+nvidia-cusolver==12.0.4.66
+nvidia-cusparse==12.6.3.3
+nvidia-cusparselt-cu13==0.8.1
+nvidia-nccl-cu13==2.30.7
+nvidia-nvjitlink==13.4.92
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvtx==13.0.85
+obstore==0.11.1
+onnxruntime==1.31.0
+openai==2.54.0
+openapi-pydantic==0.6.0
+openpyxl==3.1.5
+opentelemetry-api==1.45.1
+opentelemetry-exporter-http-transport==0.66b1
+opentelemetry-exporter-otlp-common==0.66b1
+opentelemetry-exporter-otlp-proto-common==1.45.1
+opentelemetry-exporter-otlp-proto-http==1.45.1
+opentelemetry-exporter-prometheus==0.66b1
+opentelemetry-instrumentation==0.66b1
+opentelemetry-instrumentation-asgi==0.66b1
+opentelemetry-instrumentation-fastapi==0.66b1
+opentelemetry-proto==1.45.1
+opentelemetry-sdk==1.45.1
+opentelemetry-semantic-conventions==0.66b1
+opentelemetry-util-http==0.66b1
+orjson==3.13.0
+packaging==26.3
+pandas==3.0.6
+pathable==0.6.0
+pdfminer.six==20260107
+pdfplumber==0.11.10
+pg0-embedded==0.15.2
+pgvector==0.5.1
+pillow==12.3.0
+platformdirs==4.12.4
+prometheus_client==0.26.0
+propcache==0.5.4
+protobuf==7.36.2
+psycopg2-binary==2.9.13
+py-key-value-aio==0.4.6
+pyasn1==0.6.4
+pyasn1_modules==0.4.2
+pycparser==3.1
+pydantic==2.14.0
+pydantic-extra-types==2.11.1
+pydantic-settings==2.15.0
+pydantic_core==2.50.0
+Pygments==2.21.0
+PyJWT==2.15.1
+pypdfium2==5.14.0
+pyperclip==1.11.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.4
+python-multipart==0.0.32
+python-pptx==1.0.2
+pytz==2026.5
+PyYAML==6.0.3
+referencing==0.37.0
+regex==2026.9.29
+requests==2.34.2
+rich==15.0.0
+rich-rst==2.2.0
+rich-toolkit==0.20.6
+rignore==0.8.1
+rpds-py==2026.9.1
+s3transfer==0.19.2
+safetensors==0.8.0
+scikit-learn==1.9.1
+scipy==1.18.1
+SecretStorage==3.5.0
+sentence-transformers==6.1.0
+sentry-sdk==2.71.0
+setuptools==84.0.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.10
+SQLAlchemy==2.0.54
+sse-starlette==3.5.0
+starlette==1.7.0
+sympy==1.14.0
+tenacity==9.1.4
+threadpoolctl==3.7.0
+tiktoken==0.14.0
+tokenizers==0.22.2
+toktok-rs==0.1.4
+torch==2.14.1
+tqdm==4.70.1
+transformers==5.15.1
+triton==3.8.0
+truststore==0.10.4
+typer==0.27.3
+types-requests==2.33.0.20261006
+typing-inspection==0.4.4
+typing_extensions==4.16.0
+tzlocal==5.4.4
+uncalled-for==0.4.0
+urllib3==2.8.0
+uvicorn==0.54.0
+uvloop==0.23.0
+watchfiles==1.3.0
+websockets==16.1.1
+wrapt==2.5.0
+wsproto==1.3.2
+xlrd==2.0.2
+xlsxwriter==3.2.9
+yarl==1.25.1
+zipp==4.1.1

tests/bench/requirements-letta.txt

@@ -0,0 +1,261 @@
+# TEST HARNESS ONLY: the /opt/mem-letta venv that runs the Letta server and tests/bench/memory_letta.py.
+# letta 0.16.8 (the last server release; later `letta` versions on PyPI are Letta Code) with its server, postgres and
+# sqlite extras, every package at the version of the release's own uv.lock (tag 0.16.8), as installed and checked.
+# psycopg2 (a source build) is left out: psycopg2-binary, also in the postgres extra, provides the same module.
+#   python3 -m venv /opt/mem-letta && /opt/mem-letta/bin/pip install --no-deps -r tests/bench/requirements-letta.txt
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.15
+aiomultiprocess==0.9.1
+aiosignal==1.4.0
+aiosqlite==0.21.0
+alembic==1.16.4
+annotated-types==0.7.0
+anthropic==0.75.0
+anyio==4.10.0
+APScheduler==3.11.0
+argcomplete==3.6.2
+asn1crypto==1.5.1
+asttokens==3.0.0
+async-lru==2.0.5
+asyncpg==0.30.0
+attrs==25.3.0
+Authlib==1.6.6
+babel==2.17.0
+banks==2.2.0
+beautifulsoup4==4.13.5
+black==25.1.0
+Brotli==1.1.0
+bytecode==0.17.0
+cachetools==5.5.2
+certifi==2025.8.3
+cffi==1.17.1
+chardet==5.2.0
+charset-normalizer==3.4.3
+click==8.2.1
+clickhouse-connect==0.10.0
+cobble==0.1.4
+colorama==0.4.6
+coloredlogs==15.0.1
+contourpy==1.3.3
+courlan==1.3.2
+cryptography==45.0.6
+cssselect==1.3.0
+cycler==0.12.1
+cyclopts==4.4.0
+dataclasses-json==0.6.7
+datadog==0.52.1
+datamodel-code-generator==0.33.0
+dateparser==1.2.2
+ddtrace==4.2.1
+decorator==5.2.1
+defusedxml==0.7.1
+demjson3==3.0.6
+Deprecated==1.2.18
+dirtyjson==1.0.8
+distro==1.9.0
+dnspython==2.8.0
+docstring_parser==0.16
+docutils==0.22.4
+email-validator==2.3.0
+envier==0.6.1
+eval_type_backport==0.2.2
+exa-py==1.15.4
+exceptiongroup==1.3.1
+executing==2.2.0
+Faker==37.6.0
+fastapi==0.116.1
+fastmcp==2.12.5
+filetype==1.2.0
+flatbuffers==25.2.10
+fonttools==4.59.1
+frozenlist==1.7.0
+fsspec==2025.7.0
+genson==1.3.0
+google-auth==2.40.3
+google-genai==1.52.0
+googleapis-common-protos==1.70.0
+greenlet==3.2.4
+griffe==1.13.0
+grpcio==1.74.0
+grpcio-tools==1.71.2
+h11==0.16.0
+html2text==2025.4.15
+htmldate==1.9.3
+httpcore==1.0.9
+httpx==0.28.1
+httpx-sse==0.4.0
+humanfriendly==10.0
+idna==3.10
+importlib_metadata==8.5.0
+inflect==7.5.0
+invoke==2.2.0
+ipython==9.4.0
+ipython_pygments_lexers==1.1.1
+isodate==0.7.2
+isort==6.0.1
+jedi==0.19.2
+Jinja2==3.1.6
+jiter==0.10.0
+joblib==1.5.1
+jsonschema==4.25.1
+jsonschema-path==0.3.4
+jsonschema-specifications==2025.4.1
+jusText==3.0.2
+kiwisolver==1.4.9
+lazy-object-proxy==1.12.0
+letta==0.16.8
+letta-client==1.7.12
+llama-cloud==0.1.35
+llama-cloud-services==0.6.54
+llama-index==0.14.10
+llama-index-cli==0.5.3
+llama-index-core==0.14.10
+llama-index-embeddings-openai==0.5.1
+llama-index-indices-managed-llama-cloud==0.9.4
+llama-index-instrumentation==0.4.0
+llama-index-llms-openai==0.6.10
+llama-index-readers-file==0.5.5
+llama-index-readers-llama-parse==0.5.1
+llama-index-workflows==2.11.5
+llama-parse==0.6.54
+lxml==5.4.0
+lxml_html_clean==0.4.2
+lz4==4.4.5
+magika==0.6.2
+Mako==1.3.10
+mammoth==1.10.0
+markdown-it-py==4.0.0
+markdownify==1.2.0
+markitdown==0.1.2
+MarkupSafe==3.0.2
+marshmallow==3.26.1
+marshmallow-sqlalchemy==1.4.2
+matplotlib==3.10.5
+matplotlib-inline==0.1.7
+mcp==1.12.4
+mdurl==0.1.2
+mistralai==1.9.9
+more-itertools==10.7.0
+mpmath==1.3.0
+multidict==6.6.4
+mypy_extensions==1.1.0
+nest-asyncio==1.6.0
+networkx==3.5
+nexus-rpc==1.1.0
+nltk==3.9.1
+numpy==2.3.2
+onnxruntime==1.22.1
+openai==2.25.0
+openapi-core==0.21.0
+openapi-pydantic==0.5.1
+openapi-schema-validator==0.6.3
+openapi-spec-validator==0.7.2
+opentelemetry-api==1.30.0
+opentelemetry-exporter-otlp==1.30.0
+opentelemetry-exporter-otlp-proto-common==1.30.0
+opentelemetry-exporter-otlp-proto-grpc==1.30.0
+opentelemetry-exporter-otlp-proto-http==1.30.0
+opentelemetry-instrumentation==0.51b0
+opentelemetry-instrumentation-requests==0.51b0
+opentelemetry-instrumentation-sqlalchemy==0.51b0
+opentelemetry-proto==1.30.0
+opentelemetry-sdk==1.30.0
+opentelemetry-semantic-conventions==0.51b0
+opentelemetry-util-http==0.51b0
+orjson==3.11.3
+packaging==24.2
+pandas==2.2.3
+parso==0.8.5
+pathable==0.4.4
+pathspec==0.12.1
+pathvalidate==3.3.1
+pdfminer.six==20250506
+pexpect==4.9.0
+pg8000==1.31.4
+pgvector==0.4.1
+pillow==10.4.0
+platformdirs==4.4.0
+prettytable==3.16.0
+prompt_toolkit==3.0.51
+propcache==0.3.2
+protobuf==5.29.5
+psutil==7.0.0
+psycopg2-binary==2.9.10
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pycparser==2.22
+pydantic==2.11.7
+pydantic-settings==2.10.1
+pydantic_core==2.33.2
+Pygments==2.19.2
+pyhumps==3.8.0
+pyparsing==3.2.3
+pypdf==6.4.1
+pyperclip==1.11.0
+python-box==7.3.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.1
+python-multipart==0.0.20
+python-pptx==1.0.2
+pytz==2025.2
+PyYAML==6.0.2
+questionary==2.1.0
+readability-lxml==0.8.4.1
+referencing==0.36.2
+regex==2025.7.34
+requests==2.32.5
+rfc3339-validator==0.1.4
+rich==13.9.4
+rich-rst==1.3.2
+rpds-py==0.27.0
+rsa==4.9.1
+ruff==0.15.1
+scramp==1.4.6
+sentry-sdk==2.19.1
+setuptools==80.9.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.7
+SQLAlchemy==2.0.43
+sqlalchemy-json==0.7.0
+SQLAlchemy-Utils==0.41.2
+sqlite-vec==0.1.7a2
+sqlmodel==0.0.24
+sse-starlette==3.0.2
+stack-data==0.6.3
+starlette==0.47.3
+striprtf==0.0.26
+structlog==25.4.0
+sympy==1.14.0
+tavily-python==0.7.11
+temporalio==1.17.0
+tenacity==9.1.2
+tiktoken==0.11.0
+tld==0.13.1
+tokenize_rt==6.2.0
+tqdm==4.67.1
+trafilatura==2.0.0
+traitlets==5.14.3
+ty==0.0.17
+typeguard==4.4.4
+typer==0.16.1
+types-protobuf==6.30.2.20250914
+typing-inspect==0.9.0
+typing-inspection==0.4.1
+typing_extensions==4.15.0
+tzdata==2025.2
+tzlocal==5.3.1
+urllib3==2.5.0
+uvicorn==0.29.0
+wcwidth==0.2.13
+websockets==15.0.1
+Werkzeug==3.1.3
+wrapt==1.17.3
+xlsxwriter==3.2.5
+yarl==1.20.1
+zipp==3.23.0
+zstandard==0.24.0

tests/bench/requirements-mem0.txt

@@ -0,0 +1,82 @@
+# TEST HARNESS ONLY: the venv memory_mem0.py runs in (Mem0 open source, Apache-2.0), as installed in idots-bench:
+#   python3 -m venv /opt/mem-mem0 && /opt/mem-mem0/bin/pip install -r tests/bench/requirements-mem0.txt
+#   mkdir -p /opt/mem-mem0/fastembed_cache && FASTEMBED_CACHE_PATH=/opt/mem-mem0/fastembed_cache /opt/mem-mem0/bin/python \
+#     -c 'from fastembed import SparseTextEmbedding as S; S(model_name="Qdrant/bm25")' && chmod -R a+rX /opt/mem-mem0
+# mem0ai plus what its v3 search uses when present: spacy and en_core_web_sm (entities, BM25 lemmas), fastembed (BM25).
+mem0ai==2.2.1
+spacy==3.8.16
+en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl#sha256=1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85
+fastembed==0.9.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anyio==4.15.1
+backoff==2.2.1
+blis==1.3.3
+catalogue==2.0.10
+certifi==2026.7.22
+charset-normalizer==3.5.2
+click==8.5.0
+cloudpathlib==0.26.0
+confection==1.3.3
+cymem==2.0.13
+distro==1.9.0
+filelock==4.0.12
+flatbuffers==25.12.19
+fsspec==2026.9.0
+grpcio==1.84.0
+h11==0.16.0
+h2==4.4.1
+hf-xet==1.7.0
+hpack==4.2.0
+httpcore==1.0.9
+httpcore2==2.13.1
+httpx==0.28.1
+httpx2==2.13.1
+huggingface_hub==1.33.0
+hyperframe==6.1.0
+idna==3.20
+Jinja2==3.1.6
+jiter==0.17.0
+loguru==0.7.3
+markdown-it-py==4.2.0
+MarkupSafe==3.0.4
+mdurl==0.1.2
+mmh3==5.3.1
+murmurhash==1.0.15
+numpy==2.5.3
+onnxruntime==1.31.0
+openai==3.26.1
+packaging==26.3
+pillow==12.3.0
+portalocker==3.2.0
+posthog==7.66.0
+preshed==3.0.13
+protobuf==6.33.6
+py_rust_stemmers==0.1.8
+pydantic==2.14.0
+pydantic_core==2.50.0
+Pygments==2.21.0
+pytz==2026.5
+PyYAML==6.0.3
+qdrant-client==1.19.1
+requests==2.34.2
+rich==15.0.0
+setuptools==84.0.0
+shellingham==1.5.4
+smart_open==8.0.2
+sniffio==1.3.1
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+SQLAlchemy==2.1.4
+srsly==2.5.4
+thinc==8.3.13
+tokenizers==0.23.2
+tqdm==4.70.1
+truststore==0.10.4
+typer==0.27.3
+typing-inspection==0.4.4
+typing_extensions==4.16.0
+urllib3==2.8.0
+wasabi==1.1.3
+weasel==1.0.0
+wrapt==2.5.0