fix(sdk): preserve media references in summarization archives (#3990)

Fixes #[2873](https://github.com/langchain-ai/deepagents/issues/2873)

---

## User story

As a user whose conversation history includes messages with media, I
expect that content to remain recoverable when `SummarizationMiddleware`
offloads older messages during compaction.

---

When a thread grows past the summarization threshold,
`SummarizationMiddleware` keeps the conversation small by moving older
messages out of the active context. It first saves those older messages
to `/conversation_history/{thread_id}.md`, then replaces them in the
active conversation with a concise summary and a pointer to the saved
file. See the [summarization
docs](https://docs.langchain.com/oss/deepagents/context-engineering#summarization)
for background on this flow. Crucially, when messages are offloaded to
disk, they should preserve all content originally present.

Previously, the offload step relied on LangChain's default message
rendering (via `get_buffer_string()`), which only preserved text
content. Non-text content blocks were silently dropped from the saved
history file. The issue surfaced with images, but the underlying problem
was broader: URL-based media references could be omitted, and base64
media had no durable reference once the original messages were compacted
away.

---

`SummarizationMiddleware` now preserves media references during offload.
For media that already has a URL, the offload step now writes the
message in `get_buffer_string()`'s XML format, which keeps the media
reference in the saved history file.

For base64 image content, `SummarizationMiddleware` first writes the
image bytes to the backend alongside the saved conversation history,
under `/conversation_history/media/{content_hash}.{ext}`. It then
rewrites the original base64 content block into a normal image reference
pointing at that stored file. For example, a block containing inline
base64 image data becomes a reference like:

```json
{
  "type": "image",
  "url": "/conversation_history/media/{content_hash}.png"
}
```

When the message is written to the saved history file, that reference is
preserved in the XML-rendered output rather than embedding the raw
base64 payload.

The helper handles all three base64 shapes recognized by langchain_core:

- Explicit base64 field: {"type": "image", "base64": "...", "mime_type":
"..."}
    - Top-level data: URL: {"image/png;base64,..."}
- OpenAI-style image_url: {"type": "image_url", "image_url": {"url":
"data:..."}}

If storing one of those files fails, the saved history file includes an
explicit failed-offload marker for that block instead of making it look
like the media was never there:

```xml
<image error="failed_to_offload" />
```

The difference is that this file now keeps durable references for media
content instead of preserving only the surrounding text.

---

## Dependency note

This PR should land with a Deep Agents LangChain dependency bump to a
version that includes
[langchain-ai/langchain#38171](https://github.com/langchain-ai/langchain/pull/38171).
That upstream change switches `SummarizationMiddleware` summary-input
serialization to XML so URL-backed media survives the summarization
prompt, matching the archive-preservation behavior here.

---------

Signed-off-by: Nishitha M <32355027+imnishitha@users.noreply.github.com>
Co-authored-by: Mason Daugherty <github@mdrxy.com>
Co-authored-by: Mason Daugherty <mason@langchain.dev>

libs/deepagents/deepagents/middleware/__init__.py

@@ -70,12 +70,14 @@ from deepagents.middleware.subagents import (
     SubAgentMiddleware,
 )
 from deepagents.middleware.summarization import (
+    DEEPAGENTS_DEFAULT_SUMMARY_PROMPT,
     SummarizationMiddleware,
     SummarizationToolMiddleware,
     create_summarization_tool_middleware,
 )
 
 __all__ = [
+    "DEEPAGENTS_DEFAULT_SUMMARY_PROMPT",
     "GRADER_SYSTEM_PROMPT",
     "RUBRIC_GRADER_MESSAGE_SOURCE",
     "AsyncSubAgent",

libs/deepagents/deepagents/middleware/summarization.py

@@ -29,7 +29,7 @@ from deepagents.backends import FilesystemBackend
 backend = FilesystemBackend(root_dir="/data")
 
 summ = SummarizationMiddleware(
-    model="gpt-5.4-mini",
+    model="gpt-5.5",
     backend=backend,
     trigger=("fraction", 0.85),
     keep=("fraction", 0.10),
@@ -44,14 +44,29 @@ agent = create_deep_agent(middleware=[summ, tool_mw])
 Offloaded messages are stored as markdown at `/conversation_history/{thread_id}.md`.
 
 Each summarization event appends a new section to this file, creating a running
-log of all evicted messages.
+log of all evicted messages. Base64 media in evicted messages is written
+separately under `<artifacts_root>/conversation_history/media/` and referenced
+by path from the markdown, so the history file stays text-only (see
+`_offload_inline_media` for the exact path).
+
+## Summary prompt
+
+`DEEPAGENTS_DEFAULT_SUMMARY_PROMPT` augments LangChain's `DEFAULT_SUMMARY_PROMPT`
+with a deepagents-specific addendum explaining the media reference tags that the
+offloading behavior introduces, so the summarizing model knows to preserve them.
+It is the default `summary_prompt` for `SummarizationMiddleware` and both
+factories.
 """
 
 from __future__ import annotations
 
 import asyncio
+import base64
+import hashlib
 import inspect
 import logging
+import mimetypes
+import urllib.parse
 import uuid
 import warnings
 from collections.abc import Mapping
@@ -82,6 +97,25 @@ from deepagents.backends.protocol import _resolve_backend
 from deepagents.middleware._overflow_clip import _aclip_overflow_tail, _clip_overflow_tail
 from deepagents.middleware._utils import append_to_system_message
 
+_MEDIA_REFERENCE_SUMMARY_PROMPT = """<media_reference_information>
+Conversation history may include XML media reference tags, for example:
+<image url=\"/conversation_history/media/{{hash}}.png\" />
+These tags mean the original message included media that was preserved at the referenced backend path.
+Treat the tag and path as part of the conversation context. Do not infer visual details that are not available from surrounding text.
+When the media could be important for future context, preserve the media reference in your summary.
+The model consuming the summary can call `read_file` on the referenced path if it needs to inspect the media.
+</media_reference_information>"""
+
+# NOTE: This splices the media-reference addendum in just before the
+# `<messages>` marker that `DEFAULT_SUMMARY_PROMPT` exposes. That marker is a
+# load-bearing contract -- see the `DEFAULT_SUMMARY_PROMPT` docstring in
+# langchain for the downstream-dependency note.
+DEEPAGENTS_DEFAULT_SUMMARY_PROMPT = DEFAULT_SUMMARY_PROMPT.replace(
+    "\n<messages>\n",
+    f"\n{_MEDIA_REFERENCE_SUMMARY_PROMPT}\n\n<messages>\n",
+    1,
+)
+
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Callable
 
@@ -91,7 +125,7 @@ if TYPE_CHECKING:
     from langchain_core.tools import BaseTool
     from langgraph.runtime import Runtime
 
-    from deepagents.backends.protocol import BACKEND_TYPES, BackendProtocol
+    from deepagents.backends.protocol import BACKEND_TYPES, BackendProtocol, FileUploadResponse
 
 logger = logging.getLogger(__name__)
 
@@ -260,6 +294,204 @@ def compute_summarization_defaults(model: BaseChatModel) -> SummarizationDefault
     }
 
 
+_OFFLOAD_FAILED_PLACEHOLDER = '<image error="failed_to_offload" />'
+"""Text placeholder written when a media block cannot be offloaded.
+
+Marks the spot so the saved history shows a block was present rather than
+silently omitting it.
+"""
+
+
+def _is_data_url(url: str) -> bool:
+    """Return whether `url` is an inline `data:` URL.
+
+    Any `data:` URL is treated as inline media to offload, because the XML
+    history renderer drops `data:` URL blocks entirely (only `http(s)`-style
+    references survive). This covers both base64 (`data:<mime>;base64,<payload>`)
+    and percent-encoded / plaintext (`data:<mime>,<payload>`, e.g. an inline SVG)
+    forms; whether the payload actually decodes is left to `_decode_data_url`.
+    """
+    return url.startswith("data:")
+
+
+def _extract_data_url(block: Any) -> str | None:  # noqa: ANN401
+    """Return the embedded `data:` URL for an inline-media content block.
+
+    Detects the three inline-data content-block shapes that appear across
+    LangChain messages:
+
+    1. A standard content block with an explicit `base64` field.
+    2. A `data:` URL on the `url` field.
+    3. An OpenAI-style `image_url` block whose `url` is a `data:` URL.
+
+    Both base64 (`;base64,`) and percent-encoded / plaintext `data:` URLs are
+    detected -- e.g. an inline SVG (`data:image/svg+xml,<svg .../>`) -- because
+    the XML history renderer drops *any* inline `data:` URL, so all of them must
+    be offloaded to a referenceable path rather than left inline.
+
+    Shape 3 is defensive: `content_blocks` normalizes most `image_url` blocks
+    (a base64 `data:` URL becomes shape 1; an `https` URL becomes a plain `url`
+    image block), so this branch rarely fires for normalized input; it is kept
+    for raw, un-normalized blocks.
+
+    This is pure detection and never raises: it reports *whether* a block
+    carries inline data, leaving decoding (which can fail) to `_decode_data_url`.
+
+    Args:
+        block: A single content block (usually a dict).
+
+    Returns:
+        The block's `data:` URL, or `None` if the block carries no inline data.
+    """
+    if not isinstance(block, dict):
+        return None
+
+    # 1. Standard content block with an explicit base64 field.
+    raw_b64 = block.get("base64")
+    if raw_b64:
+        mime = block.get("mime_type") or "application/octet-stream"
+        return f"data:{mime};base64,{raw_b64}"
+
+    # 2. Top-level data: URL.
+    url = block.get("url", "")
+    if isinstance(url, str) and _is_data_url(url):
+        return url
+
+    # 3. OpenAI-style image_url with a data: URL.
+    image_url = block.get("image_url")
+    if isinstance(image_url, dict):
+        inner = image_url.get("url", "")
+        if isinstance(inner, str) and _is_data_url(inner):
+            return inner
+
+    return None
+
+
+def _decode_data_url(data_url: str) -> tuple[bytes, str, str] | None:
+    """Decode a `data:` URL to raw bytes, a file extension, and a MIME type.
+
+    Handles both encodings a `data:` URL can use: a `;base64,` payload is
+    base64-decoded, while a plain `data:<mime>,<payload>` payload is treated as
+    percent-encoded text (e.g. an inline SVG).
+
+    Args:
+        data_url: A `data:<mime>[;base64],<payload>` URL.
+
+    Returns:
+        A `(raw_bytes, extension, mime_type)` tuple, or `None` if decoding fails
+            (including a malformed URL with no `,` payload separator). A failure
+            is logged here and, like an upload failure, surfaces as a
+            failed-offload placeholder that counts toward the caller's aggregate
+            warning -- it is never swallowed silently.
+    """
+    try:
+        header, payload = data_url.split(",", 1)
+        mime = header.split(":")[1].split(";")[0] if ":" in header else "application/octet-stream"
+        ext = (mimetypes.guess_extension(mime) or ".bin").lstrip(".")
+        is_base64 = "base64" in header.lower().split(";")
+        raw = base64.b64decode(payload) if is_base64 else urllib.parse.unquote_to_bytes(payload)
+    except Exception as e:  # noqa: BLE001
+        logger.warning("Failed to decode data: content block (%s): %s", type(e).__name__, e)
+        return None
+    else:
+        return raw, ext, mime
+
+
+def _media_reference_block(path: str, mime: str) -> dict[str, Any]:
+    """Build a content block referencing offloaded media by backend path.
+
+    The block type is chosen so the XML history renderer serializes the
+    reference: `image`, `audio`, and `video` map to their typed blocks, while
+    any other MIME type falls back to a text block (the renderer has no generic
+    file block and would otherwise drop it).
+
+    Args:
+        path: Backend path where the media was stored.
+        mime: MIME type of the original media, used to pick the block type.
+
+    Returns:
+        A content block carrying the path reference.
+    """
+    major = mime.split("/", 1)[0]
+    if major in {"image", "audio", "video"}:
+        return {"type": major, "url": path}
+    return {"type": "text", "text": f'<file url="{path}" />'}
+
+
+def _rewrite_data_url_blocks(
+    messages: list[AnyMessage],
+    path_map: dict[str, str],
+) -> tuple[list[AnyMessage], int]:
+    """Rewrite inline `data:` URL blocks using uploaded media paths.
+
+    Each inline-data block whose content hash appears in `path_map` becomes a
+    typed media reference block. Blocks whose upload failed -- or whose payload
+    could not be decoded -- become an `<image error="failed_to_offload" />` text
+    placeholder so the saved history records that media was present rather than
+    silently dropping it. Blocks without inline data pass through unchanged.
+
+    Args:
+        messages: Messages whose inline-data blocks should be rewritten.
+        path_map: Mapping of `sha256[:16]` to backend paths for uploaded media.
+
+    Returns:
+        A `(messages, failed_block_count)` tuple. `messages` has inline-data
+            blocks replaced (messages without inline data are returned without
+            copying). `failed_block_count` is the number of blocks rewritten to a
+            failed-offload placeholder -- covering both upload failures (key not
+            in `path_map`) and decode failures -- so the caller can report how
+            much media is unrecoverable.
+    """
+    rewritten: list[AnyMessage] = []
+    failed_blocks = 0
+    for msg in messages:
+        new_blocks: list[Any] = []
+        modified = False
+        for block in msg.content_blocks:
+            data_url = _extract_data_url(block)
+            if data_url is None:
+                new_blocks.append(block)
+                continue
+            modified = True
+            decoded = _decode_data_url(data_url)
+            if decoded is not None:
+                raw, _ext, mime = decoded
+                key = hashlib.sha256(raw).hexdigest()[:16]
+                if key in path_map:
+                    new_blocks.append(_media_reference_block(path_map[key], mime))
+                    continue
+            failed_blocks += 1
+            new_blocks.append({"type": "text", "text": _OFFLOAD_FAILED_PLACEHOLDER})
+        if modified:
+            new_msg = msg.model_copy()
+            new_msg.content = new_blocks
+            rewritten.append(new_msg)
+        else:
+            rewritten.append(msg)
+    return rewritten, failed_blocks
+
+
+def _upload_response_error(responses: list[FileUploadResponse]) -> str | None:
+    """Extract an error from a single-file batch upload result.
+
+    Args:
+        responses: Backend upload responses. `upload_files`/`aupload_files`
+            are batch APIs that return one `FileUploadResponse` per input file
+            in order. Image offloading passes exactly one file at a time, so the
+            expected length is 1 and `responses[0]` maps to that file.
+
+    Returns:
+        The upload error, `"missing_upload_response"` if the backend returned
+            no response, or `None` when the upload succeeded.
+    """
+    if not responses:
+        return "missing_upload_response"
+    error = responses[0].error
+    if error is None:
+        return None
+    return str(error)
+
+
 class _DeepAgentsSummarizationMiddleware(AgentMiddleware):
     """Summarization middleware with backend for conversation history offloading."""
 
@@ -289,7 +521,7 @@ class _DeepAgentsSummarizationMiddleware(AgentMiddleware):
         trigger: ContextSize | TriggerClause | list[ContextSize | TriggerClause] | None = None,
         keep: ContextSize = ("messages", _DEFAULT_MESSAGES_TO_KEEP),
         token_counter: TokenCounter = count_tokens_approximately,
-        summary_prompt: str = DEFAULT_SUMMARY_PROMPT,
+        summary_prompt: str = DEEPAGENTS_DEFAULT_SUMMARY_PROMPT,
         trim_tokens_to_summarize: int | None = _DEFAULT_TRIM_TOKEN_LIMIT,
         truncate_args_settings: TruncateArgsSettings | None = None,
         **deprecated_kwargs: Any,
@@ -331,7 +563,7 @@ class _DeepAgentsSummarizationMiddleware(AgentMiddleware):
             from deepagents.backends import StateBackend
 
             middleware = SummarizationMiddleware(
-                model="gpt-5.4-mini",
+                model="gpt-5.5",
                 backend=StateBackend(),
                 trigger=("tokens", 100000),
                 keep=("messages", 20),
@@ -379,6 +611,7 @@ class _DeepAgentsSummarizationMiddleware(AgentMiddleware):
 
         if _deprecated_history_prefix is not None:
             self._history_path_prefix = _deprecated_history_prefix
+        self._media_prefix = f"{self._history_path_prefix}/media"
 
         # Parse truncate_args_settings
         if truncate_args_settings is None:
@@ -850,6 +1083,141 @@ A condensed summary follows:
 
         return truncated_messages, modified
 
+    def _offload_inline_media(
+        self,
+        backend: BackendProtocol,
+        messages: list[AnyMessage],
+    ) -> tuple[list[AnyMessage], int]:
+        """Decode inline `data:` media blocks to files and replace them with path references.
+
+        Covers any inline `data:` URL (base64 or percent-encoded/plaintext), not
+        just base64, because the XML history renderer drops every inline `data:`
+        URL. The caller uploads media before both `_offload_to_backend` and
+        `_create_summary`, so both paths receive messages with inline data
+        replaced by path references (or error placeholders when an upload fails).
+        The archive keeps addressable `<image url="..." />` references, and the
+        summary prompt does not receive raw media bytes.
+
+        Each unique media file is uploaded once to
+        `{artifacts_root}/conversation_history/media/{sha256[:16]}.{ext}` (the
+        prefix follows the backend's `artifacts_root`, defaulting to `/`).
+        Identical media across messages are deduped by content hash.
+
+        Failures are tracked per block. A block whose upload failed -- or whose
+        payload could not be decoded -- is replaced with an
+        `<image error="failed_to_offload" />` text placeholder; a successfully
+        uploaded block is rewritten to a typed media reference block. The caller
+        receives the count of failed blocks so it can warn that those media are
+        unrecoverable.
+
+        Args:
+            backend: Backend to write media files to.
+            messages: Messages to process.
+
+        Returns:
+            A `(messages, failed_block_count)` tuple. `messages` has base64
+                blocks replaced by path-reference media blocks or error
+                placeholders; messages without base64 content are returned
+                unchanged. `failed_block_count` is the number of media blocks
+                that became failed-offload placeholders.
+        """
+        path_map: dict[str, str] = {}  # key -> backend path (successfully uploaded)
+        failed_keys: set[str] = set()  # keys whose upload failed
+        saw_inline_media = False
+
+        # First pass: upload each unique media file individually for per-block failure tracking.
+        for msg in messages:
+            for block in msg.content_blocks:
+                data_url = _extract_data_url(block)
+                if data_url is None:
+                    continue
+                saw_inline_media = True
+                decoded = _decode_data_url(data_url)
+                if decoded is None:
+                    continue  # undecodable; rewrite emits a failed-offload placeholder
+                raw, ext, _mime = decoded
+                key = hashlib.sha256(raw).hexdigest()[:16]
+                if key in path_map or key in failed_keys:
+                    continue
+                img_path = f"{self._media_prefix}/{key}.{ext}"
+                try:
+                    responses = backend.upload_files([(img_path, raw)])
+                    if error := _upload_response_error(responses):
+                        logger.warning(
+                            "Failed to upload media %s to backend: %s",
+                            img_path,
+                            error,
+                        )
+                        failed_keys.add(key)
+                        continue
+                    path_map[key] = img_path
+                except Exception as e:  # noqa: BLE001
+                    logger.warning(
+                        "Failed to upload media %s to backend: %s: %s",
+                        img_path,
+                        type(e).__name__,
+                        e,
+                    )
+                    failed_keys.add(key)
+
+        if not saw_inline_media:
+            return messages, 0  # no inline media present; return originals unchanged
+
+        return _rewrite_data_url_blocks(messages, path_map)
+
+    async def _aoffload_inline_media(
+        self,
+        backend: BackendProtocol,
+        messages: list[AnyMessage],
+    ) -> tuple[list[AnyMessage], int]:
+        """Async twin of `_offload_inline_media` using `aupload_files`.
+
+        See `_offload_inline_media` for full documentation, including the
+        `(messages, failed_block_count)` return contract.
+        """
+        path_map: dict[str, str] = {}
+        failed_keys: set[str] = set()
+        saw_inline_media = False
+
+        for msg in messages:
+            for block in msg.content_blocks:
+                data_url = _extract_data_url(block)
+                if data_url is None:
+                    continue
+                saw_inline_media = True
+                decoded = _decode_data_url(data_url)
+                if decoded is None:
+                    continue  # undecodable; rewrite emits a failed-offload placeholder
+                raw, ext, _mime = decoded
+                key = hashlib.sha256(raw).hexdigest()[:16]
+                if key in path_map or key in failed_keys:
+                    continue
+                img_path = f"{self._media_prefix}/{key}.{ext}"
+                try:
+                    responses = await backend.aupload_files([(img_path, raw)])
+                    if error := _upload_response_error(responses):
+                        logger.warning(
+                            "Failed to upload media %s to backend: %s",
+                            img_path,
+                            error,
+                        )
+                        failed_keys.add(key)
+                        continue
+                    path_map[key] = img_path
+                except Exception as e:  # noqa: BLE001
+                    logger.warning(
+                        "Failed to upload media %s to backend: %s: %s",
+                        img_path,
+                        type(e).__name__,
+                        e,
+                    )
+                    failed_keys.add(key)
+
+        if not saw_inline_media:
+            return messages, 0
+
+        return _rewrite_data_url_blocks(messages, path_map)
+
     def _offload_to_backend(
         self,
         backend: BackendProtocol,
@@ -875,11 +1243,12 @@ A condensed summary follows:
         """
         path = self._get_history_path()
 
-        # Filter out previous summary messages to avoid redundant storage
+        # Filter out previous summary messages to avoid redundant storage.
+        # Base64 images are already converted to path references by the caller.
         filtered_messages = self._filter_summary_messages(messages)
 
         timestamp = datetime.now(UTC).isoformat()
-        new_section = f"## Summarized at {timestamp}\n\n{get_buffer_string(filtered_messages)}\n\n"
+        new_section = f"## Summarized at {timestamp}\n\n{get_buffer_string(filtered_messages, format='xml')}\n\n"
 
         # Read existing content (if any) and append.
         # Note: We use download_files() instead of read() because read() returns
@@ -949,11 +1318,12 @@ A condensed summary follows:
         """
         path = self._get_history_path()
 
-        # Filter out previous summary messages to avoid redundant storage
+        # Filter out previous summary messages to avoid redundant storage.
+        # Base64 images are already converted to path references by the caller.
         filtered_messages = self._filter_summary_messages(messages)
 
         timestamp = datetime.now(UTC).isoformat()
-        new_section = f"## Summarized at {timestamp}\n\n{get_buffer_string(filtered_messages)}\n\n"
+        new_section = f"## Summarized at {timestamp}\n\n{get_buffer_string(filtered_messages, format='xml')}\n\n"
 
         # Read existing content (if any) and append.
         # Note: We use adownload_files() instead of aread() because read() returns
@@ -1083,16 +1453,29 @@ A condensed summary follows:
                 large_tool_results_prefix=self._large_tool_results_prefix,
             )
 
+        # Upload inline media once so both offload and summary see path references.
+        offloaded_media_messages, failed_media = self._offload_inline_media(backend, messages_to_summarize)
+
         # Offload to backend first so history is preserved before summarization.
         # If offload fails, summarization still proceeds (with file_path=None).
-        file_path = self._offload_to_backend(backend, messages_to_summarize)
+        file_path = self._offload_to_backend(backend, offloaded_media_messages)
         if file_path is None:
             msg = "Offloading conversation history to backend failed during summarization. Older messages will not be recoverable."
             logger.error(msg)
             warnings.warn(msg, stacklevel=2)
+        elif failed_media:
+            # History was saved, but some media became failed-offload placeholders.
+            # Tie the warning to the saved file so the recovery pointer is honest.
+            msg = (
+                f"Conversation history was offloaded to {file_path}, but {failed_media} media "
+                "block(s) could not be offloaded and appear as failed placeholders in the saved "
+                "history; the original media is not recoverable."
+            )
+            logger.warning(msg)
+            warnings.warn(msg, stacklevel=2)
 
         # Generate summary
-        summary = self._create_summary(messages_to_summarize)
+        summary = self._create_summary(offloaded_media_messages)
 
         # Build summary message with file path reference
         new_messages = self._build_new_messages_with_path(summary, file_path)
@@ -1204,16 +1587,30 @@ A condensed summary follows:
                 large_tool_results_prefix=self._large_tool_results_prefix,
             )
 
+        # Upload inline media once so both offload and summary see path references.
+        # This must complete before the gather since both methods consume the result.
+        offloaded_media_messages, failed_media = await self._aoffload_inline_media(backend, messages_to_summarize)
+
         # Offload to backend and generate summary concurrently -- they are independent.
         # If offload fails, summarization still proceeds (with file_path=None).
         file_path, summary = await asyncio.gather(
-            self._aoffload_to_backend(backend, messages_to_summarize),
-            self._acreate_summary(messages_to_summarize),
+            self._aoffload_to_backend(backend, offloaded_media_messages),
+            self._acreate_summary(offloaded_media_messages),
         )
         if file_path is None:
             msg = "Offloading conversation history to backend failed during summarization. Older messages will not be recoverable."
             logger.error(msg)
             warnings.warn(msg, stacklevel=2)
+        elif failed_media:
+            # History was saved, but some media became failed-offload placeholders.
+            # Tie the warning to the saved file so the recovery pointer is honest.
+            msg = (
+                f"Conversation history was offloaded to {file_path}, but {failed_media} media "
+                "block(s) could not be offloaded and appear as failed placeholders in the saved "
+                "history; the original media is not recoverable."
+            )
+            logger.warning(msg)
+            warnings.warn(msg, stacklevel=2)
 
         # Build summary message with file path reference
         new_messages = self._build_new_messages_with_path(summary, file_path)
@@ -1254,7 +1651,7 @@ def create_summarization_middleware(
     model: BaseChatModel,
     backend: BACKEND_TYPES,
     *,
-    summary_prompt: str = DEFAULT_SUMMARY_PROMPT,
+    summary_prompt: str = DEEPAGENTS_DEFAULT_SUMMARY_PROMPT,
     trim_tokens_to_summarize: int | None = None,
     token_counter: TokenCounter = count_tokens_approximately,
 ) -> _DeepAgentsSummarizationMiddleware:
@@ -1379,7 +1776,7 @@ def create_summarization_tool_middleware(
             create_summarization_tool_middleware,
         )
 
-        model = "openai:gpt-5.4"
+        model = "openai:gpt-5.5"
         agent = create_deep_agent(
             model=model,
             middleware=[
@@ -1400,7 +1797,7 @@ def create_summarization_tool_middleware(
 
         sandbox = Daytona().create()
         backend = DaytonaSandbox(sandbox=sandbox)
-        model = "openai:gpt-5.4"
+        model = "openai:gpt-5.5"
         agent = create_deep_agent(
             model=model,
             backend=backend,
@@ -1446,7 +1843,7 @@ class SummarizationToolMiddleware(AgentMiddleware):
             SummarizationToolMiddleware,
         )
 
-        summ = SummarizationMiddleware(model="gpt-5.4-mini", backend=backend)
+        summ = SummarizationMiddleware(model="gpt-5.5", backend=backend)
         tool_mw = SummarizationToolMiddleware(summ)
 
         agent = create_deep_agent(middleware=[summ, tool_mw])

libs/deepagents/pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
     "langsmith>=0.8.11",
     "langchain-anthropic>=1.4.6,<2.0.0",
     "langchain-google-genai>=4.2.5,<5.0.0",
-    "langchain>=1.3.9,<2.0.0",
+    "langchain>=1.3.10,<2.0.0",
     "wcmatch>=10.1",
 ]
 

libs/deepagents/tests/unit_tests/middleware/test_summarization_factory.py

@@ -44,6 +44,21 @@ def test_factory_uses_fallback_defaults_without_profile() -> None:
     assert middleware._truncate_args_keep == ("messages", 20)
 
 
+def test_factory_default_prompt_explains_media_references() -> None:
+    """Explains preserved media tags in the default summary prompt."""
+    model = _make_model(with_profile_limit=None)
+    middleware = create_summarization_middleware(model, cast("Any", MagicMock()))
+
+    # The prompt is consumed via str.format(messages=...), so the example's
+    # braces must be escaped in the template and survive formatting. Assert
+    # against the rendered result -- this also guards against the literal
+    # `{hash}` regression that made format() raise KeyError.
+    rendered = middleware._lc_helper.summary_prompt.format(messages="<conversation>")
+    assert '<image url="/conversation_history/media/{hash}.png" />' in rendered
+    assert "preserve the media reference in your summary" in rendered
+    assert "call `read_file` on the referenced path" in rendered
+
+
 def test_factory_surfaces_summarization_knobs() -> None:
     """Passes explicit summarization settings through to the middleware."""
     model = _make_model(with_profile_limit=120_000)

libs/deepagents/tests/unit_tests/middleware/test_summarization_middleware.py

@@ -1,7 +1,10 @@
 """Unit tests for `SummarizationMiddleware` with backend offloading."""
 
 import asyncio
+import base64 as _base64
+import hashlib
 import inspect
+import re
 import time
 from collections.abc import Callable, Generator
 from contextlib import contextmanager
@@ -13,10 +16,11 @@ from langchain.agents.middleware.types import ExtendedModelResponse, ModelReques
 from langchain_core.exceptions import ContextOverflowError
 from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage, ToolMessage
 
-from deepagents.backends.protocol import BackendProtocol, EditResult, FileDownloadResponse, ReadResult, WriteResult
+from deepagents.backends.protocol import BackendProtocol, EditResult, FileDownloadResponse, FileUploadResponse, ReadResult, WriteResult
 from deepagents.middleware.summarization import (
     SummarizationMiddleware,
     _token_counter_accepts_tools,
+    _upload_response_error,
 )
 
 if TYPE_CHECKING:
@@ -189,6 +193,17 @@ class MockBackend(BackendProtocol):
             raise RuntimeError(msg)
         return self.edit(path, old_string, new_string, replace_all)
 
+    def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+        """Record binary file uploads."""
+        responses = []
+        for path, _ in files:
+            self.write_calls.append((path, "<binary>"))
+            responses.append(FileUploadResponse(path=path, error=None))
+        return responses
+
+    async def aupload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+        return self.upload_files(files)
+
 
 def make_mock_runtime() -> MagicMock:
     """Create a mock `Runtime`.
@@ -365,6 +380,7 @@ class TestSummarizationMiddlewareInit:
             )
 
         assert middleware._history_path_prefix == "/custom/history"
+        assert middleware._media_prefix == "/custom/history/media"
 
     def test_deprecated_history_path_prefix_overrides_default(self) -> None:
         """Deprecated history_path_prefix takes precedence over the default."""
@@ -415,7 +431,598 @@ class TestOffloadingBasic:
         assert path == "/conversation_history/test-thread-123.md"
 
         assert "## Summarized at" in content
-        assert "Human:" in content or "AI:" in content
+        assert '<message type="human">' in content or '<message type="ai">' in content
+
+    def test_offload_preserves_image_urls(self) -> None:
+        """Image URLs in evicted messages survive the offload (issue #2873).
+
+        The default `get_buffer_string` drops image content; the XML format keeps
+        image URLs, so the archived markdown still references them.
+        """
+        backend = MockBackend()
+        mock_model = make_mock_model()
+
+        middleware = SummarizationMiddleware(
+            model=mock_model,
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        image_url = "https://example.com/diagram.png"
+        messages: list[BaseMessage] = [
+            HumanMessage(
+                content=[
+                    {"type": "text", "text": "Here is the diagram"},
+                    {"type": "image_url", "image_url": {"url": image_url}},
+                ],
+                id="img-msg",
+            ),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        assert len(backend.write_calls) == 1
+        _, content = backend.write_calls[0]
+        assert image_url in content
+
+    def test_offload_rewrites_non_base64_data_url(self) -> None:
+        """Non-base64 inline `data:` media (e.g. a URL-encoded SVG) is offloaded too.
+
+        `get_buffer_string(format="xml")` drops *any* `data:` URL block, not just
+        base64 ones, so a percent-encoded/plaintext `data:` URL must also be
+        decoded (here via percent-decoding) to a file and rewritten to a path
+        reference -- otherwise the inline media silently disappears from the
+        archive (PR #3990 review).
+        """
+        backend = MockBackend()
+        mock_model = make_mock_model()
+
+        middleware = SummarizationMiddleware(
+            model=mock_model,
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        svg_url = "data:image/svg+xml,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%3E%3C%2Fsvg%3E"
+        raw_svg = b'<svg xmlns="http://www.w3.org/2000/svg"></svg>'  # percent-decoded payload
+        expected_key = hashlib.sha256(raw_svg).hexdigest()[:16]
+        expected_path = f"/conversation_history/media/{expected_key}.svg"
+
+        messages: list[BaseMessage] = [
+            HumanMessage(
+                content=[
+                    {"type": "text", "text": "Here is the inline SVG"},
+                    {"type": "image", "url": svg_url},
+                ],
+                id="svg-data-url",
+            ),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        media_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+        assert media_uploads == [(expected_path, "<binary>")]
+
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        assert "Here is the inline SVG" in archive_write[1]
+        # Referenced by path; the raw inline data: URL is gone and nothing failed.
+        assert f'<image url="{expected_path}" />' in archive_write[1]
+        assert svg_url not in archive_write[1]
+        assert 'error="failed_to_offload"' not in archive_write[1]
+
+    def test_offload_rewrites_base64_images(self) -> None:
+        """Base64 image data is decoded to files and referenced by path in the archive (issue #2873).
+
+        `get_buffer_string(format="xml")` drops base64 blocks, so the middleware
+        decodes each one to `/conversation_history/media/{sha256}.{ext}` with
+        `upload_files`, then rewrites the block to `<image url="…" />`. This
+        covers all three base64 shapes recognized by langchain_core.
+        """
+        backend = MockBackend()
+        mock_model = make_mock_model()
+
+        middleware = SummarizationMiddleware(
+            model=mock_model,
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        # 1x1 transparent PNG — minimal valid PNG bytes
+        raw_png = _base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")
+        b64 = _base64.b64encode(raw_png).decode()
+        expected_key = hashlib.sha256(raw_png).hexdigest()[:16]
+        expected_path = f"/conversation_history/media/{expected_key}.png"
+
+        messages: list[BaseMessage] = [
+            # Shape 1: explicit base64 field
+            HumanMessage(
+                content=[
+                    {"type": "text", "text": "explicit base64 field"},
+                    {"type": "image", "base64": b64, "mime_type": "image/png"},
+                ],
+                id="b64-field",
+            ),
+            # Shape 2: top-level data: URL
+            HumanMessage(
+                content=[{"type": "image", "url": f"data:image/png;base64,{b64}"}],
+                id="b64-url",
+            ),
+            # Shape 3: OpenAI-style image_url with data: URL
+            HumanMessage(
+                content=[{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}],
+                id="b64-openai",
+            ),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        # Identical images are deduped — only one upload despite three messages.
+        image_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+        assert len(image_uploads) == 1
+        assert image_uploads[0][0] == expected_path
+
+        # Archive markdown references the image path three times (once per message).
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        assert archive_write[1].count(expected_path) == 3
+        # No raw base64 payload in the archive.
+        assert b64 not in archive_write[1]
+
+    def test_offload_media_uses_deprecated_history_path_prefix(self) -> None:
+        """Media files are written under the final history prefix."""
+        backend = MockBackend()
+        mock_model = make_mock_model()
+
+        with pytest.warns(match="history_path_prefix"):
+            middleware = SummarizationMiddleware(
+                model=mock_model,
+                backend=backend,
+                trigger=("messages", 5),
+                keep=("messages", 2),
+                history_path_prefix="/custom/history",
+            )
+
+        raw_png = _base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")
+        b64 = _base64.b64encode(raw_png).decode()
+        expected_key = hashlib.sha256(raw_png).hexdigest()[:16]
+        expected_path = f"/custom/history/media/{expected_key}.png"
+
+        messages: list[BaseMessage] = [
+            HumanMessage(
+                content=[{"type": "image", "url": f"data:image/png;base64,{b64}"}],
+                id="custom-prefix-image",
+            ),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        image_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/custom/history/media/")]
+        assert len(image_uploads) == 1
+        assert image_uploads[0][0] == expected_path
+
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        assert archive_write[0] == "/custom/history/test-thread-123.md"
+        assert expected_path in archive_write[1]
+
+    def test_offload_per_block_upload_failure(self) -> None:
+        """A failed upload writes a placeholder while other images are still rewritten (issue #2873).
+
+        Per-block failure tracking ensures one failed upload does not discard the
+        entire batch. The successfully uploaded image gets a path reference in the
+        archive; the failed image is replaced with an
+        `<image error="failed_to_offload" />` text placeholder.
+        """
+        raw_a = b"\x89PNG_FAKE_DATA_A"
+        raw_b = b"\x89PNG_FAKE_DATA_B"
+        b64_a = _base64.b64encode(raw_a).decode()
+        b64_b = _base64.b64encode(raw_b).decode()
+        key_a = hashlib.sha256(raw_a).hexdigest()[:16]
+        key_b = hashlib.sha256(raw_b).hexdigest()[:16]
+        expected_path_a = f"/conversation_history/media/{key_a}.png"
+        expected_path_b = f"/conversation_history/media/{key_b}.png"
+
+        class PartialUploadBackend(MockBackend):
+            """Backend that raises for any upload path containing key_b."""
+
+            def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+                responses = []
+                for path, _content in files:
+                    if key_b in path:
+                        msg = f"Simulated upload failure for {path}"
+                        raise RuntimeError(msg)
+                    self.write_calls.append((path, "<binary>"))
+                    responses.append(FileUploadResponse(path=path, error=None))
+                return responses
+
+            async def aupload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+                return self.upload_files(files)
+
+        backend = PartialUploadBackend()
+        middleware = SummarizationMiddleware(
+            model=make_mock_model(),
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        messages: list[BaseMessage] = [
+            HumanMessage(
+                content=[{"type": "image", "base64": b64_a, "mime_type": "image/png"}],
+                id="img-a",
+            ),
+            HumanMessage(
+                content=[{"type": "image", "base64": b64_b, "mime_type": "image/png"}],
+                id="img-b",
+            ),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config(), pytest.warns(UserWarning, match="could not be offloaded"):
+            call_wrap_model_call(middleware, state, runtime)
+
+        # Only image A was uploaded — B's upload raised.
+        image_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+        assert len(image_uploads) == 1
+        assert image_uploads[0][0] == expected_path_a
+
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        # Image A is referenced by path in the archive.
+        assert expected_path_a in archive_write[1]
+        # Image B gets an error placeholder, not a path reference or raw base64.
+        assert expected_path_b not in archive_write[1]
+        assert b64_b not in archive_write[1]
+        assert 'error="failed_to_offload"' in archive_write[1]
+
+    def test_offload_upload_response_error_writes_placeholder(self) -> None:
+        """Upload responses with errors are treated as failed image offloads."""
+        raw_a = b"\x89PNG_RESPONSE_OK"
+        raw_b = b"\x89PNG_RESPONSE_DENIED"
+        b64_a = _base64.b64encode(raw_a).decode()
+        b64_b = _base64.b64encode(raw_b).decode()
+        key_a = hashlib.sha256(raw_a).hexdigest()[:16]
+        key_b = hashlib.sha256(raw_b).hexdigest()[:16]
+        expected_path_a = f"/conversation_history/media/{key_a}.png"
+        expected_path_b = f"/conversation_history/media/{key_b}.png"
+
+        class ResponseErrorBackend(MockBackend):
+            """Backend that reports one image upload failure in the response."""
+
+            def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+                responses = []
+                for path, _content in files:
+                    if key_b in path:
+                        responses.append(FileUploadResponse(path=path, error="permission_denied"))
+                    else:
+                        self.write_calls.append((path, "<binary>"))
+                        responses.append(FileUploadResponse(path=path, error=None))
+                return responses
+
+            async def aupload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+                return self.upload_files(files)
+
+        backend = ResponseErrorBackend()
+        middleware = SummarizationMiddleware(
+            model=make_mock_model(),
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        messages: list[BaseMessage] = [
+            HumanMessage(content=[{"type": "image", "base64": b64_a, "mime_type": "image/png"}], id="img-a"),
+            HumanMessage(content=[{"type": "image", "base64": b64_b, "mime_type": "image/png"}], id="img-b"),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        image_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+        assert image_uploads == [(expected_path_a, "<binary>")]
+
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        assert expected_path_a in archive_write[1]
+        assert expected_path_b not in archive_write[1]
+        assert b64_b not in archive_write[1]
+        assert 'error="failed_to_offload"' in archive_write[1]
+
+    def test_offload_all_uploads_raise_writes_placeholders(self) -> None:
+        """All-raised upload failures still rewrite raw base64 to placeholders."""
+        raw_png = b"\x89PNG_ALL_RAISE"
+        b64 = _base64.b64encode(raw_png).decode()
+
+        class RaisingUploadBackend(MockBackend):
+            """Backend that raises for every image upload."""
+
+            def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+                msg = f"Simulated upload failure for {files[0][0]}"
+                raise RuntimeError(msg)
+
+            async def aupload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+                return self.upload_files(files)
+
+        backend = RaisingUploadBackend()
+        middleware = SummarizationMiddleware(
+            model=make_mock_model(),
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        messages: list[BaseMessage] = [
+            HumanMessage(content=[{"type": "image", "base64": b64, "mime_type": "image/png"}], id="img"),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        assert b64 not in archive_write[1]
+        assert 'error="failed_to_offload"' in archive_write[1]
+
+    def test_offload_rewrites_non_image_media(self) -> None:
+        """Non-image base64 media is offloaded and referenced with its own block type.
+
+        The offload path is media-generic: an audio block is decoded to
+        `/conversation_history/media/{sha256}.{ext}` and rewritten so the XML
+        archive renders an `<audio url="..." />` reference rather than
+        mislabeling it as an image.
+        """
+        backend = MockBackend()
+        middleware = SummarizationMiddleware(
+            model=make_mock_model(),
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        raw_audio = b"ID3_FAKE_MP3_DATA"
+        b64 = _base64.b64encode(raw_audio).decode()
+        expected_key = hashlib.sha256(raw_audio).hexdigest()[:16]
+        expected_path = f"/conversation_history/media/{expected_key}.mp3"
+
+        messages: list[BaseMessage] = [
+            HumanMessage(
+                content=[{"type": "audio", "base64": b64, "mime_type": "audio/mpeg"}],
+                id="audio-msg",
+            ),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        image_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+        assert image_uploads == [(expected_path, "<binary>")]
+
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        # Rendered as an audio reference, not an image, and not raw base64.
+        assert f'<audio url="{expected_path}" />' in archive_write[1]
+        assert b64 not in archive_write[1]
+
+    def test_offload_decode_failure_writes_placeholder(self) -> None:
+        """A detected-but-undecodable base64 block becomes a placeholder, not a silent drop.
+
+        A base64 `data:` URL with an undecodable payload is recognized as media
+        but fails to decode. The block must still surface as an
+        `<image error="failed_to_offload" />` placeholder so the archive records
+        that media was present, and the failure must count toward the user-facing
+        warning -- decode failures are as unrecoverable as upload failures.
+        """
+        backend = MockBackend()
+        middleware = SummarizationMiddleware(
+            model=make_mock_model(),
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        # Detected as a base64 data URL (has the comma + "base64" header), but the
+        # single-character payload is undecodable -> _decode_data_url returns None.
+        malformed = "data:image/png;base64,a"
+        messages: list[BaseMessage] = [
+            HumanMessage(content=[{"type": "image", "url": malformed}], id="bad-img"),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config(), pytest.warns(UserWarning, match="could not be offloaded"):
+            call_wrap_model_call(middleware, state, runtime)
+
+        # Nothing was uploaded (decode failed before any upload).
+        assert not [p for p, _ in backend.write_calls if p.startswith("/conversation_history/media/")]
+
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        assert 'error="failed_to_offload"' in archive_write[1]
+        assert malformed not in archive_write[1]
+
+    def test_offload_non_av_media_uses_file_reference(self) -> None:
+        """Non-(image/audio/video) media falls back to an XML-escaped `<file>` reference.
+
+        A PDF has no typed XML block, so `_media_reference_block` emits a text
+        block `<file url="..." />`. The XML renderer escapes text blocks, so the
+        archive contains the escaped form `&lt;file url="..." /&gt;` -- the
+        reference is preserved (not dropped) but rendered as literal text.
+        """
+        backend = MockBackend()
+        middleware = SummarizationMiddleware(
+            model=make_mock_model(),
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        raw_pdf = b"%PDF-1.4 fake pdf bytes"
+        b64 = _base64.b64encode(raw_pdf).decode()
+        expected_key = hashlib.sha256(raw_pdf).hexdigest()[:16]
+        expected_path = f"/conversation_history/media/{expected_key}.pdf"
+
+        messages: list[BaseMessage] = [
+            HumanMessage(
+                content=[{"type": "file", "base64": b64, "mime_type": "application/pdf"}],
+                id="pdf-msg",
+            ),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        media_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+        assert media_uploads == [(expected_path, "<binary>")]
+
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        # Text-block fallback renders XML-escaped, and raw base64 is absent.
+        assert f'&lt;file url="{expected_path}" /&gt;' in archive_write[1]
+        assert b64 not in archive_write[1]
+
+    def test_offload_no_inline_media_returns_messages_unchanged(self) -> None:
+        """With no inline media, offload returns the original list and uploads nothing.
+
+        Guards the `saw_inline_media` short-circuit: text-only messages must skip the
+        copy/rewrite path entirely (identity return) so no media files are written.
+        """
+        backend = MockBackend()
+        middleware = SummarizationMiddleware(
+            model=make_mock_model(),
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        messages = make_conversation_messages(num_old=5, num_recent=2)
+        result, failed = middleware._offload_inline_media(backend, messages)
+
+        assert result is messages  # identity: no copy when there is no inline media
+        assert failed == 0
+        assert not [p for p, _ in backend.write_calls if p.startswith("/conversation_history/media/")]
+
+    def test_offload_base64_alongside_non_standard_block(self) -> None:
+        """Base64 media offloads cleanly even when a non-standard block shares the message.
+
+        A provider-specific `tool_use` block normalizes to a `non_standard`
+        content block when rewritten. This must not corrupt the archive: the
+        image is still referenced by path and no raw base64 leaks, regardless of
+        the sibling block.
+        """
+        backend = MockBackend()
+        middleware = SummarizationMiddleware(
+            model=make_mock_model(),
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        raw_png = b"\x89PNG_WITH_TOOL_USE"
+        b64 = _base64.b64encode(raw_png).decode()
+        expected_key = hashlib.sha256(raw_png).hexdigest()[:16]
+        expected_path = f"/conversation_history/media/{expected_key}.png"
+
+        messages: list[BaseMessage] = [
+            AIMessage(
+                content=[
+                    {"type": "text", "text": "calling a tool with an image"},
+                    {"type": "tool_use", "id": "t1", "name": "lookup", "input": {"q": 1}},
+                    {"type": "image", "base64": b64, "mime_type": "image/png"},
+                ],
+                id="ai-tool-img",
+            ),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        media_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+        assert media_uploads == [(expected_path, "<binary>")]
+
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        assert expected_path in archive_write[1]
+        assert b64 not in archive_write[1]
+
+    def test_upload_runs_once_before_offload_and_summary(self) -> None:
+        """Image upload runs once and the result is shared by offload and summary.
+
+        `_create_summary` uses prefix format internally, so image blocks are
+        stripped from the summary prompt regardless. This test verifies that raw
+        base64 never reaches the summary model prompt, the archive gets the
+        uploaded image path, and each unique image is uploaded only once.
+        """
+        raw_png = _base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")
+        b64 = _base64.b64encode(raw_png).decode()
+        expected_key = hashlib.sha256(raw_png).hexdigest()[:16]
+        expected_path = f"/conversation_history/media/{expected_key}.png"
+
+        backend = MockBackend()
+        mock_model = make_mock_model()
+        middleware = SummarizationMiddleware(
+            model=mock_model,
+            backend=backend,
+            trigger=("messages", 5),
+            keep=("messages", 2),
+        )
+
+        messages: list[BaseMessage] = [
+            HumanMessage(
+                content=[
+                    {"type": "text", "text": "Here is an image:"},
+                    {"type": "image", "base64": b64, "mime_type": "image/png"},
+                ],
+                id="b64-msg",
+            ),
+            *make_conversation_messages(num_old=5, num_recent=2),
+        ]
+        state = cast("AgentState[Any]", {"messages": messages})
+        runtime = make_mock_runtime()
+
+        with mock_get_config():
+            call_wrap_model_call(middleware, state, runtime)
+
+        # upload_files is called exactly once even though both offload and
+        # summary consume the result.
+        image_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+        assert len(image_uploads) == 1
+        assert image_uploads[0][0] == expected_path
+
+        # Archive has the uploaded image path.
+        archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+        assert expected_path in archive_write[1]
+
+        # Raw base64 never reaches the summary model prompt.
+        invoke_prompt: str = mock_model.invoke.call_args[0][0]
+        assert b64 not in invoke_prompt
 
     def test_offload_appends_to_existing_content(self) -> None:
         """Test that second summarization appends to existing file."""
@@ -1151,8 +1758,8 @@ class TestMarkdownFormatting:
         # Verify the offloaded content is markdown formatted
         _, content = backend.write_calls[0]
 
-        # Should contain human-readable message prefixes
-        assert "Human:" in content or "AI:" in content
+        # Should contain XML-tagged messages (get_buffer_string format="xml")
+        assert '<message type="human">' in content or '<message type="ai">' in content
         # Should contain the actual message content
         assert "User message" in content
 
@@ -2193,8 +2800,8 @@ def test_chained_summarization_cutoff_index() -> None:
         return [HumanMessage(content=f"S{i}", id=f"s{i}") if i % 2 == 0 else AIMessage(content=f"S{i}", id=f"s{i}") for i in range(n)]
 
     def offloaded_labels(write_call_content: str) -> list[str]:
-        """Extract S-labels from backend write content (e.g. "Human: S0" -> "S0")."""
-        return [word for word in write_call_content.split() if word.startswith("S") and word[1:].isdigit()]
+        """Extract S-labels from backend write content (e.g. `<message ...>S0</message>` -> "S0")."""
+        return re.findall(r"S\d+", write_call_content)
 
     # --- Round 1: first summarization, no previous event ---
     state = cast("AgentState[Any]", {"messages": make_state_messages(8)})
@@ -2847,3 +3454,176 @@ class TestTokenCounterToolsProbe:
         messages = make_conversation_messages()
         tools = [{"type": "function", "function": {"name": "noop"}}]
         assert middleware._count_tokens(messages, None, tools) == 7
+
+
+@pytest.mark.anyio
+async def test_async_offloads_base64_images() -> None:
+    """Async path offloads base64 images via aupload_files before gather (issue #2873).
+
+    `awrap_model_call` must upload image data before
+    `asyncio.gather(offload, summary)` so both coroutines see path references,
+    not raw base64. This mirrors `test_offload_rewrites_base64_images` but
+    exercises the async code path end-to-end.
+    """
+    raw_png = _base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")
+    b64 = _base64.b64encode(raw_png).decode()
+    expected_key = hashlib.sha256(raw_png).hexdigest()[:16]
+    expected_path = f"/conversation_history/media/{expected_key}.png"
+
+    backend = MockBackend()
+    mock_model = make_mock_model()
+    mock_model.ainvoke = MagicMock(return_value=MagicMock(text="Async summary"))
+
+    middleware = SummarizationMiddleware(
+        model=mock_model,
+        backend=backend,
+        trigger=("messages", 5),
+        keep=("messages", 2),
+    )
+
+    messages: list[BaseMessage] = [
+        # Shape 1: explicit base64 field
+        HumanMessage(
+            content=[{"type": "image", "base64": b64, "mime_type": "image/png"}],
+            id="b64-field",
+        ),
+        # Shape 2: top-level data: URL
+        HumanMessage(
+            content=[{"type": "image", "url": f"data:image/png;base64,{b64}"}],
+            id="b64-url",
+        ),
+        # Shape 3: OpenAI-style image_url
+        HumanMessage(
+            content=[{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}],
+            id="b64-openai",
+        ),
+        *make_conversation_messages(num_old=5, num_recent=2),
+    ]
+    state = cast("AgentState[Any]", {"messages": messages})
+    runtime = make_mock_runtime()
+
+    with mock_get_config():
+        await call_awrap_model_call(middleware, state, runtime)
+
+    # aupload_files delegates to upload_files in MockBackend, so write_calls captures it.
+    # Identical images are deduped — only one upload despite three messages.
+    image_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+    assert len(image_uploads) == 1
+    assert image_uploads[0][0] == expected_path
+
+    # Archive markdown references the offloaded path three times.
+    archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+    assert archive_write[1].count(expected_path) == 3
+    assert b64 not in archive_write[1]
+
+    # Raw base64 never reaches the async summary model prompt (symmetric with the
+    # sync `test_upload_runs_once_before_offload_and_summary` guard).
+    assert b64 not in str(mock_model.ainvoke.call_args)
+
+
+@pytest.mark.anyio
+async def test_async_upload_response_error_writes_placeholder() -> None:
+    """Async upload responses with errors are treated as failed image offloads."""
+    raw_a = b"\x89PNG_ASYNC_RESPONSE_OK"
+    raw_b = b"\x89PNG_ASYNC_RESPONSE_DENIED"
+    b64_a = _base64.b64encode(raw_a).decode()
+    b64_b = _base64.b64encode(raw_b).decode()
+    key_a = hashlib.sha256(raw_a).hexdigest()[:16]
+    key_b = hashlib.sha256(raw_b).hexdigest()[:16]
+    expected_path_a = f"/conversation_history/media/{key_a}.png"
+    expected_path_b = f"/conversation_history/media/{key_b}.png"
+
+    class ResponseErrorBackend(MockBackend):
+        """Backend that reports one async image upload failure in the response."""
+
+        async def aupload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+            responses = []
+            for path, _content in files:
+                if key_b in path:
+                    responses.append(FileUploadResponse(path=path, error="permission_denied"))
+                else:
+                    self.write_calls.append((path, "<binary>"))
+                    responses.append(FileUploadResponse(path=path, error=None))
+            return responses
+
+    backend = ResponseErrorBackend()
+    mock_model = make_mock_model()
+    mock_model.ainvoke = MagicMock(return_value=MagicMock(text="Async summary"))
+    middleware = SummarizationMiddleware(
+        model=mock_model,
+        backend=backend,
+        trigger=("messages", 5),
+        keep=("messages", 2),
+    )
+
+    messages: list[BaseMessage] = [
+        HumanMessage(content=[{"type": "image", "base64": b64_a, "mime_type": "image/png"}], id="img-a"),
+        HumanMessage(content=[{"type": "image", "base64": b64_b, "mime_type": "image/png"}], id="img-b"),
+        *make_conversation_messages(num_old=5, num_recent=2),
+    ]
+    state = cast("AgentState[Any]", {"messages": messages})
+    runtime = make_mock_runtime()
+
+    with mock_get_config():
+        await call_awrap_model_call(middleware, state, runtime)
+
+    image_uploads = [(p, c) for p, c in backend.write_calls if p.startswith("/conversation_history/media/")]
+    assert image_uploads == [(expected_path_a, "<binary>")]
+
+    archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+    assert expected_path_a in archive_write[1]
+    assert expected_path_b not in archive_write[1]
+    assert b64_b not in archive_write[1]
+    assert 'error="failed_to_offload"' in archive_write[1]
+
+
+@pytest.mark.anyio
+async def test_async_all_uploads_raise_writes_placeholders() -> None:
+    """Async all-raised upload failures still rewrite raw base64 to placeholders."""
+    raw_png = b"\x89PNG_ASYNC_ALL_RAISE"
+    b64 = _base64.b64encode(raw_png).decode()
+
+    class RaisingUploadBackend(MockBackend):
+        """Backend that raises for every async image upload."""
+
+        async def aupload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
+            msg = f"Simulated upload failure for {files[0][0]}"
+            raise RuntimeError(msg)
+
+    backend = RaisingUploadBackend()
+    mock_model = make_mock_model()
+    mock_model.ainvoke = MagicMock(return_value=MagicMock(text="Async summary"))
+    middleware = SummarizationMiddleware(
+        model=mock_model,
+        backend=backend,
+        trigger=("messages", 5),
+        keep=("messages", 2),
+    )
+
+    messages: list[BaseMessage] = [
+        HumanMessage(content=[{"type": "image", "base64": b64, "mime_type": "image/png"}], id="img"),
+        *make_conversation_messages(num_old=5, num_recent=2),
+    ]
+    state = cast("AgentState[Any]", {"messages": messages})
+    runtime = make_mock_runtime()
+
+    with mock_get_config(), pytest.warns(UserWarning, match="could not be offloaded"):
+        await call_awrap_model_call(middleware, state, runtime)
+
+    archive_write = next((p, c) for p, c in backend.write_calls if p.endswith(".md"))
+    assert b64 not in archive_write[1]
+    assert 'error="failed_to_offload"' in archive_write[1]
+
+
+def test_upload_response_error_classifies_batch_result() -> None:
+    """`_upload_response_error` distinguishes empty, error, and success responses.
+
+    Covers the defensive empty-list branch (a backend that returns no response
+    for a requested upload), which the integration tests don't exercise.
+    """
+    # Empty list: backend returned no response for the single requested file.
+    assert _upload_response_error([]) == "missing_upload_response"
+    # Populated error is surfaced as a string.
+    assert _upload_response_error([FileUploadResponse(path="/m.png", error="permission_denied")]) == "permission_denied"
+    # Success (error is None) returns None.
+    assert _upload_response_error([FileUploadResponse(path="/m.png", error=None)]) is None

libs/deepagents/uv.lock

@@ -449,7 +449,7 @@ test = [
 
 [package.metadata]
 requires-dist = [
-    { name = "langchain", specifier = ">=1.3.9,<2.0.0" },
+    { name = "langchain", specifier = ">=1.3.10,<2.0.0" },
     { name = "langchain-anthropic", specifier = ">=1.4.6,<2.0.0" },
     { name = "langchain-core", specifier = ">=1.4.7,<2.0.0" },
     { name = "langchain-google-genai", specifier = ">=4.2.5,<5.0.0" },
@@ -812,16 +812,16 @@ wheels = [
 
 [[package]]
 name = "langchain"
-version = "1.3.9"
+version = "1.3.10"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "langchain-core" },
     { name = "langgraph" },
     { name = "pydantic" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/56/7c/651d0dc4913a7a892156c03dd343b99cfe19ee729e6911ab1f4fe7567b8b/langchain-1.3.9.tar.gz", hash = "sha256:9b14ef0db9ef314299ded858b22ca2a40b8f1b05c8c9cb6b82d53a53075fef00", size = 631514, upload-time = "2026-06-12T16:53:27.083Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/3b/f6/e351d85c7828b9b90c5729de66170457c882c754efef0712904cfcd3192d/langchain-1.3.10.tar.gz", hash = "sha256:fd6ac9da86c479e4ff376e772d9e17a9232bd3113e9f2ddcb70cdc4bf7afc119", size = 632522, upload-time = "2026-06-18T19:43:00.86Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/b7/55/3481619d21b9bdfbfda8680fba5cfc6cfe926789b8eaaad95353078cfa20/langchain-1.3.9-py3-none-any.whl", hash = "sha256:4af49ad1095799e4408b489fb79d4b8b49292453618b202d8a697fca59bb6871", size = 132873, upload-time = "2026-06-12T16:53:25.489Z" },
+    { url = "https://files.pythonhosted.org/packages/59/f6/a682e68d004a2e23cae6c5c42e3c0d071bc0e7768167bd12277992f096f9/langchain-1.3.10-py3-none-any.whl", hash = "sha256:5da67f21aa56119744ad51b3e46ffac570c88f4fae0876e3b1c6a1c4bc0e344e", size = 133038, upload-time = "2026-06-18T19:42:58.918Z" },
 ]
 
 [[package]]