fix(code): strip media placeholders from model-facing message text (#4462)

- Display-only `[image N]`/`[video N]` placeholders no longer appear in
the model-facing message text or LangSmith trace; only the structured
media block is sent.
- Manually typing `[image N]`/`[video N]` text that looks like a
placeholder no longer triggers atomic deletion on backspace — it edits
character by character like ordinary text.

---

When a user attaches an image or video in the Deep Agents Code TUI, a
display-only `[image N]` or `[video N]` placeholder appears in the input
area. The placeholder is purely visual — the actual media travels as a
structured content block alongside the text. Two bugs caused that
display convention to leak into model-facing behavior:

1. **Tracing/model leak:** `create_multimodal_content` copied the raw
input text verbatim into the message text block, so the canonical
`HumanMessage` sent to the model and serialized to LangSmith contained
the display-only `[image 1]` as if the user had typed it. It now strips
only the exact placeholder tokens bound to media actually attached to
that message (via `strip_media_placeholders`), leaving look-alike
literal text intact and requiring no escaping mechanism.

2. **Editing false positive:** `ChatTextArea._find_placeholder_span`
treated any text matching the placeholder regex as an atomic token, so
manually typing `[image 2]` meant a single backspace deleted the whole
token. It now validates each regex match against real state — the media
tracker for image/video placeholders and `_pasted_contents` for paste
placeholders — so only tokens bound to real attachments delete
atomically while typed look-alikes edit character by character.

Disambiguating a display token from a user-typed duplicate of the same
string (e.g. attaching an image that gets `[image 1]`, then the user
also types `[image 1]` in the same message) requires tracking the exact
character span of each display token, not just its text. `ImageData` and
`VideoData` now carry a `placeholder_span` that the tracker updates
through edits via `sync_to_text` (using a diff-based span mapper) and
re-maps at submit time through text transforms — whitespace trimming,
paste expansion, prefix prepending — that shift offsets. `add_media`
also skips placeholder IDs already present in the draft so a literal
`[image 1]` in the text doesn't cause a newly attached image to reuse
the same ID.


Made by [Open
SWE](https://openswe.vercel.app/agents/0fc423a2-7100-74d7-2a83-4ea855c9d4bb)

---------

Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
This commit is contained in:
Mason Daugherty
2026-07-06 03:21:35 -04:00
committed by GitHub
parent 9d4d7ba4eb
commit aa0ae36b00
5 changed files with 978 additions and 21 deletions
+198 -12
View File
@@ -4,6 +4,7 @@ import logging
import re
import shlex
from dataclasses import dataclass
from difflib import SequenceMatcher
from pathlib import Path
from typing import Literal
from urllib.parse import unquote, urlparse
@@ -118,49 +119,65 @@ class MediaTracker:
self.next_image_id: int = 1
self.next_video_id: int = 1
def add_media(self, data: ImageData | VideoData, kind: MediaKind) -> str:
def add_media(
self,
data: ImageData | VideoData,
kind: MediaKind,
*,
existing_text: str = "",
) -> str:
"""Add a media item and return its placeholder text.
Args:
data: The image or video data to track.
kind: Media type key.
existing_text: Current draft text. Placeholder IDs already present
here are skipped so literal user text is not bound to new media.
Returns:
Placeholder string like "[image 1]" or "[video 1]".
"""
if kind == "image":
while f"[image {self.next_image_id}]" in existing_text:
self.next_image_id += 1
placeholder = f"[image {self.next_image_id}]"
data.placeholder = placeholder
self.images.append(data) # ty: ignore[invalid-argument-type]
self.next_image_id += 1
else:
while f"[video {self.next_video_id}]" in existing_text:
self.next_video_id += 1
placeholder = f"[video {self.next_video_id}]"
data.placeholder = placeholder
self.videos.append(data) # ty: ignore[invalid-argument-type]
self.next_video_id += 1
return placeholder
def add_image(self, image_data: ImageData) -> str:
def add_image(self, image_data: ImageData, *, existing_text: str = "") -> str:
"""Add an image and return its placeholder text.
Args:
image_data: The image data to track.
existing_text: Current draft text. Placeholder IDs already present
here are skipped so literal user text is not bound to new media.
Returns:
Placeholder string like "[image 1]".
"""
return self.add_media(image_data, "image")
return self.add_media(image_data, "image", existing_text=existing_text)
def add_video(self, video_data: VideoData) -> str:
def add_video(self, video_data: VideoData, *, existing_text: str = "") -> str:
"""Add a video and return its placeholder text.
Args:
video_data: The video data to track.
existing_text: Current draft text. Placeholder IDs already present
here are skipped so literal user text is not bound to new media.
Returns:
Placeholder string like "[video 1]".
"""
return self.add_media(video_data, "video")
return self.add_media(video_data, "video", existing_text=existing_text)
def get_images(self) -> list[ImageData]:
"""Get all tracked images.
@@ -185,28 +202,82 @@ class MediaTracker:
self.next_image_id = 1
self.next_video_id = 1
def sync_to_text(self, text: str) -> None:
def sync_to_text(
self,
text: str,
*,
previous_text: str | None = None,
cursor_offset: int | None = None,
) -> None:
"""Retain only media still referenced by placeholders in current text.
Args:
text: Current input text shown to the user.
previous_text: Previous input text, used to keep tracking the same
placeholder occurrence when duplicate literal tokens are added.
cursor_offset: Current cursor offset, used to disambiguate whole-paste
edits that create duplicate placeholder tokens.
"""
img_found = self._sync_kind_images(text)
vid_found = self._sync_kind_videos(text)
img_found = self._sync_kind_images(
text, previous_text=previous_text, cursor_offset=cursor_offset
)
vid_found = self._sync_kind_videos(
text, previous_text=previous_text, cursor_offset=cursor_offset
)
if not img_found and not vid_found:
self.clear()
def _sync_kind_images(self, text: str) -> bool:
def remap_spans_to_text(self, text: str, *, previous_text: str) -> None:
"""Re-map tracked placeholder spans onto a transformed copy of the text.
Submission rewrites the draft before the display placeholders are
stripped from the model-facing text: whitespace is trimmed, collapsed
pastes expand back to full content, dropped paths become placeholders,
and a mode prefix may be prepended. Every one of those shifts character
offsets, so a `placeholder_span` captured against the draft would be
stale by the time `strip_media_placeholders` consumes it — silently
stripping the wrong occurrence when a user-typed duplicate is present.
Re-mapping each span through the same before/after diff keeps it pointing
at its own display token in `text`. Spans that cannot be cleanly mapped
become `None`, degrading to the token-count fallback rather than a wrong
strip.
Args:
text: The transformed text the spans must line up with.
previous_text: The draft text the current spans were captured
against.
"""
for item in (*self.images, *self.videos):
if item.placeholder_span is None:
continue
item.placeholder_span = self._map_placeholder_span(
item.placeholder_span, previous_text, text
)
def _sync_kind_images(
self,
text: str,
*,
previous_text: str | None = None,
cursor_offset: int | None = None,
) -> bool:
"""Sync image list to surviving placeholders in text.
Args:
text: Current input text.
previous_text: Previous input text, used to map existing spans.
cursor_offset: Current cursor offset for duplicate disambiguation.
Returns:
Whether any image placeholders were found.
"""
placeholders = {m.group(0) for m in IMAGE_PLACEHOLDER_PATTERN.finditer(text)}
matches = list(IMAGE_PLACEHOLDER_PATTERN.finditer(text))
placeholders = {m.group(0) for m in matches}
self.images = [img for img in self.images if img.placeholder in placeholders]
self._update_placeholder_spans(
self.images, matches, text, previous_text, cursor_offset
)
if not self.images:
self.next_image_id = 1
else:
@@ -215,17 +286,29 @@ class MediaTracker:
)
return bool(placeholders)
def _sync_kind_videos(self, text: str) -> bool:
def _sync_kind_videos(
self,
text: str,
*,
previous_text: str | None = None,
cursor_offset: int | None = None,
) -> bool:
"""Sync video list to surviving placeholders in text.
Args:
text: Current input text.
previous_text: Previous input text, used to map existing spans.
cursor_offset: Current cursor offset for duplicate disambiguation.
Returns:
Whether any video placeholders were found.
"""
placeholders = {m.group(0) for m in VIDEO_PLACEHOLDER_PATTERN.finditer(text)}
matches = list(VIDEO_PLACEHOLDER_PATTERN.finditer(text))
placeholders = {m.group(0) for m in matches}
self.videos = [vid for vid in self.videos if vid.placeholder in placeholders]
self._update_placeholder_spans(
self.videos, matches, text, previous_text, cursor_offset
)
if not self.videos:
self.next_video_id = 1
else:
@@ -234,6 +317,109 @@ class MediaTracker:
)
return bool(placeholders)
def _update_placeholder_spans(
self,
items: list[ImageData] | list[VideoData],
matches: list[re.Match[str]],
text: str,
previous_text: str | None,
cursor_offset: int | None,
) -> None:
"""Refresh tracked placeholder spans for surviving media items.
Args:
items: Surviving tracked media items.
matches: Placeholder regex matches in the current text.
text: Current input text.
previous_text: Previous input text, used to map existing spans.
cursor_offset: Current cursor offset for duplicate disambiguation.
"""
spans_by_token: dict[str, list[tuple[int, int]]] = {}
for match in matches:
spans_by_token.setdefault(match.group(0), []).append(match.span())
for item in items:
spans = spans_by_token.get(item.placeholder, [])
cursor_span = self._placeholder_span_after_cursor(spans, cursor_offset)
mapped = self._map_placeholder_span(
item.placeholder_span, previous_text, text
)
had_duplicate = (
previous_text is not None and previous_text.count(item.placeholder) > 1
)
if had_duplicate and mapped is not None and mapped in spans:
item.placeholder_span = mapped
elif cursor_span is not None and (
item.placeholder_span is None or mapped != item.placeholder_span
):
item.placeholder_span = cursor_span
elif len(spans) == 1:
item.placeholder_span = spans[0]
elif mapped is not None and mapped in spans:
item.placeholder_span = mapped
elif item.placeholder_span not in spans:
item.placeholder_span = None
@staticmethod
def _placeholder_span_after_cursor(
spans: list[tuple[int, int]], cursor_offset: int | None
) -> tuple[int, int] | None:
"""Return the first duplicate placeholder span at or after the cursor.
Only meaningful when the token is duplicated (`len(spans) > 1`); returns
`None` otherwise, or when no span starts at/after the cursor.
Args:
spans: Placeholder spans for one token in current text.
cursor_offset: Current cursor offset, or `None` when unknown.
"""
if cursor_offset is None or len(spans) <= 1:
return None
for span in spans:
start, _end = span
if start >= cursor_offset:
return span
return None
@staticmethod
def _map_placeholder_span(
span: tuple[int, int] | None,
previous_text: str | None,
text: str,
) -> tuple[int, int] | None:
"""Map a placeholder span from the previous text into current text.
Uses a `SequenceMatcher` diff so the span survives edits elsewhere in the
text: it returns the shifted span only when its whole range falls inside
an unchanged (`equal`) block, and `None` when the token's own characters
were edited or the span cannot be located.
Args:
span: The placeholder span in `previous_text`, or `None`.
previous_text: Text the span was captured against, or `None`.
text: Text to map the span into.
Returns:
The mapped span when the same placeholder occurrence survives,
otherwise `None`.
"""
if span is None or previous_text is None:
return None
start, end = span
if not (0 <= start < end <= len(previous_text)):
return None
matcher = SequenceMatcher(a=previous_text, b=text, autojunk=False)
for tag, old_start, old_end, new_start, _new_end in matcher.get_opcodes():
if tag == "equal" and old_start <= start and end <= old_end:
offset = new_start - old_start
return start + offset, end + offset
if old_start <= start and end <= old_end:
return None
if old_start > end:
break
return None
@staticmethod
def _max_placeholder_id(
items: list[ImageData] | list[VideoData],
+137 -3
View File
@@ -5,12 +5,15 @@ import io
import logging
import os
import pathlib
import re
import shutil
# S404: subprocess needed for clipboard access via pngpaste/osascript
import subprocess # noqa: S404
import sys
import tempfile
from collections import Counter
from collections.abc import Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
@@ -50,6 +53,118 @@ MAX_MEDIA_BYTES: int = 20 * 1024 * 1024
"""Maximum media file size (20 MB). Keeps base64 payload under ~27 MB."""
def strip_media_placeholders(
text: str,
placeholders: Iterable[str],
*,
placeholder_spans: Iterable[tuple[int, int]] | None = None,
) -> str:
"""Remove display-only media placeholders from user text.
Placeholders like `[image 1]` are inserted into the terminal input purely for
display; the actual media travels as structured content blocks. They must not
leak into the canonical model-facing message or LangSmith trace as if the user
typed them.
When available, tracked placeholder spans identify the exact display tokens to
strip so user-authored literal duplicates with the same token are preserved.
The token fallback removes one matching occurrence per tracked media item for
callers that only have placeholder text.
Args:
text: Raw user text that may contain media placeholders.
placeholders: Exact placeholder tokens for the media actually attached to
this message (e.g. ``["[image 1]", "[video 1]"]``).
placeholder_spans: Exact `(start, end)` spans for tracked display tokens
in `text`, when known.
Returns:
Text with the given media placeholders removed and surrounding whitespace
tidied. Newlines are preserved so multi-line prompts keep their structure.
Returns an empty string when only whitespace remains after removal, so
callers can treat a placeholder-only message as having no text block.
"""
tokens = [p for p in placeholders if p]
if not tokens:
return text
valid_spans = _valid_placeholder_spans(text, tokens, placeholder_spans)
spans = [(start, end) for start, end, _token in valid_spans]
counts = Counter(tokens)
for _start, _end, token in valid_spans:
counts[token] -= 1
for token, count in counts.items():
if count <= 0:
continue
pattern = re.compile(r"[ \t]*" + re.escape(token))
matches = list(pattern.finditer(text))
if len(matches) > count:
# No (or too few) tracked spans, yet the token appears more times
# than we intend to strip: we can't tell the display token from a
# user-typed literal and fall back to removing the leading
# occurrence(s). That guess can strip a literal and leave the real
# display token behind, so leave a breadcrumb (never the text).
logger.debug(
"Ambiguous media placeholder strip: removing %d of %d "
"occurrences of %r with no tracked span to disambiguate",
count,
len(matches),
token,
)
for index, match in enumerate(matches):
if index >= count:
break
spans.append(match.span())
# Only strip spaces/tabs (not newlines) so code indentation on lines after a
# removed placeholder is preserved. A full .strip() would collapse
# "[image 1]\n def foo():" to "def foo():", losing the leading indent.
cleaned = text
for start, end in sorted(spans, reverse=True):
cleaned = cleaned[:start] + cleaned[end:]
cleaned = cleaned.strip(" \t")
return cleaned if cleaned.strip() else ""
def _valid_placeholder_spans(
text: str,
tokens: list[str],
spans: Iterable[tuple[int, int]] | None,
) -> list[tuple[int, int, str]]:
"""Return valid display placeholder spans expanded over adjacent padding.
A span is kept only when it is in range and its slice equals one of the
bound tokens, so a stale span (e.g. left by an offset shift) is dropped
rather than used to delete arbitrary text; the caller then falls back to
token matching for that item.
Args:
text: Text the spans index into.
tokens: Bound placeholder tokens for the attached media.
spans: Candidate `(start, end)` display-token spans, or `None`.
Returns:
`(start, end, token)` triples with `start` expanded left over any
adjacent spaces/tabs, for spans that validate against `text`.
"""
if spans is None:
return []
token_set = set(tokens)
valid: list[tuple[int, int, str]] = []
for start, end in spans:
if not (0 <= start < end <= len(text)):
continue
token = text[start:end]
if token not in token_set:
continue
expanded_start = start
while expanded_start > 0 and text[expanded_start - 1] in " \t":
expanded_start -= 1
valid.append((expanded_start, end, token))
return valid
def _get_executable(name: str) -> str | None:
"""Get full path to an executable using shutil.which().
@@ -69,6 +184,7 @@ class ImageData:
base64_data: str
format: str # "png", "jpeg", etc.
placeholder: str # Display text like "[image 1]"
placeholder_span: tuple[int, int] | None = None
def to_message_content(self) -> dict:
"""Convert to LangChain message content format.
@@ -89,6 +205,7 @@ class VideoData:
base64_data: str
format: str # "mp4", "quicktime", etc.
placeholder: str # Display text like "[video 1]"
placeholder_span: tuple[int, int] | None = None
def to_message_content(self) -> "VideoContentBlock":
"""Convert to LangChain `VideoContentBlock` format.
@@ -464,9 +581,26 @@ def create_multimodal_content(
"""
content_blocks = []
# Add text block
if text.strip():
content_blocks.append({"type": "text", "text": text})
# Add text block. Strip only the display-only placeholders bound to the media
# actually attached here (e.g. "[image 1]") so the canonical/model-facing text
# never contains fake user-authored placeholder text. When a span is known,
# text that merely resembles the schema is preserved exactly; without a span
# `strip_media_placeholders` falls back to removing one occurrence per item,
# which can catch a look-alike literal. The media itself is carried by the
# structured blocks below.
#
# `placeholders` and `spans` are passed as parallel-but-unzipped lists on
# purpose: `strip_media_placeholders` recovers each span's token from the
# text slice, and each tracked media item has a unique token, so it never
# needs index alignment between the two.
media = [*images, *(videos or [])]
placeholders = [item.placeholder for item in media]
spans = [
item.placeholder_span for item in media if item.placeholder_span is not None
]
clean_text = strip_media_placeholders(text, placeholders, placeholder_spans=spans)
if clean_text:
content_blocks.append({"type": "text", "text": clean_text})
# Add image blocks
content_blocks.extend(image.to_message_content() for image in images)
@@ -1329,6 +1329,33 @@ class ChatTextArea(TextArea):
self.move_cursor(start_location)
return True
def _bound_media_placeholders(self) -> set[str]:
"""Return placeholder tokens bound to currently tracked media.
Returns:
The set of `[image N]`/`[video N]` tokens for media the tracker is
actually holding. Empty when there is no owner/tracker.
"""
owner = self._chat_input_owner
tracker = owner._image_tracker if owner is not None else None
if tracker is None:
return set()
placeholders = {img.placeholder for img in tracker.images}
placeholders.update(video.placeholder for video in tracker.videos)
return placeholders
def _bound_paste_ids(self) -> set[int]:
"""Return paste ids that have backing content in the owner.
Returns:
The set of paste ids present in `ChatInput._pasted_contents`. Empty
when there is no owner.
"""
owner = self._chat_input_owner
if owner is None:
return set()
return set(owner._pasted_contents)
def _find_placeholder_span(
self, cursor_offset: int, *, backwards: bool
) -> tuple[int, int] | None:
@@ -1340,6 +1367,12 @@ class ChatTextArea(TextArea):
here so an undo can restore the token with its content (it is cleared
only at submit).
Only tokens bound to real attachments are treated as atomic: image/video
placeholders must correspond to a tracked media item and paste
placeholders to an entry in `ChatInput._pasted_contents`. Placeholder-
shaped text the user typed by hand (e.g. literally typing ``[image 2]``)
is left as ordinary text and edits character by character.
Args:
cursor_offset: Character offset of the cursor from the start of text.
backwards: Whether the delete action is backwards (backspace) or
@@ -1347,15 +1380,23 @@ class ChatTextArea(TextArea):
Returns:
The `(start, end)` character span of the placeholder to delete, or
`None` when the cursor is not adjacent to a placeholder token.
`None` when the cursor is not adjacent to a bound placeholder
token.
"""
text = self.text
media_placeholders = self._bound_media_placeholders()
pasted_ids = self._bound_paste_ids()
for pattern in (
IMAGE_PLACEHOLDER_PATTERN,
VIDEO_PLACEHOLDER_PATTERN,
PASTE_PLACEHOLDER_PATTERN,
):
for match in pattern.finditer(text):
if pattern is PASTE_PLACEHOLDER_PATTERN:
if int(match.group(1)) not in pasted_ids:
continue
elif match.group(0) not in media_placeholders:
continue
start, end = match.span()
if backwards:
# Cursor is inside token or right after a trailing space inserted
@@ -1927,8 +1968,11 @@ class ChatInput(Vertical):
# preserves replacement edits where selected text is replaced by a path
# of similar length.
should_check_path_payload = self._should_check_path_payload(text)
previous_text = self._prev_text
self._sync_media_tracker_to_text(
text, previous_text=previous_text, cursor_offset=self._get_cursor_offset()
)
self._prev_text = text
self._sync_media_tracker_to_text(text)
# History handlers explicitly decide mode and stripped display text.
# Skip mode detection here so recalled entries don't inherit stale mode.
@@ -2328,6 +2372,16 @@ class ChatInput(Vertical):
if prefix and not value.startswith(prefix):
value = prefix + value
# Placeholder spans were captured against the raw draft; the transforms
# above (whitespace strip, paste expansion, path substitution, prefix)
# shifted offsets. Re-map spans onto the final submitted text so the
# adapter strips the correct display token from the model-facing message
# instead of a same-looking literal the user typed.
if self._text_area is not None and self._image_tracker is not None:
self._image_tracker.remap_spans_to_text(
value, previous_text=self._text_area.text
)
self._history.add(value)
self.post_message(self.Submitted(value, mode))
@@ -2343,11 +2397,19 @@ class ChatInput(Vertical):
self._next_paste_id = 1
self.mode = "normal"
def _sync_media_tracker_to_text(self, text: str) -> None:
def _sync_media_tracker_to_text(
self,
text: str,
*,
previous_text: str | None = None,
cursor_offset: int | None = None,
) -> None:
"""Keep tracked media aligned with placeholder tokens in input text.
Args:
text: Current text in the input area.
previous_text: Previous text in the input area.
cursor_offset: Current cursor offset in the input area.
"""
if not self._image_tracker:
return
@@ -2361,7 +2423,9 @@ class ChatInput(Vertical):
else:
self._skip_media_sync_events -= 1
return
self._image_tracker.sync_to_text(text)
self._image_tracker.sync_to_text(
text, previous_text=previous_text, cursor_offset=cursor_offset
)
def on_chat_text_area_typing(
self,
@@ -2579,7 +2643,14 @@ class ChatInput(Vertical):
media = get_media_from_path(path)
if media is not None:
kind = "image" if isinstance(media, ImageData) else "video"
parts.append(self._image_tracker.add_media(media, kind))
existing_text = self._text_area.text if self._text_area else raw_text
parts.append(
self._image_tracker.add_media(
media,
kind,
existing_text=existing_text,
)
)
attached = True
continue
+198 -1
View File
@@ -16,7 +16,7 @@ from textual.widgets.text_area import Selection
from deepagents_code import _textual_patches as _textual_patches
from deepagents_code.command_registry import SLASH_COMMANDS
from deepagents_code.input import MediaTracker
from deepagents_code.media_utils import ImageData
from deepagents_code.media_utils import ImageData, create_multimodal_content
from deepagents_code.widgets import chat_input as chat_input_module
from deepagents_code.widgets.autocomplete import MAX_SUGGESTIONS
from deepagents_code.widgets.chat_input import (
@@ -2821,6 +2821,133 @@ class TestDroppedImagePaste:
assert len(app.tracker.get_images()) == 1
assert app.tracker.next_image_id == 2
async def test_typed_image_placeholder_is_not_atomic(self) -> None:
"""Manually typed `[image N]` (no attachment) edits char-by-char.
Regression test: placeholder-shaped text the user typed must not be
treated as an atomic media token, so backspace removes a single
character instead of deleting the whole `[image 2]`.
"""
app = _ImagePasteApp()
async with app.run_test() as pilot:
chat = app.query_one(ChatInput)
assert chat._text_area is not None
chat._text_area.text = "[image 2]"
await pilot.pause()
assert app.tracker.get_images() == []
chat._text_area.move_cursor((0, len("[image 2]")))
await pilot.pause()
await pilot.press("backspace")
await pilot.pause()
assert chat._text_area.text == "[image 2"
async def test_typed_placeholder_not_atomic_alongside_real_image(
self, tmp_path
) -> None:
"""A typed look-alike is char-editable while a real one stays atomic."""
img_path = tmp_path / "real.png"
from PIL import Image
image = Image.new("RGB", (4, 4), color="green")
image.save(img_path, format="PNG")
app = _ImagePasteApp()
async with app.run_test() as pilot:
chat = app.query_one(ChatInput)
assert chat._text_area is not None
chat.handle_external_paste(str(img_path))
await pilot.pause()
assert chat._text_area.text == "[image 1] "
# Append a manually typed placeholder-shaped token that is not
# backed by any attachment.
chat._text_area.text = "[image 1] [image 2]"
await pilot.pause()
assert len(app.tracker.get_images()) == 1
chat._text_area.move_cursor((0, len("[image 1] [image 2]")))
await pilot.pause()
await pilot.press("backspace")
await pilot.pause()
# Only one character of the typed token is removed; the real
# `[image 1]` placeholder is untouched and still tracked.
assert chat._text_area.text == "[image 1] [image 2"
assert len(app.tracker.get_images()) == 1
async def test_real_image_placeholder_still_atomic_with_typed_lookalike(
self, tmp_path
) -> None:
"""The real `[image 1]` deletes atomically even beside a typed token."""
img_path = tmp_path / "atomic.png"
from PIL import Image
image = Image.new("RGB", (4, 4), color="purple")
image.save(img_path, format="PNG")
app = _ImagePasteApp()
async with app.run_test() as pilot:
chat = app.query_one(ChatInput)
assert chat._text_area is not None
chat.handle_external_paste(str(img_path))
await pilot.pause()
chat._text_area.text = "[image 2] [image 1]"
await pilot.pause()
assert len(app.tracker.get_images()) == 1
# Cursor just after the real trailing `[image 1]` token.
chat._text_area.move_cursor((0, len("[image 2] [image 1]")))
await pilot.pause()
await pilot.press("backspace")
await pilot.pause()
# The whole real placeholder is removed atomically, leaving the
# typed look-alike intact.
assert chat._text_area.text == "[image 2] "
async def test_submit_remaps_span_onto_stripped_value(self, tmp_path) -> None:
"""`_submit_value` re-maps placeholder spans onto the final submitted text.
Regression: spans captured against the raw draft go stale when submit
strips leading whitespace (and expands pastes), so the adapter would
strip the wrong token from the model-facing message. The span must
follow the transform.
"""
img_path = tmp_path / "submit.png"
from PIL import Image
Image.new("RGB", (4, 4), color="navy").save(img_path, format="PNG")
app = _ImagePasteRecordingApp()
async with app.run_test() as pilot:
chat = app.query_one(ChatInput)
assert chat._text_area is not None
chat.handle_external_paste(str(img_path))
await pilot.pause()
# Leading whitespace shifts every offset when submit strips it.
chat._text_area.text = " look [image 1]"
await pilot.pause()
img = app.tracker.get_images()[0]
assert img.placeholder_span == (7, 16)
chat._submit_value(chat._text_area.text.strip())
await pilot.pause()
assert app.submitted[-1].value == "look [image 1]"
# The span now indexes the submitted value, not the raw draft.
assert img.placeholder_span == (5, 14)
content = create_multimodal_content(
app.submitted[-1].value, app.tracker.get_images()
)
assert content[0]["text"] == "look"
async def test_handle_external_paste_attaches_dropped_image(self, tmp_path) -> None:
"""External paste routing should attach dropped images."""
img_path = tmp_path / "external.png"
@@ -2895,6 +3022,31 @@ class TestDroppedImagePaste:
assert chat._text_area.text.strip() == "[image 1]"
assert len(app.tracker.get_images()) == 1
async def test_paste_image_path_skips_literal_placeholder_in_draft(
self, tmp_path
) -> None:
"""Attaching media does not bind a literal placeholder already in text."""
img_path = tmp_path / "drop.png"
from PIL import Image
image = Image.new("RGB", (4, 4), color="blue")
image.save(img_path, format="PNG")
app = _ImagePasteApp()
async with app.run_test() as pilot:
chat = app.query_one(ChatInput)
assert chat._text_area is not None
chat._text_area.text = "restore [image 1] "
chat._text_area.move_cursor_to_end()
await chat._text_area._on_paste(events.Paste(str(img_path)))
await pilot.pause()
assert chat._text_area.text == "restore [image 1] [image 2] "
assert [img.placeholder for img in app.tracker.get_images()] == [
"[image 2]"
]
async def test_paste_non_image_path_keeps_original_text(self, tmp_path) -> None:
"""Non-image dropped paths should keep the default path paste behavior."""
file_path = tmp_path / "notes.txt"
@@ -3269,6 +3421,29 @@ class TestDroppedVideoPaste:
assert "[video" not in chat._text_area.text
assert app.tracker.get_videos() == []
async def test_typed_video_placeholder_is_not_atomic(self) -> None:
"""Manually typed `[video N]` (no attachment) edits char-by-char.
Mirrors the image look-alike guard for the video code path, which shares
`_bound_media_placeholders` but was otherwise only tested for bound
tokens.
"""
app = _ImagePasteApp()
async with app.run_test() as pilot:
chat = app.query_one(ChatInput)
assert chat._text_area is not None
chat._text_area.text = "[video 2]"
await pilot.pause()
assert app.tracker.get_videos() == []
chat._text_area.move_cursor((0, len("[video 2]")))
await pilot.pause()
await pilot.press("backspace")
await pilot.pause()
assert chat._text_area.text == "[video 2"
async def test_mixed_image_and_video_drop(self, tmp_path: Path) -> None:
"""Dropping an image and video should produce both placeholder types."""
from PIL import Image
@@ -4816,6 +4991,28 @@ class TestPasteCollapseIntegration:
# satisfy a `"[Pasted text #2]" not in text` assertion.
assert chat._text_area.text == "[Pasted text #1]"
async def test_typed_paste_placeholder_is_not_atomic(self) -> None:
"""A paste placeholder with no backing content edits char-by-char.
Regression test for the bound-token guard: `[Pasted text #99]` that the
user typed (or a stale token whose id is absent from `_pasted_contents`)
must not delete atomically, so backspace removes a single character.
Without the `id not in pasted_ids` arm this whole token would vanish.
"""
app = _RecordingApp()
async with app.run_test() as pilot:
chat = app.query_one(ChatInput)
assert chat._text_area is not None
assert chat._pasted_contents == {}
chat._text_area.text = "[Pasted text #99]"
chat._text_area.move_cursor((0, len("[Pasted text #99]")))
await pilot.pause()
await pilot.press("backspace")
await pilot.pause()
assert chat._text_area.text == "[Pasted text #99"
async def test_backspace_removes_multiline_paste_placeholder(self) -> None:
"""Backspace atomically deletes the `+M lines` placeholder variant."""
multi_line = "\n".join(f"line {i}" for i in range(5))
@@ -20,6 +20,7 @@ from deepagents_code.media_utils import (
get_clipboard_image,
get_image_from_path,
get_video_from_path,
strip_media_placeholders,
)
@@ -111,6 +112,28 @@ class TestMediaTracker:
assert placeholder == "[image 1]"
def test_add_image_skips_placeholder_already_in_text(self) -> None:
"""A literal placeholder in the draft is not rebound to new media."""
tracker = MediaTracker()
img = ImageData(base64_data="abc", format="png", placeholder="")
placeholder = tracker.add_image(img, existing_text="restore [image 1]")
assert placeholder == "[image 2]"
assert img.placeholder == "[image 2]"
assert tracker.next_image_id == 3
def test_add_video_skips_placeholder_already_in_text(self) -> None:
"""Video attachment IDs skip literal placeholders in the draft."""
tracker = MediaTracker()
vid = VideoData(base64_data="abc", format="mp4", placeholder="")
placeholder = tracker.add_video(vid, existing_text="restore [video 1]")
assert placeholder == "[video 2]"
assert vid.placeholder == "[video 2]"
assert tracker.next_video_id == 3
def test_sync_to_text_resets_when_placeholders_removed(self) -> None:
"""Removing placeholders from input should clear tracked images and IDs."""
tracker = MediaTracker()
@@ -137,6 +160,120 @@ class TestMediaTracker:
assert len(tracker.images) == 1
assert tracker.images[0].placeholder == "[image 2]"
def test_sync_to_text_tracks_duplicate_inserted_before_placeholder(self) -> None:
"""A typed duplicate before the display token does not steal the media span."""
tracker = MediaTracker()
img = ImageData(base64_data="abc", format="png", placeholder="")
tracker.add_image(img)
tracker.sync_to_text("[image 1]")
text = "literal [image 1] then actual [image 1]"
tracker.sync_to_text(text, previous_text="[image 1]", cursor_offset=30)
assert img.placeholder_span == (30, 39)
def test_sync_to_text_preserves_mapped_duplicate_after_prefix_edit(self) -> None:
"""Edits before duplicates keep the mapped display token bound to media."""
tracker = MediaTracker()
img = ImageData(base64_data="abc", format="png", placeholder="")
tracker.add_image(img)
tracker.sync_to_text("[image 1]")
previous_text = "literal [image 1] actual [image 1]"
tracker.sync_to_text(
previous_text,
previous_text="[image 1]",
cursor_offset=25,
)
text = "Xliteral [image 1] actual [image 1]"
tracker.sync_to_text(text, previous_text=previous_text, cursor_offset=1)
result = create_multimodal_content(text, tracker.images)
assert img.placeholder_span == (26, 35)
assert result[0]["text"] == "Xliteral [image 1] actual"
def test_sync_to_text_tracks_duplicate_inserted_after_placeholder(self) -> None:
"""A typed duplicate after the display token does not steal the media span."""
tracker = MediaTracker()
img = ImageData(base64_data="abc", format="png", placeholder="")
tracker.add_image(img)
tracker.sync_to_text("[image 1]")
text = "[image 1] literal [image 1]"
tracker.sync_to_text(text, previous_text="[image 1]")
assert img.placeholder_span == (0, 9)
def test_sync_to_text_edit_between_placeholder_and_duplicate_keeps_actual(
self,
) -> None:
"""Typing between an actual placeholder and a duplicate does not rebind it."""
tracker = MediaTracker()
img = ImageData(base64_data="abc", format="png", placeholder="")
tracker.add_image(img)
tracker.sync_to_text("[image 1]")
tracker.sync_to_text("[image 1] literal [image 1]", previous_text="[image 1]")
text = "[image 1] edited literal [image 1]"
tracker.sync_to_text(
text,
previous_text="[image 1] literal [image 1]",
cursor_offset=17,
)
assert img.placeholder_span == (0, 9)
def test_remap_spans_to_text_shifts_span_and_strips_correct_duplicate(
self,
) -> None:
"""Remapping keeps the real (second) token bound after an offset shift.
Regression: spans are captured against the draft but consumed after
submit-time rewrites (whitespace trim, paste expansion) shift offsets.
Without remapping, the stale span is discarded and the token-count
fallback strips the *first* occurrence — the user's literal — leaving the
real display token in the model-facing text. Remapping fixes the offset.
"""
tracker = MediaTracker()
img = ImageData(base64_data="abc", format="png", placeholder="")
tracker.add_image(img)
tracker.sync_to_text("[image 1]")
# A literal duplicate precedes the real display token in the draft.
draft = "see [image 1] then real [image 1]"
tracker.sync_to_text(draft, previous_text="[image 1]", cursor_offset=24)
assert img.placeholder_span == (24, 33)
# Submit expands a paste before the token, shifting it right by 15 chars.
final = "EXPANDED PASTE see [image 1] then real [image 1]"
tracker.remap_spans_to_text(final, previous_text=draft)
assert img.placeholder_span == (39, 48)
result = create_multimodal_content(final, tracker.get_images())
assert result[0]["text"] == "EXPANDED PASTE see [image 1] then real"
def test_remap_spans_to_text_drops_span_when_token_edited(self) -> None:
"""A span whose token was edited in the transformed text becomes None."""
tracker = MediaTracker()
img = ImageData(base64_data="abc", format="png", placeholder="")
tracker.add_image(img)
tracker.sync_to_text("[image 1]")
assert img.placeholder_span == (0, 9)
# The token characters themselves changed: cannot be cleanly mapped.
tracker.remap_spans_to_text("[image 2]", previous_text="[image 1]")
assert img.placeholder_span is None
def test_remap_spans_to_text_ignores_items_without_span(self) -> None:
"""Items with no captured span are left as None (token-fallback path)."""
tracker = MediaTracker()
img = ImageData(base64_data="abc", format="png", placeholder="[image 1]")
tracker.images.append(img) # attached but never span-synced
tracker.remap_spans_to_text("[image 1]", previous_text="[image 1]")
assert img.placeholder_span is None
class TestEncodeImageToBase64:
"""Tests for base64 encoding."""
@@ -214,6 +351,238 @@ class TestCreateMultimodalContent:
assert len(result) == 1
assert result[0]["type"] == "image_url"
def test_placeholder_not_leaked_into_text_block(self) -> None:
"""The `[image N]` display placeholder must not reach model-facing text.
Regression test: sending an image previously serialized the display
placeholder as literal user-authored text in the traced/model message.
"""
img = ImageData(base64_data="abc", format="png", placeholder="[image 1]")
result = create_multimodal_content("[image 1] what's in this image?", [img])
assert len(result) == 2
assert result[0]["type"] == "text"
# Placeholder is gone but the surrounding user text is preserved.
assert "[image 1]" not in result[0]["text"]
assert result[0]["text"] == "what's in this image?"
assert result[1]["type"] == "image_url"
def test_literal_duplicate_placeholder_preserved_in_text_block(self) -> None:
"""Only the display placeholder occurrence is stripped from text."""
img = ImageData(
base64_data="abc",
format="png",
placeholder="[image 1]",
placeholder_span=(0, 9),
)
result = create_multimodal_content(
"[image 1] compare with literal [image 1]",
[img],
)
assert len(result) == 2
assert result[0]["type"] == "text"
assert result[0]["text"] == "compare with literal [image 1]"
assert result[1]["type"] == "image_url"
def test_literal_duplicate_before_placeholder_preserved_in_text_block(self) -> None:
"""A literal duplicate before the display placeholder is preserved."""
img = ImageData(
base64_data="abc",
format="png",
placeholder="[image 1]",
placeholder_span=(30, 39),
)
result = create_multimodal_content(
"literal [image 1] then actual [image 1]",
[img],
)
assert len(result) == 2
assert result[0]["type"] == "text"
assert result[0]["text"] == "literal [image 1] then actual"
assert result[1]["type"] == "image_url"
def test_placeholder_removed_when_only_placeholder(self) -> None:
"""A message that is only a placeholder yields no text block."""
img = ImageData(base64_data="abc", format="png", placeholder="[image 1]")
result = create_multimodal_content("[image 1]", [img])
assert len(result) == 1
assert result[0]["type"] == "image_url"
def test_placeholders_removed_for_video(self) -> None:
"""Video placeholders are also stripped from model-facing text."""
vid = VideoData(base64_data="vid", format="mp4", placeholder="[video 1]")
result = create_multimodal_content("summarize [video 1] please", [], [vid])
assert result[0]["type"] == "text"
assert "[video 1]" not in result[0]["text"]
assert result[0]["text"] == "summarize please"
def test_multiple_placeholders_all_stripped(self) -> None:
"""All placeholders are removed while surrounding text is preserved."""
img1 = ImageData(base64_data="a", format="png", placeholder="[image 1]")
img2 = ImageData(base64_data="b", format="png", placeholder="[image 2]")
result = create_multimodal_content(
"[image 1] and [image 2] differ how?", [img1, img2]
)
assert result[0]["type"] == "text"
assert "[image 1]" not in result[0]["text"]
assert "[image 2]" not in result[0]["text"]
assert result[0]["text"] == "and differ how?"
def test_mixed_image_and_video_placeholders_stripped(self) -> None:
"""Image and video placeholders in one message are both stripped.
Exercises the combined `images + videos` placeholder assembly and the
heterogeneous `|`-token strip together, which same-type cases miss.
"""
img = ImageData(base64_data="a", format="png", placeholder="[image 1]")
vid = VideoData(base64_data="v", format="mp4", placeholder="[video 1]")
result = create_multimodal_content("[image 1] vs [video 1]", [img], [vid])
assert result[0]["type"] == "text"
assert result[0]["text"] == "vs"
# Both media blocks follow the single text block.
assert result[1]["type"] == "image_url"
assert result[2]["type"] == "video"
def test_unbound_placeholder_like_text_preserved(self) -> None:
"""Placeholder-shaped text not bound to attached media is preserved.
Regression test for false positives: a user attaches image 1 but their
prompt also literally mentions `[image 2]` (which is not attached). Only
the real `[image 1]` token is stripped; the literal `[image 2]` stays.
"""
img = ImageData(base64_data="abc", format="png", placeholder="[image 1]")
result = create_multimodal_content(
"[image 1] see the note about [image 2] in the docs", [img]
)
assert result[0]["type"] == "text"
assert result[0]["text"] == "see the note about [image 2] in the docs"
def test_placeholder_like_text_without_attachment_untouched(self) -> None:
"""With no media attached, placeholder-shaped text is never stripped."""
result = create_multimodal_content("compare [image 1] vs [image 2]", [])
assert len(result) == 1
assert result[0]["type"] == "text"
assert result[0]["text"] == "compare [image 1] vs [image 2]"
class TestStripMediaPlaceholders:
"""Tests for `strip_media_placeholders`."""
def test_leading_placeholder(self) -> None:
"""A leading placeholder is removed along with its trailing space."""
assert strip_media_placeholders("[image 1] hello", ["[image 1]"]) == "hello"
def test_inline_placeholder_no_double_space(self) -> None:
"""An inline placeholder does not leave a double space behind."""
assert (
strip_media_placeholders("before [image 1] after", ["[image 1]"])
== "before after"
)
def test_duplicate_literal_placeholder_preserved(self) -> None:
"""Duplicate literal text is preserved when one matching media is attached."""
assert (
strip_media_placeholders(
"[image 1] describe literal [image 1]",
["[image 1]"],
placeholder_spans=[(0, 9)],
)
== "describe literal [image 1]"
)
def test_literal_duplicate_before_placeholder_preserved(self) -> None:
"""Span tracking strips the display token, not the first duplicate."""
assert (
strip_media_placeholders(
"literal [image 1] then actual [image 1]",
["[image 1]"],
placeholder_spans=[(30, 39)],
)
== "literal [image 1] then actual"
)
def test_only_bound_placeholders_removed(self) -> None:
"""Only tokens in the bound set are removed; look-alikes are preserved."""
result = strip_media_placeholders(
"keep [image 2] drop [image 1]", ["[image 1]"]
)
assert result == "keep [image 2] drop"
def test_stale_span_is_discarded_and_falls_back_to_token(self) -> None:
"""A span whose slice no longer matches the token is ignored, not used.
Guards the graceful-degradation contract: an out-of-date span (here the
offset points at non-token text) must not delete arbitrary characters;
the token fallback removes the real occurrence instead.
"""
result = strip_media_placeholders(
"hello [image 1] world",
["[image 1]"],
placeholder_spans=[(0, 5)], # points at "hello", not the token
)
assert result == "hello world"
def test_ambiguous_fallback_strips_leading_occurrence(self) -> None:
"""With no span and duplicate tokens, the fallback strips the first one."""
result = strip_media_placeholders(
"first [image 1] second [image 1]",
["[image 1]"],
)
assert result == "first second [image 1]"
def test_mixed_image_and_video_tokens_removed(self) -> None:
"""Heterogeneous image and video tokens are stripped in one pass."""
result = strip_media_placeholders(
"[image 1] and [video 1] together", ["[image 1]", "[video 1]"]
)
assert result == "and together"
def test_text_without_placeholder_unchanged(self) -> None:
"""Text without a bound placeholder is unchanged (aside from trim)."""
assert (
strip_media_placeholders("plain text only", ["[image 1]"])
== "plain text only"
)
def test_no_placeholders_returns_text_verbatim(self) -> None:
"""An empty placeholder set returns the text unchanged, including whitespace."""
assert strip_media_placeholders(" [image 1] ", []) == " [image 1] "
def test_only_placeholder_becomes_empty(self) -> None:
"""A string that is only a bound placeholder becomes empty."""
assert strip_media_placeholders("[video 2]", ["[video 2]"]) == ""
def test_newlines_preserved(self) -> None:
"""Newlines around a placeholder are preserved."""
assert (
strip_media_placeholders("line1\n[image 1]\nline2", ["[image 1]"])
== "line1\n\nline2"
)
def test_special_regex_chars_in_placeholder_are_escaped(self) -> None:
"""Placeholder tokens are treated literally, not as regex."""
assert strip_media_placeholders("a [image (1)] b", ["[image (1)]"]) == "a b"
def test_indentation_preserved_after_placeholder(self) -> None:
r"""Leading newlines and code indentation after a placeholder survive.
Regression: ``.strip()`` removed all leading whitespace, so attaching
an image before indented code (``[image 1]\n def foo():``) would
lose the four-space indent. Only spaces/tabs on each edge are trimmed.
"""
assert (
strip_media_placeholders("[image 1]\n def foo():", ["[image 1]"])
== "\n def foo():"
)
class TestGetClipboardImage:
"""Tests for clipboard image detection."""