mirror of
https://github.com/langchain-ai/deepagents.git
synced 2026-07-22 01:35:28 -04:00
fix(code): strip media placeholders from model-facing message text (#4462)
- Display-only `[image N]`/`[video N]` placeholders no longer appear in the model-facing message text or LangSmith trace; only the structured media block is sent. - Manually typing `[image N]`/`[video N]` text that looks like a placeholder no longer triggers atomic deletion on backspace — it edits character by character like ordinary text. --- When a user attaches an image or video in the Deep Agents Code TUI, a display-only `[image N]` or `[video N]` placeholder appears in the input area. The placeholder is purely visual — the actual media travels as a structured content block alongside the text. Two bugs caused that display convention to leak into model-facing behavior: 1. **Tracing/model leak:** `create_multimodal_content` copied the raw input text verbatim into the message text block, so the canonical `HumanMessage` sent to the model and serialized to LangSmith contained the display-only `[image 1]` as if the user had typed it. It now strips only the exact placeholder tokens bound to media actually attached to that message (via `strip_media_placeholders`), leaving look-alike literal text intact and requiring no escaping mechanism. 2. **Editing false positive:** `ChatTextArea._find_placeholder_span` treated any text matching the placeholder regex as an atomic token, so manually typing `[image 2]` meant a single backspace deleted the whole token. It now validates each regex match against real state — the media tracker for image/video placeholders and `_pasted_contents` for paste placeholders — so only tokens bound to real attachments delete atomically while typed look-alikes edit character by character. Disambiguating a display token from a user-typed duplicate of the same string (e.g. attaching an image that gets `[image 1]`, then the user also types `[image 1]` in the same message) requires tracking the exact character span of each display token, not just its text. `ImageData` and `VideoData` now carry a `placeholder_span` that the tracker updates through edits via `sync_to_text` (using a diff-based span mapper) and re-maps at submit time through text transforms — whitespace trimming, paste expansion, prefix prepending — that shift offsets. `add_media` also skips placeholder IDs already present in the draft so a literal `[image 1]` in the text doesn't cause a newly attached image to reuse the same ID. Made by [Open SWE](https://openswe.vercel.app/agents/0fc423a2-7100-74d7-2a83-4ea855c9d4bb) --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
This commit is contained in:
@@ -4,6 +4,7 @@ import logging
|
||||
import re
|
||||
import shlex
|
||||
from dataclasses import dataclass
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from urllib.parse import unquote, urlparse
|
||||
@@ -118,49 +119,65 @@ class MediaTracker:
|
||||
self.next_image_id: int = 1
|
||||
self.next_video_id: int = 1
|
||||
|
||||
def add_media(self, data: ImageData | VideoData, kind: MediaKind) -> str:
|
||||
def add_media(
|
||||
self,
|
||||
data: ImageData | VideoData,
|
||||
kind: MediaKind,
|
||||
*,
|
||||
existing_text: str = "",
|
||||
) -> str:
|
||||
"""Add a media item and return its placeholder text.
|
||||
|
||||
Args:
|
||||
data: The image or video data to track.
|
||||
kind: Media type key.
|
||||
existing_text: Current draft text. Placeholder IDs already present
|
||||
here are skipped so literal user text is not bound to new media.
|
||||
|
||||
Returns:
|
||||
Placeholder string like "[image 1]" or "[video 1]".
|
||||
"""
|
||||
if kind == "image":
|
||||
while f"[image {self.next_image_id}]" in existing_text:
|
||||
self.next_image_id += 1
|
||||
placeholder = f"[image {self.next_image_id}]"
|
||||
data.placeholder = placeholder
|
||||
self.images.append(data) # ty: ignore[invalid-argument-type]
|
||||
self.next_image_id += 1
|
||||
else:
|
||||
while f"[video {self.next_video_id}]" in existing_text:
|
||||
self.next_video_id += 1
|
||||
placeholder = f"[video {self.next_video_id}]"
|
||||
data.placeholder = placeholder
|
||||
self.videos.append(data) # ty: ignore[invalid-argument-type]
|
||||
self.next_video_id += 1
|
||||
return placeholder
|
||||
|
||||
def add_image(self, image_data: ImageData) -> str:
|
||||
def add_image(self, image_data: ImageData, *, existing_text: str = "") -> str:
|
||||
"""Add an image and return its placeholder text.
|
||||
|
||||
Args:
|
||||
image_data: The image data to track.
|
||||
existing_text: Current draft text. Placeholder IDs already present
|
||||
here are skipped so literal user text is not bound to new media.
|
||||
|
||||
Returns:
|
||||
Placeholder string like "[image 1]".
|
||||
"""
|
||||
return self.add_media(image_data, "image")
|
||||
return self.add_media(image_data, "image", existing_text=existing_text)
|
||||
|
||||
def add_video(self, video_data: VideoData) -> str:
|
||||
def add_video(self, video_data: VideoData, *, existing_text: str = "") -> str:
|
||||
"""Add a video and return its placeholder text.
|
||||
|
||||
Args:
|
||||
video_data: The video data to track.
|
||||
existing_text: Current draft text. Placeholder IDs already present
|
||||
here are skipped so literal user text is not bound to new media.
|
||||
|
||||
Returns:
|
||||
Placeholder string like "[video 1]".
|
||||
"""
|
||||
return self.add_media(video_data, "video")
|
||||
return self.add_media(video_data, "video", existing_text=existing_text)
|
||||
|
||||
def get_images(self) -> list[ImageData]:
|
||||
"""Get all tracked images.
|
||||
@@ -185,28 +202,82 @@ class MediaTracker:
|
||||
self.next_image_id = 1
|
||||
self.next_video_id = 1
|
||||
|
||||
def sync_to_text(self, text: str) -> None:
|
||||
def sync_to_text(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
previous_text: str | None = None,
|
||||
cursor_offset: int | None = None,
|
||||
) -> None:
|
||||
"""Retain only media still referenced by placeholders in current text.
|
||||
|
||||
Args:
|
||||
text: Current input text shown to the user.
|
||||
previous_text: Previous input text, used to keep tracking the same
|
||||
placeholder occurrence when duplicate literal tokens are added.
|
||||
cursor_offset: Current cursor offset, used to disambiguate whole-paste
|
||||
edits that create duplicate placeholder tokens.
|
||||
"""
|
||||
img_found = self._sync_kind_images(text)
|
||||
vid_found = self._sync_kind_videos(text)
|
||||
img_found = self._sync_kind_images(
|
||||
text, previous_text=previous_text, cursor_offset=cursor_offset
|
||||
)
|
||||
vid_found = self._sync_kind_videos(
|
||||
text, previous_text=previous_text, cursor_offset=cursor_offset
|
||||
)
|
||||
if not img_found and not vid_found:
|
||||
self.clear()
|
||||
|
||||
def _sync_kind_images(self, text: str) -> bool:
|
||||
def remap_spans_to_text(self, text: str, *, previous_text: str) -> None:
|
||||
"""Re-map tracked placeholder spans onto a transformed copy of the text.
|
||||
|
||||
Submission rewrites the draft before the display placeholders are
|
||||
stripped from the model-facing text: whitespace is trimmed, collapsed
|
||||
pastes expand back to full content, dropped paths become placeholders,
|
||||
and a mode prefix may be prepended. Every one of those shifts character
|
||||
offsets, so a `placeholder_span` captured against the draft would be
|
||||
stale by the time `strip_media_placeholders` consumes it — silently
|
||||
stripping the wrong occurrence when a user-typed duplicate is present.
|
||||
|
||||
Re-mapping each span through the same before/after diff keeps it pointing
|
||||
at its own display token in `text`. Spans that cannot be cleanly mapped
|
||||
become `None`, degrading to the token-count fallback rather than a wrong
|
||||
strip.
|
||||
|
||||
Args:
|
||||
text: The transformed text the spans must line up with.
|
||||
previous_text: The draft text the current spans were captured
|
||||
against.
|
||||
"""
|
||||
for item in (*self.images, *self.videos):
|
||||
if item.placeholder_span is None:
|
||||
continue
|
||||
item.placeholder_span = self._map_placeholder_span(
|
||||
item.placeholder_span, previous_text, text
|
||||
)
|
||||
|
||||
def _sync_kind_images(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
previous_text: str | None = None,
|
||||
cursor_offset: int | None = None,
|
||||
) -> bool:
|
||||
"""Sync image list to surviving placeholders in text.
|
||||
|
||||
Args:
|
||||
text: Current input text.
|
||||
previous_text: Previous input text, used to map existing spans.
|
||||
cursor_offset: Current cursor offset for duplicate disambiguation.
|
||||
|
||||
Returns:
|
||||
Whether any image placeholders were found.
|
||||
"""
|
||||
placeholders = {m.group(0) for m in IMAGE_PLACEHOLDER_PATTERN.finditer(text)}
|
||||
matches = list(IMAGE_PLACEHOLDER_PATTERN.finditer(text))
|
||||
placeholders = {m.group(0) for m in matches}
|
||||
self.images = [img for img in self.images if img.placeholder in placeholders]
|
||||
self._update_placeholder_spans(
|
||||
self.images, matches, text, previous_text, cursor_offset
|
||||
)
|
||||
if not self.images:
|
||||
self.next_image_id = 1
|
||||
else:
|
||||
@@ -215,17 +286,29 @@ class MediaTracker:
|
||||
)
|
||||
return bool(placeholders)
|
||||
|
||||
def _sync_kind_videos(self, text: str) -> bool:
|
||||
def _sync_kind_videos(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
previous_text: str | None = None,
|
||||
cursor_offset: int | None = None,
|
||||
) -> bool:
|
||||
"""Sync video list to surviving placeholders in text.
|
||||
|
||||
Args:
|
||||
text: Current input text.
|
||||
previous_text: Previous input text, used to map existing spans.
|
||||
cursor_offset: Current cursor offset for duplicate disambiguation.
|
||||
|
||||
Returns:
|
||||
Whether any video placeholders were found.
|
||||
"""
|
||||
placeholders = {m.group(0) for m in VIDEO_PLACEHOLDER_PATTERN.finditer(text)}
|
||||
matches = list(VIDEO_PLACEHOLDER_PATTERN.finditer(text))
|
||||
placeholders = {m.group(0) for m in matches}
|
||||
self.videos = [vid for vid in self.videos if vid.placeholder in placeholders]
|
||||
self._update_placeholder_spans(
|
||||
self.videos, matches, text, previous_text, cursor_offset
|
||||
)
|
||||
if not self.videos:
|
||||
self.next_video_id = 1
|
||||
else:
|
||||
@@ -234,6 +317,109 @@ class MediaTracker:
|
||||
)
|
||||
return bool(placeholders)
|
||||
|
||||
def _update_placeholder_spans(
|
||||
self,
|
||||
items: list[ImageData] | list[VideoData],
|
||||
matches: list[re.Match[str]],
|
||||
text: str,
|
||||
previous_text: str | None,
|
||||
cursor_offset: int | None,
|
||||
) -> None:
|
||||
"""Refresh tracked placeholder spans for surviving media items.
|
||||
|
||||
Args:
|
||||
items: Surviving tracked media items.
|
||||
matches: Placeholder regex matches in the current text.
|
||||
text: Current input text.
|
||||
previous_text: Previous input text, used to map existing spans.
|
||||
cursor_offset: Current cursor offset for duplicate disambiguation.
|
||||
"""
|
||||
spans_by_token: dict[str, list[tuple[int, int]]] = {}
|
||||
for match in matches:
|
||||
spans_by_token.setdefault(match.group(0), []).append(match.span())
|
||||
|
||||
for item in items:
|
||||
spans = spans_by_token.get(item.placeholder, [])
|
||||
cursor_span = self._placeholder_span_after_cursor(spans, cursor_offset)
|
||||
mapped = self._map_placeholder_span(
|
||||
item.placeholder_span, previous_text, text
|
||||
)
|
||||
had_duplicate = (
|
||||
previous_text is not None and previous_text.count(item.placeholder) > 1
|
||||
)
|
||||
if had_duplicate and mapped is not None and mapped in spans:
|
||||
item.placeholder_span = mapped
|
||||
elif cursor_span is not None and (
|
||||
item.placeholder_span is None or mapped != item.placeholder_span
|
||||
):
|
||||
item.placeholder_span = cursor_span
|
||||
elif len(spans) == 1:
|
||||
item.placeholder_span = spans[0]
|
||||
elif mapped is not None and mapped in spans:
|
||||
item.placeholder_span = mapped
|
||||
elif item.placeholder_span not in spans:
|
||||
item.placeholder_span = None
|
||||
|
||||
@staticmethod
|
||||
def _placeholder_span_after_cursor(
|
||||
spans: list[tuple[int, int]], cursor_offset: int | None
|
||||
) -> tuple[int, int] | None:
|
||||
"""Return the first duplicate placeholder span at or after the cursor.
|
||||
|
||||
Only meaningful when the token is duplicated (`len(spans) > 1`); returns
|
||||
`None` otherwise, or when no span starts at/after the cursor.
|
||||
|
||||
Args:
|
||||
spans: Placeholder spans for one token in current text.
|
||||
cursor_offset: Current cursor offset, or `None` when unknown.
|
||||
"""
|
||||
if cursor_offset is None or len(spans) <= 1:
|
||||
return None
|
||||
for span in spans:
|
||||
start, _end = span
|
||||
if start >= cursor_offset:
|
||||
return span
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _map_placeholder_span(
|
||||
span: tuple[int, int] | None,
|
||||
previous_text: str | None,
|
||||
text: str,
|
||||
) -> tuple[int, int] | None:
|
||||
"""Map a placeholder span from the previous text into current text.
|
||||
|
||||
Uses a `SequenceMatcher` diff so the span survives edits elsewhere in the
|
||||
text: it returns the shifted span only when its whole range falls inside
|
||||
an unchanged (`equal`) block, and `None` when the token's own characters
|
||||
were edited or the span cannot be located.
|
||||
|
||||
Args:
|
||||
span: The placeholder span in `previous_text`, or `None`.
|
||||
previous_text: Text the span was captured against, or `None`.
|
||||
text: Text to map the span into.
|
||||
|
||||
Returns:
|
||||
The mapped span when the same placeholder occurrence survives,
|
||||
otherwise `None`.
|
||||
"""
|
||||
if span is None or previous_text is None:
|
||||
return None
|
||||
start, end = span
|
||||
if not (0 <= start < end <= len(previous_text)):
|
||||
return None
|
||||
|
||||
matcher = SequenceMatcher(a=previous_text, b=text, autojunk=False)
|
||||
for tag, old_start, old_end, new_start, _new_end in matcher.get_opcodes():
|
||||
if tag == "equal" and old_start <= start and end <= old_end:
|
||||
offset = new_start - old_start
|
||||
return start + offset, end + offset
|
||||
if old_start <= start and end <= old_end:
|
||||
return None
|
||||
if old_start > end:
|
||||
break
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _max_placeholder_id(
|
||||
items: list[ImageData] | list[VideoData],
|
||||
|
||||
@@ -5,12 +5,15 @@ import io
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
|
||||
# S404: subprocess needed for clipboard access via pngpaste/osascript
|
||||
import subprocess # noqa: S404
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
@@ -50,6 +53,118 @@ MAX_MEDIA_BYTES: int = 20 * 1024 * 1024
|
||||
"""Maximum media file size (20 MB). Keeps base64 payload under ~27 MB."""
|
||||
|
||||
|
||||
def strip_media_placeholders(
|
||||
text: str,
|
||||
placeholders: Iterable[str],
|
||||
*,
|
||||
placeholder_spans: Iterable[tuple[int, int]] | None = None,
|
||||
) -> str:
|
||||
"""Remove display-only media placeholders from user text.
|
||||
|
||||
Placeholders like `[image 1]` are inserted into the terminal input purely for
|
||||
display; the actual media travels as structured content blocks. They must not
|
||||
leak into the canonical model-facing message or LangSmith trace as if the user
|
||||
typed them.
|
||||
|
||||
When available, tracked placeholder spans identify the exact display tokens to
|
||||
strip so user-authored literal duplicates with the same token are preserved.
|
||||
The token fallback removes one matching occurrence per tracked media item for
|
||||
callers that only have placeholder text.
|
||||
|
||||
Args:
|
||||
text: Raw user text that may contain media placeholders.
|
||||
placeholders: Exact placeholder tokens for the media actually attached to
|
||||
this message (e.g. ``["[image 1]", "[video 1]"]``).
|
||||
placeholder_spans: Exact `(start, end)` spans for tracked display tokens
|
||||
in `text`, when known.
|
||||
|
||||
Returns:
|
||||
Text with the given media placeholders removed and surrounding whitespace
|
||||
tidied. Newlines are preserved so multi-line prompts keep their structure.
|
||||
Returns an empty string when only whitespace remains after removal, so
|
||||
callers can treat a placeholder-only message as having no text block.
|
||||
"""
|
||||
tokens = [p for p in placeholders if p]
|
||||
if not tokens:
|
||||
return text
|
||||
|
||||
valid_spans = _valid_placeholder_spans(text, tokens, placeholder_spans)
|
||||
spans = [(start, end) for start, end, _token in valid_spans]
|
||||
counts = Counter(tokens)
|
||||
for _start, _end, token in valid_spans:
|
||||
counts[token] -= 1
|
||||
for token, count in counts.items():
|
||||
if count <= 0:
|
||||
continue
|
||||
pattern = re.compile(r"[ \t]*" + re.escape(token))
|
||||
matches = list(pattern.finditer(text))
|
||||
if len(matches) > count:
|
||||
# No (or too few) tracked spans, yet the token appears more times
|
||||
# than we intend to strip: we can't tell the display token from a
|
||||
# user-typed literal and fall back to removing the leading
|
||||
# occurrence(s). That guess can strip a literal and leave the real
|
||||
# display token behind, so leave a breadcrumb (never the text).
|
||||
logger.debug(
|
||||
"Ambiguous media placeholder strip: removing %d of %d "
|
||||
"occurrences of %r with no tracked span to disambiguate",
|
||||
count,
|
||||
len(matches),
|
||||
token,
|
||||
)
|
||||
for index, match in enumerate(matches):
|
||||
if index >= count:
|
||||
break
|
||||
spans.append(match.span())
|
||||
|
||||
# Only strip spaces/tabs (not newlines) so code indentation on lines after a
|
||||
# removed placeholder is preserved. A full .strip() would collapse
|
||||
# "[image 1]\n def foo():" to "def foo():", losing the leading indent.
|
||||
cleaned = text
|
||||
for start, end in sorted(spans, reverse=True):
|
||||
cleaned = cleaned[:start] + cleaned[end:]
|
||||
cleaned = cleaned.strip(" \t")
|
||||
return cleaned if cleaned.strip() else ""
|
||||
|
||||
|
||||
def _valid_placeholder_spans(
|
||||
text: str,
|
||||
tokens: list[str],
|
||||
spans: Iterable[tuple[int, int]] | None,
|
||||
) -> list[tuple[int, int, str]]:
|
||||
"""Return valid display placeholder spans expanded over adjacent padding.
|
||||
|
||||
A span is kept only when it is in range and its slice equals one of the
|
||||
bound tokens, so a stale span (e.g. left by an offset shift) is dropped
|
||||
rather than used to delete arbitrary text; the caller then falls back to
|
||||
token matching for that item.
|
||||
|
||||
Args:
|
||||
text: Text the spans index into.
|
||||
tokens: Bound placeholder tokens for the attached media.
|
||||
spans: Candidate `(start, end)` display-token spans, or `None`.
|
||||
|
||||
Returns:
|
||||
`(start, end, token)` triples with `start` expanded left over any
|
||||
adjacent spaces/tabs, for spans that validate against `text`.
|
||||
"""
|
||||
if spans is None:
|
||||
return []
|
||||
|
||||
token_set = set(tokens)
|
||||
valid: list[tuple[int, int, str]] = []
|
||||
for start, end in spans:
|
||||
if not (0 <= start < end <= len(text)):
|
||||
continue
|
||||
token = text[start:end]
|
||||
if token not in token_set:
|
||||
continue
|
||||
expanded_start = start
|
||||
while expanded_start > 0 and text[expanded_start - 1] in " \t":
|
||||
expanded_start -= 1
|
||||
valid.append((expanded_start, end, token))
|
||||
return valid
|
||||
|
||||
|
||||
def _get_executable(name: str) -> str | None:
|
||||
"""Get full path to an executable using shutil.which().
|
||||
|
||||
@@ -69,6 +184,7 @@ class ImageData:
|
||||
base64_data: str
|
||||
format: str # "png", "jpeg", etc.
|
||||
placeholder: str # Display text like "[image 1]"
|
||||
placeholder_span: tuple[int, int] | None = None
|
||||
|
||||
def to_message_content(self) -> dict:
|
||||
"""Convert to LangChain message content format.
|
||||
@@ -89,6 +205,7 @@ class VideoData:
|
||||
base64_data: str
|
||||
format: str # "mp4", "quicktime", etc.
|
||||
placeholder: str # Display text like "[video 1]"
|
||||
placeholder_span: tuple[int, int] | None = None
|
||||
|
||||
def to_message_content(self) -> "VideoContentBlock":
|
||||
"""Convert to LangChain `VideoContentBlock` format.
|
||||
@@ -464,9 +581,26 @@ def create_multimodal_content(
|
||||
"""
|
||||
content_blocks = []
|
||||
|
||||
# Add text block
|
||||
if text.strip():
|
||||
content_blocks.append({"type": "text", "text": text})
|
||||
# Add text block. Strip only the display-only placeholders bound to the media
|
||||
# actually attached here (e.g. "[image 1]") so the canonical/model-facing text
|
||||
# never contains fake user-authored placeholder text. When a span is known,
|
||||
# text that merely resembles the schema is preserved exactly; without a span
|
||||
# `strip_media_placeholders` falls back to removing one occurrence per item,
|
||||
# which can catch a look-alike literal. The media itself is carried by the
|
||||
# structured blocks below.
|
||||
#
|
||||
# `placeholders` and `spans` are passed as parallel-but-unzipped lists on
|
||||
# purpose: `strip_media_placeholders` recovers each span's token from the
|
||||
# text slice, and each tracked media item has a unique token, so it never
|
||||
# needs index alignment between the two.
|
||||
media = [*images, *(videos or [])]
|
||||
placeholders = [item.placeholder for item in media]
|
||||
spans = [
|
||||
item.placeholder_span for item in media if item.placeholder_span is not None
|
||||
]
|
||||
clean_text = strip_media_placeholders(text, placeholders, placeholder_spans=spans)
|
||||
if clean_text:
|
||||
content_blocks.append({"type": "text", "text": clean_text})
|
||||
|
||||
# Add image blocks
|
||||
content_blocks.extend(image.to_message_content() for image in images)
|
||||
|
||||
@@ -1329,6 +1329,33 @@ class ChatTextArea(TextArea):
|
||||
self.move_cursor(start_location)
|
||||
return True
|
||||
|
||||
def _bound_media_placeholders(self) -> set[str]:
|
||||
"""Return placeholder tokens bound to currently tracked media.
|
||||
|
||||
Returns:
|
||||
The set of `[image N]`/`[video N]` tokens for media the tracker is
|
||||
actually holding. Empty when there is no owner/tracker.
|
||||
"""
|
||||
owner = self._chat_input_owner
|
||||
tracker = owner._image_tracker if owner is not None else None
|
||||
if tracker is None:
|
||||
return set()
|
||||
placeholders = {img.placeholder for img in tracker.images}
|
||||
placeholders.update(video.placeholder for video in tracker.videos)
|
||||
return placeholders
|
||||
|
||||
def _bound_paste_ids(self) -> set[int]:
|
||||
"""Return paste ids that have backing content in the owner.
|
||||
|
||||
Returns:
|
||||
The set of paste ids present in `ChatInput._pasted_contents`. Empty
|
||||
when there is no owner.
|
||||
"""
|
||||
owner = self._chat_input_owner
|
||||
if owner is None:
|
||||
return set()
|
||||
return set(owner._pasted_contents)
|
||||
|
||||
def _find_placeholder_span(
|
||||
self, cursor_offset: int, *, backwards: bool
|
||||
) -> tuple[int, int] | None:
|
||||
@@ -1340,6 +1367,12 @@ class ChatTextArea(TextArea):
|
||||
here so an undo can restore the token with its content (it is cleared
|
||||
only at submit).
|
||||
|
||||
Only tokens bound to real attachments are treated as atomic: image/video
|
||||
placeholders must correspond to a tracked media item and paste
|
||||
placeholders to an entry in `ChatInput._pasted_contents`. Placeholder-
|
||||
shaped text the user typed by hand (e.g. literally typing ``[image 2]``)
|
||||
is left as ordinary text and edits character by character.
|
||||
|
||||
Args:
|
||||
cursor_offset: Character offset of the cursor from the start of text.
|
||||
backwards: Whether the delete action is backwards (backspace) or
|
||||
@@ -1347,15 +1380,23 @@ class ChatTextArea(TextArea):
|
||||
|
||||
Returns:
|
||||
The `(start, end)` character span of the placeholder to delete, or
|
||||
`None` when the cursor is not adjacent to a placeholder token.
|
||||
`None` when the cursor is not adjacent to a bound placeholder
|
||||
token.
|
||||
"""
|
||||
text = self.text
|
||||
media_placeholders = self._bound_media_placeholders()
|
||||
pasted_ids = self._bound_paste_ids()
|
||||
for pattern in (
|
||||
IMAGE_PLACEHOLDER_PATTERN,
|
||||
VIDEO_PLACEHOLDER_PATTERN,
|
||||
PASTE_PLACEHOLDER_PATTERN,
|
||||
):
|
||||
for match in pattern.finditer(text):
|
||||
if pattern is PASTE_PLACEHOLDER_PATTERN:
|
||||
if int(match.group(1)) not in pasted_ids:
|
||||
continue
|
||||
elif match.group(0) not in media_placeholders:
|
||||
continue
|
||||
start, end = match.span()
|
||||
if backwards:
|
||||
# Cursor is inside token or right after a trailing space inserted
|
||||
@@ -1927,8 +1968,11 @@ class ChatInput(Vertical):
|
||||
# preserves replacement edits where selected text is replaced by a path
|
||||
# of similar length.
|
||||
should_check_path_payload = self._should_check_path_payload(text)
|
||||
previous_text = self._prev_text
|
||||
self._sync_media_tracker_to_text(
|
||||
text, previous_text=previous_text, cursor_offset=self._get_cursor_offset()
|
||||
)
|
||||
self._prev_text = text
|
||||
self._sync_media_tracker_to_text(text)
|
||||
|
||||
# History handlers explicitly decide mode and stripped display text.
|
||||
# Skip mode detection here so recalled entries don't inherit stale mode.
|
||||
@@ -2328,6 +2372,16 @@ class ChatInput(Vertical):
|
||||
if prefix and not value.startswith(prefix):
|
||||
value = prefix + value
|
||||
|
||||
# Placeholder spans were captured against the raw draft; the transforms
|
||||
# above (whitespace strip, paste expansion, path substitution, prefix)
|
||||
# shifted offsets. Re-map spans onto the final submitted text so the
|
||||
# adapter strips the correct display token from the model-facing message
|
||||
# instead of a same-looking literal the user typed.
|
||||
if self._text_area is not None and self._image_tracker is not None:
|
||||
self._image_tracker.remap_spans_to_text(
|
||||
value, previous_text=self._text_area.text
|
||||
)
|
||||
|
||||
self._history.add(value)
|
||||
self.post_message(self.Submitted(value, mode))
|
||||
|
||||
@@ -2343,11 +2397,19 @@ class ChatInput(Vertical):
|
||||
self._next_paste_id = 1
|
||||
self.mode = "normal"
|
||||
|
||||
def _sync_media_tracker_to_text(self, text: str) -> None:
|
||||
def _sync_media_tracker_to_text(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
previous_text: str | None = None,
|
||||
cursor_offset: int | None = None,
|
||||
) -> None:
|
||||
"""Keep tracked media aligned with placeholder tokens in input text.
|
||||
|
||||
Args:
|
||||
text: Current text in the input area.
|
||||
previous_text: Previous text in the input area.
|
||||
cursor_offset: Current cursor offset in the input area.
|
||||
"""
|
||||
if not self._image_tracker:
|
||||
return
|
||||
@@ -2361,7 +2423,9 @@ class ChatInput(Vertical):
|
||||
else:
|
||||
self._skip_media_sync_events -= 1
|
||||
return
|
||||
self._image_tracker.sync_to_text(text)
|
||||
self._image_tracker.sync_to_text(
|
||||
text, previous_text=previous_text, cursor_offset=cursor_offset
|
||||
)
|
||||
|
||||
def on_chat_text_area_typing(
|
||||
self,
|
||||
@@ -2579,7 +2643,14 @@ class ChatInput(Vertical):
|
||||
media = get_media_from_path(path)
|
||||
if media is not None:
|
||||
kind = "image" if isinstance(media, ImageData) else "video"
|
||||
parts.append(self._image_tracker.add_media(media, kind))
|
||||
existing_text = self._text_area.text if self._text_area else raw_text
|
||||
parts.append(
|
||||
self._image_tracker.add_media(
|
||||
media,
|
||||
kind,
|
||||
existing_text=existing_text,
|
||||
)
|
||||
)
|
||||
attached = True
|
||||
continue
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ from textual.widgets.text_area import Selection
|
||||
from deepagents_code import _textual_patches as _textual_patches
|
||||
from deepagents_code.command_registry import SLASH_COMMANDS
|
||||
from deepagents_code.input import MediaTracker
|
||||
from deepagents_code.media_utils import ImageData
|
||||
from deepagents_code.media_utils import ImageData, create_multimodal_content
|
||||
from deepagents_code.widgets import chat_input as chat_input_module
|
||||
from deepagents_code.widgets.autocomplete import MAX_SUGGESTIONS
|
||||
from deepagents_code.widgets.chat_input import (
|
||||
@@ -2821,6 +2821,133 @@ class TestDroppedImagePaste:
|
||||
assert len(app.tracker.get_images()) == 1
|
||||
assert app.tracker.next_image_id == 2
|
||||
|
||||
async def test_typed_image_placeholder_is_not_atomic(self) -> None:
|
||||
"""Manually typed `[image N]` (no attachment) edits char-by-char.
|
||||
|
||||
Regression test: placeholder-shaped text the user typed must not be
|
||||
treated as an atomic media token, so backspace removes a single
|
||||
character instead of deleting the whole `[image 2]`.
|
||||
"""
|
||||
app = _ImagePasteApp()
|
||||
async with app.run_test() as pilot:
|
||||
chat = app.query_one(ChatInput)
|
||||
assert chat._text_area is not None
|
||||
|
||||
chat._text_area.text = "[image 2]"
|
||||
await pilot.pause()
|
||||
assert app.tracker.get_images() == []
|
||||
|
||||
chat._text_area.move_cursor((0, len("[image 2]")))
|
||||
await pilot.pause()
|
||||
await pilot.press("backspace")
|
||||
await pilot.pause()
|
||||
|
||||
assert chat._text_area.text == "[image 2"
|
||||
|
||||
async def test_typed_placeholder_not_atomic_alongside_real_image(
|
||||
self, tmp_path
|
||||
) -> None:
|
||||
"""A typed look-alike is char-editable while a real one stays atomic."""
|
||||
img_path = tmp_path / "real.png"
|
||||
from PIL import Image
|
||||
|
||||
image = Image.new("RGB", (4, 4), color="green")
|
||||
image.save(img_path, format="PNG")
|
||||
|
||||
app = _ImagePasteApp()
|
||||
async with app.run_test() as pilot:
|
||||
chat = app.query_one(ChatInput)
|
||||
assert chat._text_area is not None
|
||||
|
||||
chat.handle_external_paste(str(img_path))
|
||||
await pilot.pause()
|
||||
assert chat._text_area.text == "[image 1] "
|
||||
|
||||
# Append a manually typed placeholder-shaped token that is not
|
||||
# backed by any attachment.
|
||||
chat._text_area.text = "[image 1] [image 2]"
|
||||
await pilot.pause()
|
||||
assert len(app.tracker.get_images()) == 1
|
||||
|
||||
chat._text_area.move_cursor((0, len("[image 1] [image 2]")))
|
||||
await pilot.pause()
|
||||
await pilot.press("backspace")
|
||||
await pilot.pause()
|
||||
|
||||
# Only one character of the typed token is removed; the real
|
||||
# `[image 1]` placeholder is untouched and still tracked.
|
||||
assert chat._text_area.text == "[image 1] [image 2"
|
||||
assert len(app.tracker.get_images()) == 1
|
||||
|
||||
async def test_real_image_placeholder_still_atomic_with_typed_lookalike(
|
||||
self, tmp_path
|
||||
) -> None:
|
||||
"""The real `[image 1]` deletes atomically even beside a typed token."""
|
||||
img_path = tmp_path / "atomic.png"
|
||||
from PIL import Image
|
||||
|
||||
image = Image.new("RGB", (4, 4), color="purple")
|
||||
image.save(img_path, format="PNG")
|
||||
|
||||
app = _ImagePasteApp()
|
||||
async with app.run_test() as pilot:
|
||||
chat = app.query_one(ChatInput)
|
||||
assert chat._text_area is not None
|
||||
|
||||
chat.handle_external_paste(str(img_path))
|
||||
await pilot.pause()
|
||||
chat._text_area.text = "[image 2] [image 1]"
|
||||
await pilot.pause()
|
||||
assert len(app.tracker.get_images()) == 1
|
||||
|
||||
# Cursor just after the real trailing `[image 1]` token.
|
||||
chat._text_area.move_cursor((0, len("[image 2] [image 1]")))
|
||||
await pilot.pause()
|
||||
await pilot.press("backspace")
|
||||
await pilot.pause()
|
||||
|
||||
# The whole real placeholder is removed atomically, leaving the
|
||||
# typed look-alike intact.
|
||||
assert chat._text_area.text == "[image 2] "
|
||||
|
||||
async def test_submit_remaps_span_onto_stripped_value(self, tmp_path) -> None:
|
||||
"""`_submit_value` re-maps placeholder spans onto the final submitted text.
|
||||
|
||||
Regression: spans captured against the raw draft go stale when submit
|
||||
strips leading whitespace (and expands pastes), so the adapter would
|
||||
strip the wrong token from the model-facing message. The span must
|
||||
follow the transform.
|
||||
"""
|
||||
img_path = tmp_path / "submit.png"
|
||||
from PIL import Image
|
||||
|
||||
Image.new("RGB", (4, 4), color="navy").save(img_path, format="PNG")
|
||||
|
||||
app = _ImagePasteRecordingApp()
|
||||
async with app.run_test() as pilot:
|
||||
chat = app.query_one(ChatInput)
|
||||
assert chat._text_area is not None
|
||||
|
||||
chat.handle_external_paste(str(img_path))
|
||||
await pilot.pause()
|
||||
|
||||
# Leading whitespace shifts every offset when submit strips it.
|
||||
chat._text_area.text = " look [image 1]"
|
||||
await pilot.pause()
|
||||
img = app.tracker.get_images()[0]
|
||||
assert img.placeholder_span == (7, 16)
|
||||
|
||||
chat._submit_value(chat._text_area.text.strip())
|
||||
await pilot.pause()
|
||||
|
||||
assert app.submitted[-1].value == "look [image 1]"
|
||||
# The span now indexes the submitted value, not the raw draft.
|
||||
assert img.placeholder_span == (5, 14)
|
||||
content = create_multimodal_content(
|
||||
app.submitted[-1].value, app.tracker.get_images()
|
||||
)
|
||||
assert content[0]["text"] == "look"
|
||||
|
||||
async def test_handle_external_paste_attaches_dropped_image(self, tmp_path) -> None:
|
||||
"""External paste routing should attach dropped images."""
|
||||
img_path = tmp_path / "external.png"
|
||||
@@ -2895,6 +3022,31 @@ class TestDroppedImagePaste:
|
||||
assert chat._text_area.text.strip() == "[image 1]"
|
||||
assert len(app.tracker.get_images()) == 1
|
||||
|
||||
async def test_paste_image_path_skips_literal_placeholder_in_draft(
|
||||
self, tmp_path
|
||||
) -> None:
|
||||
"""Attaching media does not bind a literal placeholder already in text."""
|
||||
img_path = tmp_path / "drop.png"
|
||||
from PIL import Image
|
||||
|
||||
image = Image.new("RGB", (4, 4), color="blue")
|
||||
image.save(img_path, format="PNG")
|
||||
|
||||
app = _ImagePasteApp()
|
||||
async with app.run_test() as pilot:
|
||||
chat = app.query_one(ChatInput)
|
||||
assert chat._text_area is not None
|
||||
chat._text_area.text = "restore [image 1] "
|
||||
chat._text_area.move_cursor_to_end()
|
||||
|
||||
await chat._text_area._on_paste(events.Paste(str(img_path)))
|
||||
await pilot.pause()
|
||||
|
||||
assert chat._text_area.text == "restore [image 1] [image 2] "
|
||||
assert [img.placeholder for img in app.tracker.get_images()] == [
|
||||
"[image 2]"
|
||||
]
|
||||
|
||||
async def test_paste_non_image_path_keeps_original_text(self, tmp_path) -> None:
|
||||
"""Non-image dropped paths should keep the default path paste behavior."""
|
||||
file_path = tmp_path / "notes.txt"
|
||||
@@ -3269,6 +3421,29 @@ class TestDroppedVideoPaste:
|
||||
assert "[video" not in chat._text_area.text
|
||||
assert app.tracker.get_videos() == []
|
||||
|
||||
async def test_typed_video_placeholder_is_not_atomic(self) -> None:
|
||||
"""Manually typed `[video N]` (no attachment) edits char-by-char.
|
||||
|
||||
Mirrors the image look-alike guard for the video code path, which shares
|
||||
`_bound_media_placeholders` but was otherwise only tested for bound
|
||||
tokens.
|
||||
"""
|
||||
app = _ImagePasteApp()
|
||||
async with app.run_test() as pilot:
|
||||
chat = app.query_one(ChatInput)
|
||||
assert chat._text_area is not None
|
||||
|
||||
chat._text_area.text = "[video 2]"
|
||||
await pilot.pause()
|
||||
assert app.tracker.get_videos() == []
|
||||
|
||||
chat._text_area.move_cursor((0, len("[video 2]")))
|
||||
await pilot.pause()
|
||||
await pilot.press("backspace")
|
||||
await pilot.pause()
|
||||
|
||||
assert chat._text_area.text == "[video 2"
|
||||
|
||||
async def test_mixed_image_and_video_drop(self, tmp_path: Path) -> None:
|
||||
"""Dropping an image and video should produce both placeholder types."""
|
||||
from PIL import Image
|
||||
@@ -4816,6 +4991,28 @@ class TestPasteCollapseIntegration:
|
||||
# satisfy a `"[Pasted text #2]" not in text` assertion.
|
||||
assert chat._text_area.text == "[Pasted text #1]"
|
||||
|
||||
async def test_typed_paste_placeholder_is_not_atomic(self) -> None:
|
||||
"""A paste placeholder with no backing content edits char-by-char.
|
||||
|
||||
Regression test for the bound-token guard: `[Pasted text #99]` that the
|
||||
user typed (or a stale token whose id is absent from `_pasted_contents`)
|
||||
must not delete atomically, so backspace removes a single character.
|
||||
Without the `id not in pasted_ids` arm this whole token would vanish.
|
||||
"""
|
||||
app = _RecordingApp()
|
||||
async with app.run_test() as pilot:
|
||||
chat = app.query_one(ChatInput)
|
||||
assert chat._text_area is not None
|
||||
assert chat._pasted_contents == {}
|
||||
|
||||
chat._text_area.text = "[Pasted text #99]"
|
||||
chat._text_area.move_cursor((0, len("[Pasted text #99]")))
|
||||
await pilot.pause()
|
||||
await pilot.press("backspace")
|
||||
await pilot.pause()
|
||||
|
||||
assert chat._text_area.text == "[Pasted text #99"
|
||||
|
||||
async def test_backspace_removes_multiline_paste_placeholder(self) -> None:
|
||||
"""Backspace atomically deletes the `+M lines` placeholder variant."""
|
||||
multi_line = "\n".join(f"line {i}" for i in range(5))
|
||||
|
||||
@@ -20,6 +20,7 @@ from deepagents_code.media_utils import (
|
||||
get_clipboard_image,
|
||||
get_image_from_path,
|
||||
get_video_from_path,
|
||||
strip_media_placeholders,
|
||||
)
|
||||
|
||||
|
||||
@@ -111,6 +112,28 @@ class TestMediaTracker:
|
||||
|
||||
assert placeholder == "[image 1]"
|
||||
|
||||
def test_add_image_skips_placeholder_already_in_text(self) -> None:
|
||||
"""A literal placeholder in the draft is not rebound to new media."""
|
||||
tracker = MediaTracker()
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="")
|
||||
|
||||
placeholder = tracker.add_image(img, existing_text="restore [image 1]")
|
||||
|
||||
assert placeholder == "[image 2]"
|
||||
assert img.placeholder == "[image 2]"
|
||||
assert tracker.next_image_id == 3
|
||||
|
||||
def test_add_video_skips_placeholder_already_in_text(self) -> None:
|
||||
"""Video attachment IDs skip literal placeholders in the draft."""
|
||||
tracker = MediaTracker()
|
||||
vid = VideoData(base64_data="abc", format="mp4", placeholder="")
|
||||
|
||||
placeholder = tracker.add_video(vid, existing_text="restore [video 1]")
|
||||
|
||||
assert placeholder == "[video 2]"
|
||||
assert vid.placeholder == "[video 2]"
|
||||
assert tracker.next_video_id == 3
|
||||
|
||||
def test_sync_to_text_resets_when_placeholders_removed(self) -> None:
|
||||
"""Removing placeholders from input should clear tracked images and IDs."""
|
||||
tracker = MediaTracker()
|
||||
@@ -137,6 +160,120 @@ class TestMediaTracker:
|
||||
assert len(tracker.images) == 1
|
||||
assert tracker.images[0].placeholder == "[image 2]"
|
||||
|
||||
def test_sync_to_text_tracks_duplicate_inserted_before_placeholder(self) -> None:
|
||||
"""A typed duplicate before the display token does not steal the media span."""
|
||||
tracker = MediaTracker()
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="")
|
||||
tracker.add_image(img)
|
||||
tracker.sync_to_text("[image 1]")
|
||||
|
||||
text = "literal [image 1] then actual [image 1]"
|
||||
tracker.sync_to_text(text, previous_text="[image 1]", cursor_offset=30)
|
||||
|
||||
assert img.placeholder_span == (30, 39)
|
||||
|
||||
def test_sync_to_text_preserves_mapped_duplicate_after_prefix_edit(self) -> None:
|
||||
"""Edits before duplicates keep the mapped display token bound to media."""
|
||||
tracker = MediaTracker()
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="")
|
||||
tracker.add_image(img)
|
||||
tracker.sync_to_text("[image 1]")
|
||||
|
||||
previous_text = "literal [image 1] actual [image 1]"
|
||||
tracker.sync_to_text(
|
||||
previous_text,
|
||||
previous_text="[image 1]",
|
||||
cursor_offset=25,
|
||||
)
|
||||
|
||||
text = "Xliteral [image 1] actual [image 1]"
|
||||
tracker.sync_to_text(text, previous_text=previous_text, cursor_offset=1)
|
||||
result = create_multimodal_content(text, tracker.images)
|
||||
|
||||
assert img.placeholder_span == (26, 35)
|
||||
assert result[0]["text"] == "Xliteral [image 1] actual"
|
||||
|
||||
def test_sync_to_text_tracks_duplicate_inserted_after_placeholder(self) -> None:
|
||||
"""A typed duplicate after the display token does not steal the media span."""
|
||||
tracker = MediaTracker()
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="")
|
||||
tracker.add_image(img)
|
||||
tracker.sync_to_text("[image 1]")
|
||||
|
||||
text = "[image 1] literal [image 1]"
|
||||
tracker.sync_to_text(text, previous_text="[image 1]")
|
||||
|
||||
assert img.placeholder_span == (0, 9)
|
||||
|
||||
def test_sync_to_text_edit_between_placeholder_and_duplicate_keeps_actual(
|
||||
self,
|
||||
) -> None:
|
||||
"""Typing between an actual placeholder and a duplicate does not rebind it."""
|
||||
tracker = MediaTracker()
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="")
|
||||
tracker.add_image(img)
|
||||
tracker.sync_to_text("[image 1]")
|
||||
tracker.sync_to_text("[image 1] literal [image 1]", previous_text="[image 1]")
|
||||
|
||||
text = "[image 1] edited literal [image 1]"
|
||||
tracker.sync_to_text(
|
||||
text,
|
||||
previous_text="[image 1] literal [image 1]",
|
||||
cursor_offset=17,
|
||||
)
|
||||
|
||||
assert img.placeholder_span == (0, 9)
|
||||
|
||||
def test_remap_spans_to_text_shifts_span_and_strips_correct_duplicate(
|
||||
self,
|
||||
) -> None:
|
||||
"""Remapping keeps the real (second) token bound after an offset shift.
|
||||
|
||||
Regression: spans are captured against the draft but consumed after
|
||||
submit-time rewrites (whitespace trim, paste expansion) shift offsets.
|
||||
Without remapping, the stale span is discarded and the token-count
|
||||
fallback strips the *first* occurrence — the user's literal — leaving the
|
||||
real display token in the model-facing text. Remapping fixes the offset.
|
||||
"""
|
||||
tracker = MediaTracker()
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="")
|
||||
tracker.add_image(img)
|
||||
tracker.sync_to_text("[image 1]")
|
||||
|
||||
# A literal duplicate precedes the real display token in the draft.
|
||||
draft = "see [image 1] then real [image 1]"
|
||||
tracker.sync_to_text(draft, previous_text="[image 1]", cursor_offset=24)
|
||||
assert img.placeholder_span == (24, 33)
|
||||
|
||||
# Submit expands a paste before the token, shifting it right by 15 chars.
|
||||
final = "EXPANDED PASTE see [image 1] then real [image 1]"
|
||||
tracker.remap_spans_to_text(final, previous_text=draft)
|
||||
assert img.placeholder_span == (39, 48)
|
||||
|
||||
result = create_multimodal_content(final, tracker.get_images())
|
||||
assert result[0]["text"] == "EXPANDED PASTE see [image 1] then real"
|
||||
|
||||
def test_remap_spans_to_text_drops_span_when_token_edited(self) -> None:
|
||||
"""A span whose token was edited in the transformed text becomes None."""
|
||||
tracker = MediaTracker()
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="")
|
||||
tracker.add_image(img)
|
||||
tracker.sync_to_text("[image 1]")
|
||||
assert img.placeholder_span == (0, 9)
|
||||
|
||||
# The token characters themselves changed: cannot be cleanly mapped.
|
||||
tracker.remap_spans_to_text("[image 2]", previous_text="[image 1]")
|
||||
assert img.placeholder_span is None
|
||||
|
||||
def test_remap_spans_to_text_ignores_items_without_span(self) -> None:
|
||||
"""Items with no captured span are left as None (token-fallback path)."""
|
||||
tracker = MediaTracker()
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="[image 1]")
|
||||
tracker.images.append(img) # attached but never span-synced
|
||||
|
||||
tracker.remap_spans_to_text("[image 1]", previous_text="[image 1]")
|
||||
assert img.placeholder_span is None
|
||||
|
||||
|
||||
class TestEncodeImageToBase64:
|
||||
"""Tests for base64 encoding."""
|
||||
@@ -214,6 +351,238 @@ class TestCreateMultimodalContent:
|
||||
assert len(result) == 1
|
||||
assert result[0]["type"] == "image_url"
|
||||
|
||||
def test_placeholder_not_leaked_into_text_block(self) -> None:
|
||||
"""The `[image N]` display placeholder must not reach model-facing text.
|
||||
|
||||
Regression test: sending an image previously serialized the display
|
||||
placeholder as literal user-authored text in the traced/model message.
|
||||
"""
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="[image 1]")
|
||||
result = create_multimodal_content("[image 1] what's in this image?", [img])
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0]["type"] == "text"
|
||||
# Placeholder is gone but the surrounding user text is preserved.
|
||||
assert "[image 1]" not in result[0]["text"]
|
||||
assert result[0]["text"] == "what's in this image?"
|
||||
assert result[1]["type"] == "image_url"
|
||||
|
||||
def test_literal_duplicate_placeholder_preserved_in_text_block(self) -> None:
|
||||
"""Only the display placeholder occurrence is stripped from text."""
|
||||
img = ImageData(
|
||||
base64_data="abc",
|
||||
format="png",
|
||||
placeholder="[image 1]",
|
||||
placeholder_span=(0, 9),
|
||||
)
|
||||
result = create_multimodal_content(
|
||||
"[image 1] compare with literal [image 1]",
|
||||
[img],
|
||||
)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0]["type"] == "text"
|
||||
assert result[0]["text"] == "compare with literal [image 1]"
|
||||
assert result[1]["type"] == "image_url"
|
||||
|
||||
def test_literal_duplicate_before_placeholder_preserved_in_text_block(self) -> None:
|
||||
"""A literal duplicate before the display placeholder is preserved."""
|
||||
img = ImageData(
|
||||
base64_data="abc",
|
||||
format="png",
|
||||
placeholder="[image 1]",
|
||||
placeholder_span=(30, 39),
|
||||
)
|
||||
result = create_multimodal_content(
|
||||
"literal [image 1] then actual [image 1]",
|
||||
[img],
|
||||
)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0]["type"] == "text"
|
||||
assert result[0]["text"] == "literal [image 1] then actual"
|
||||
assert result[1]["type"] == "image_url"
|
||||
|
||||
def test_placeholder_removed_when_only_placeholder(self) -> None:
|
||||
"""A message that is only a placeholder yields no text block."""
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="[image 1]")
|
||||
result = create_multimodal_content("[image 1]", [img])
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["type"] == "image_url"
|
||||
|
||||
def test_placeholders_removed_for_video(self) -> None:
|
||||
"""Video placeholders are also stripped from model-facing text."""
|
||||
vid = VideoData(base64_data="vid", format="mp4", placeholder="[video 1]")
|
||||
result = create_multimodal_content("summarize [video 1] please", [], [vid])
|
||||
|
||||
assert result[0]["type"] == "text"
|
||||
assert "[video 1]" not in result[0]["text"]
|
||||
assert result[0]["text"] == "summarize please"
|
||||
|
||||
def test_multiple_placeholders_all_stripped(self) -> None:
|
||||
"""All placeholders are removed while surrounding text is preserved."""
|
||||
img1 = ImageData(base64_data="a", format="png", placeholder="[image 1]")
|
||||
img2 = ImageData(base64_data="b", format="png", placeholder="[image 2]")
|
||||
result = create_multimodal_content(
|
||||
"[image 1] and [image 2] differ how?", [img1, img2]
|
||||
)
|
||||
|
||||
assert result[0]["type"] == "text"
|
||||
assert "[image 1]" not in result[0]["text"]
|
||||
assert "[image 2]" not in result[0]["text"]
|
||||
assert result[0]["text"] == "and differ how?"
|
||||
|
||||
def test_mixed_image_and_video_placeholders_stripped(self) -> None:
|
||||
"""Image and video placeholders in one message are both stripped.
|
||||
|
||||
Exercises the combined `images + videos` placeholder assembly and the
|
||||
heterogeneous `|`-token strip together, which same-type cases miss.
|
||||
"""
|
||||
img = ImageData(base64_data="a", format="png", placeholder="[image 1]")
|
||||
vid = VideoData(base64_data="v", format="mp4", placeholder="[video 1]")
|
||||
result = create_multimodal_content("[image 1] vs [video 1]", [img], [vid])
|
||||
|
||||
assert result[0]["type"] == "text"
|
||||
assert result[0]["text"] == "vs"
|
||||
# Both media blocks follow the single text block.
|
||||
assert result[1]["type"] == "image_url"
|
||||
assert result[2]["type"] == "video"
|
||||
|
||||
def test_unbound_placeholder_like_text_preserved(self) -> None:
|
||||
"""Placeholder-shaped text not bound to attached media is preserved.
|
||||
|
||||
Regression test for false positives: a user attaches image 1 but their
|
||||
prompt also literally mentions `[image 2]` (which is not attached). Only
|
||||
the real `[image 1]` token is stripped; the literal `[image 2]` stays.
|
||||
"""
|
||||
img = ImageData(base64_data="abc", format="png", placeholder="[image 1]")
|
||||
result = create_multimodal_content(
|
||||
"[image 1] see the note about [image 2] in the docs", [img]
|
||||
)
|
||||
|
||||
assert result[0]["type"] == "text"
|
||||
assert result[0]["text"] == "see the note about [image 2] in the docs"
|
||||
|
||||
def test_placeholder_like_text_without_attachment_untouched(self) -> None:
|
||||
"""With no media attached, placeholder-shaped text is never stripped."""
|
||||
result = create_multimodal_content("compare [image 1] vs [image 2]", [])
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["type"] == "text"
|
||||
assert result[0]["text"] == "compare [image 1] vs [image 2]"
|
||||
|
||||
|
||||
class TestStripMediaPlaceholders:
|
||||
"""Tests for `strip_media_placeholders`."""
|
||||
|
||||
def test_leading_placeholder(self) -> None:
|
||||
"""A leading placeholder is removed along with its trailing space."""
|
||||
assert strip_media_placeholders("[image 1] hello", ["[image 1]"]) == "hello"
|
||||
|
||||
def test_inline_placeholder_no_double_space(self) -> None:
|
||||
"""An inline placeholder does not leave a double space behind."""
|
||||
assert (
|
||||
strip_media_placeholders("before [image 1] after", ["[image 1]"])
|
||||
== "before after"
|
||||
)
|
||||
|
||||
def test_duplicate_literal_placeholder_preserved(self) -> None:
|
||||
"""Duplicate literal text is preserved when one matching media is attached."""
|
||||
assert (
|
||||
strip_media_placeholders(
|
||||
"[image 1] describe literal [image 1]",
|
||||
["[image 1]"],
|
||||
placeholder_spans=[(0, 9)],
|
||||
)
|
||||
== "describe literal [image 1]"
|
||||
)
|
||||
|
||||
def test_literal_duplicate_before_placeholder_preserved(self) -> None:
|
||||
"""Span tracking strips the display token, not the first duplicate."""
|
||||
assert (
|
||||
strip_media_placeholders(
|
||||
"literal [image 1] then actual [image 1]",
|
||||
["[image 1]"],
|
||||
placeholder_spans=[(30, 39)],
|
||||
)
|
||||
== "literal [image 1] then actual"
|
||||
)
|
||||
|
||||
def test_only_bound_placeholders_removed(self) -> None:
|
||||
"""Only tokens in the bound set are removed; look-alikes are preserved."""
|
||||
result = strip_media_placeholders(
|
||||
"keep [image 2] drop [image 1]", ["[image 1]"]
|
||||
)
|
||||
assert result == "keep [image 2] drop"
|
||||
|
||||
def test_stale_span_is_discarded_and_falls_back_to_token(self) -> None:
|
||||
"""A span whose slice no longer matches the token is ignored, not used.
|
||||
|
||||
Guards the graceful-degradation contract: an out-of-date span (here the
|
||||
offset points at non-token text) must not delete arbitrary characters;
|
||||
the token fallback removes the real occurrence instead.
|
||||
"""
|
||||
result = strip_media_placeholders(
|
||||
"hello [image 1] world",
|
||||
["[image 1]"],
|
||||
placeholder_spans=[(0, 5)], # points at "hello", not the token
|
||||
)
|
||||
assert result == "hello world"
|
||||
|
||||
def test_ambiguous_fallback_strips_leading_occurrence(self) -> None:
|
||||
"""With no span and duplicate tokens, the fallback strips the first one."""
|
||||
result = strip_media_placeholders(
|
||||
"first [image 1] second [image 1]",
|
||||
["[image 1]"],
|
||||
)
|
||||
assert result == "first second [image 1]"
|
||||
|
||||
def test_mixed_image_and_video_tokens_removed(self) -> None:
|
||||
"""Heterogeneous image and video tokens are stripped in one pass."""
|
||||
result = strip_media_placeholders(
|
||||
"[image 1] and [video 1] together", ["[image 1]", "[video 1]"]
|
||||
)
|
||||
assert result == "and together"
|
||||
|
||||
def test_text_without_placeholder_unchanged(self) -> None:
|
||||
"""Text without a bound placeholder is unchanged (aside from trim)."""
|
||||
assert (
|
||||
strip_media_placeholders("plain text only", ["[image 1]"])
|
||||
== "plain text only"
|
||||
)
|
||||
|
||||
def test_no_placeholders_returns_text_verbatim(self) -> None:
|
||||
"""An empty placeholder set returns the text unchanged, including whitespace."""
|
||||
assert strip_media_placeholders(" [image 1] ", []) == " [image 1] "
|
||||
|
||||
def test_only_placeholder_becomes_empty(self) -> None:
|
||||
"""A string that is only a bound placeholder becomes empty."""
|
||||
assert strip_media_placeholders("[video 2]", ["[video 2]"]) == ""
|
||||
|
||||
def test_newlines_preserved(self) -> None:
|
||||
"""Newlines around a placeholder are preserved."""
|
||||
assert (
|
||||
strip_media_placeholders("line1\n[image 1]\nline2", ["[image 1]"])
|
||||
== "line1\n\nline2"
|
||||
)
|
||||
|
||||
def test_special_regex_chars_in_placeholder_are_escaped(self) -> None:
|
||||
"""Placeholder tokens are treated literally, not as regex."""
|
||||
assert strip_media_placeholders("a [image (1)] b", ["[image (1)]"]) == "a b"
|
||||
|
||||
def test_indentation_preserved_after_placeholder(self) -> None:
|
||||
r"""Leading newlines and code indentation after a placeholder survive.
|
||||
|
||||
Regression: ``.strip()`` removed all leading whitespace, so attaching
|
||||
an image before indented code (``[image 1]\n def foo():``) would
|
||||
lose the four-space indent. Only spaces/tabs on each edge are trimmed.
|
||||
"""
|
||||
assert (
|
||||
strip_media_placeholders("[image 1]\n def foo():", ["[image 1]"])
|
||||
== "\n def foo():"
|
||||
)
|
||||
|
||||
|
||||
class TestGetClipboardImage:
|
||||
"""Tests for clipboard image detection."""
|
||||
|
||||
Reference in New Issue
Block a user