mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 06:15:58 -10:00
serve: damaged tool-call arguments, a stray closing think tag and a missing </function> no longer cost the call (#4427 #4077)
#4427: tool calls came back as text or with a wrong argument. - A JSON array or object argument with a slip in it (a trailing comma, single quotes, a bare string between two members, brackets left open) is read as written instead of reaching the client as a string that fails its schema; the reporter's todo_write array with "pending", "in_progress" in one object is the test. Valid JSON is untouched; text that is no array or object stays text. The repair is counted (/metrics strata:tool_args_repaired_total) and logged. - A parameter's </parameter> followed by </tool_call> ends the value: the call whose </function> was left out is a call. - The same holds for a call written inside the thinking, and one whose output ends on </function> inside the thinking; a following opener confirms an omitted </tool_call> there too, as in the answer. - The call announced while streaming and finished by the end of the output carried empty arguments in its final event (read by MCP and the Responses assembler): it now carries the parsed ones. #4077: after the reasoning ended at a closing think tag, one more at the start of a line in the answer is the stray: the model quoted the tag while thinking. It is dropped and the answer's code tracking starts over, so a call that follows is not taken for a quote by a fence or backtick the reasoning left open. The reasoning after the stray is still sent as the answer (a stream cannot take it back); the call arrives. Logged and counted. Adds serve/test_tool_corpus.py (49 specimens: the reporters' examples, the Qwen3.8-Flash-Next form, the drift shapes, DeepSeek DSML; each whole, in pieces of 1, 3 and 7 bytes and cut in two at every byte, streaming on and off) and serve/test_tool_routes.py (Chat, Messages and Responses, whole and streamed).
This commit is contained in:
Niko1221 committed
1 parent
b72f56e02a
commit
d71132738e
4 files changed
+661
-23
No files matched your search
+211
-23
@@ -754,7 +754,8 @@ _QUOTED_OPEN = re.compile(r"<function=[^<>\s]+>\s*(?:<parameter=|</function>)")
|
||||
|
||||
def param_end(text: str, final: bool = False, name: str | None = None) -> int:
|
||||
"""Where a parameter value in `text` ends: the first PARAM_END followed (after whitespace) by the next
|
||||
PARAM_START or FUNC_END - the same text inside a value (a file that documents the call format, #210) is
|
||||
PARAM_START or FUNC_END, or by the wrapper's CALL_END when the call's `</function>` was left out (#4427) - the
|
||||
same text inside a value (a file that documents the call format, #210) is
|
||||
part of the value. #4078: a close inside a call the value itself quotes (more _QUOTED_OPEN than FUNC_END before
|
||||
it), or one followed by the same parameter `name` again, is part of the value too. `text` starts at the value.
|
||||
-1: none yet; -2: a candidate whose follower has not arrived (streaming; `final` accepts it)."""
|
||||
@@ -768,9 +769,9 @@ def param_end(text: str, final: bool = False, name: str | None = None) -> int:
|
||||
return at if final else -2
|
||||
if name is None or after[len(PARAM_START):gt].strip() != name:
|
||||
return at
|
||||
elif after.startswith(FUNC_END):
|
||||
elif after.startswith((FUNC_END, CALL_END)): # #4427: the model left out its `</function>`
|
||||
return at
|
||||
elif not after or PARAM_START.startswith(after) or FUNC_END.startswith(after):
|
||||
elif not after or PARAM_START.startswith(after) or FUNC_END.startswith(after) or CALL_END.startswith(after):
|
||||
return at if final else -2
|
||||
at = text.find(PARAM_END, at + 1)
|
||||
return -1
|
||||
@@ -916,7 +917,151 @@ def json_tool_call(body: str, schemas: dict) -> ToolCall | None:
|
||||
return ToolCall(name=obj["name"], arguments=args) if isinstance(args, dict) else None
|
||||
|
||||
|
||||
def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
|
||||
LENIENT_MAX = 1 << 20 # the longest value lenient_json will try (it is a python loop)
|
||||
|
||||
|
||||
class _Lenient:
|
||||
"""A JSON reader for the slips a model makes inside an array or object argument (#4427): a trailing or doubled comma,
|
||||
single quotes and Python's True/False/None, a bare string between two members of an object (`"status": "pending",
|
||||
"in_progress", "pending": false`), brackets left open at the end, extra closers at the end. It is only asked about
|
||||
text json.loads has refused. It never guesses at a value: a string cut short, an unknown word, a missing colon
|
||||
after a key, anything else raises ValueError."""
|
||||
|
||||
_WORDS = {"true": True, "false": False, "null": None, "True": True, "False": False, "None": None}
|
||||
_ESC = {'"': '"', "'": "'", "\\": "\\", "/": "/", "b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t"}
|
||||
_TOKEN = re.compile(r"[^\s,\]}:]+")
|
||||
|
||||
def __init__(self, text: str):
|
||||
self.s, self.i = text, 0
|
||||
|
||||
def peek(self) -> str:
|
||||
s, n = self.s, len(self.s)
|
||||
while self.i < n and s[self.i] in " \t\r\n":
|
||||
self.i += 1
|
||||
return s[self.i:self.i + 1]
|
||||
|
||||
def string(self) -> str:
|
||||
s, q = self.s, self.s[self.i]
|
||||
self.i += 1
|
||||
out = []
|
||||
while self.i < len(s):
|
||||
c = s[self.i]
|
||||
if c == q:
|
||||
self.i += 1
|
||||
return "".join(out)
|
||||
if c == "\\":
|
||||
self.i += 1
|
||||
e = s[self.i:self.i + 1]
|
||||
if e == "u" and re.fullmatch(r"[0-9a-fA-F]{4}", s[self.i + 1:self.i + 5]):
|
||||
out.append(chr(int(s[self.i + 1:self.i + 5], 16)))
|
||||
self.i += 5
|
||||
continue
|
||||
if e not in self._ESC:
|
||||
raise ValueError("bad escape")
|
||||
out.append(self._ESC[e])
|
||||
else:
|
||||
out.append(c)
|
||||
self.i += 1
|
||||
raise ValueError("string not closed")
|
||||
|
||||
def value(self, depth: int = 0):
|
||||
if depth > 64:
|
||||
raise ValueError("too deep")
|
||||
c = self.peek()
|
||||
if c == "{":
|
||||
return self.obj(depth + 1)
|
||||
if c == "[":
|
||||
return self.arr(depth + 1)
|
||||
if c in ('"', "'"):
|
||||
return self.string()
|
||||
m = self._TOKEN.match(self.s, self.i)
|
||||
if not m:
|
||||
raise ValueError("no value")
|
||||
self.i = m.end()
|
||||
if m.group() in self._WORDS:
|
||||
return self._WORDS[m.group()]
|
||||
return json.loads(m.group()) # a number, or ValueError
|
||||
|
||||
def obj(self, depth: int):
|
||||
self.i += 1
|
||||
out: dict = {}
|
||||
while True:
|
||||
c = self.peek()
|
||||
if not c or c == "}": # (no closer: the brackets were left open at the end)
|
||||
self.i += 1
|
||||
return out
|
||||
if c == ",":
|
||||
self.i += 1
|
||||
continue
|
||||
if c not in ('"', "'"):
|
||||
raise ValueError("object member")
|
||||
key = self.string()
|
||||
if self.peek() == ":":
|
||||
self.i += 1
|
||||
out[key] = self.value(depth)
|
||||
elif self.peek() not in (",", "}", ""): # a bare string where a member belongs is dropped
|
||||
raise ValueError("object member")
|
||||
|
||||
def arr(self, depth: int):
|
||||
self.i += 1
|
||||
out: list = []
|
||||
while True:
|
||||
c = self.peek()
|
||||
if not c or c == "]":
|
||||
self.i += 1
|
||||
return out
|
||||
if c == ",":
|
||||
self.i += 1
|
||||
continue
|
||||
out.append(self.value(depth))
|
||||
if self.peek() not in (",", "]", ""):
|
||||
raise ValueError("array element")
|
||||
|
||||
|
||||
def lenient_json(text: str):
|
||||
"""The JSON array or object `text` was meant to be, read past the slips of _Lenient; ValueError for anything else
|
||||
(and for text that is not an array or object). Valid JSON is not its business: callers try json.loads first."""
|
||||
t = text.strip()
|
||||
m = _JSON_FENCE.match(t)
|
||||
if m:
|
||||
t = t[m.end():].rstrip()
|
||||
t = t[:-3].rstrip() if t.endswith("```") else t
|
||||
if not t.startswith(("[", "{")) or len(t) > LENIENT_MAX:
|
||||
raise ValueError("not an array or object")
|
||||
r = _Lenient(t)
|
||||
v = r.value()
|
||||
if t[r.i:].strip("]}) \t\r\n"):
|
||||
raise ValueError("text after the value")
|
||||
return v
|
||||
|
||||
|
||||
def coerce_value(value: str, declared, notes: list | None = None):
|
||||
"""A non-string parameter's text -> its value by the tool's declared type: JSON when it parses; else an array or an
|
||||
object read with lenient_json (a note says so in `notes`); else the text itself, as before. An array or object that
|
||||
arrived as a JSON string of itself is decoded once more."""
|
||||
if declared == "string":
|
||||
return value
|
||||
try:
|
||||
v = json.loads(value)
|
||||
except ValueError:
|
||||
if not value.lstrip().startswith(("[", "{", "`")) or (isinstance(declared, list) and "string" in declared):
|
||||
return value
|
||||
try:
|
||||
v = lenient_json(value)
|
||||
except (ValueError, RecursionError):
|
||||
return value
|
||||
if notes is not None:
|
||||
notes.append("a damaged JSON " + ("array" if isinstance(v, list) else "object") + " argument was read as written")
|
||||
return v
|
||||
if isinstance(v, str) and declared in ("array", "object") and v.lstrip().startswith(("[", "{")):
|
||||
try:
|
||||
return json.loads(v)
|
||||
except ValueError:
|
||||
return coerce_value(v, declared, notes)
|
||||
return v
|
||||
|
||||
|
||||
def parse_tool_call(body: str, schema: dict | None = None, notes: list | None = None) -> ToolCall:
|
||||
"""`<function=NAME>\\n<parameter=P>\\nVALUE\\n</parameter>...</function>` -> ToolCall. Values are JSON-decoded
|
||||
when the tool's schema says the parameter is not a string (or, without a schema, when they parse as JSON
|
||||
objects/arrays/numbers/booleans)."""
|
||||
@@ -938,14 +1083,7 @@ def parse_tool_call(body: str, schema: dict | None = None) -> ToolCall:
|
||||
value = value[1:]
|
||||
if value.endswith("\n"):
|
||||
value = value[:-1]
|
||||
declared = (props.get(pname) or {}).get("type")
|
||||
if declared == "string":
|
||||
args[pname] = value
|
||||
else:
|
||||
try:
|
||||
args[pname] = json.loads(value)
|
||||
except ValueError:
|
||||
args[pname] = value
|
||||
args[pname] = coerce_value(value, (props.get(pname) or {}).get("type"), notes)
|
||||
return ToolCall(name=name, arguments=args)
|
||||
|
||||
|
||||
@@ -1000,6 +1138,9 @@ class OutputParser:
|
||||
# reasoning text before them is tracked (code fence, inline code, current line) to tell an act from a quote.
|
||||
self.pending: list[list] = []
|
||||
self.rescued = 0 # calls delivered from the reasoning
|
||||
self.notes: list[str] = [] # #4427: what was repaired in the arguments of the calls (a damaged JSON array or object)
|
||||
self.after_think = False # #4077: the reasoning ended at a bare `</think>` (a second one in the answer is a stray)
|
||||
self.stray_think = 0 # ... and the strays of that kind dropped from the answer
|
||||
self.refused = 0 # declared calls kept as reasoning text (quoted, or the turn was cut)
|
||||
self.fence, self.line, self.ticks = "", "", 0
|
||||
# recover (opt-in, the config's "tool_call_recovery"): a declared tool's call written next to the template's
|
||||
@@ -1165,6 +1306,18 @@ class OutputParser:
|
||||
return None
|
||||
return after[0] not in "\r\n"
|
||||
|
||||
def _stray_think_at(self) -> int:
|
||||
"""#4077: where self.buf has a `</think>` at the start of a line, in the answer of a turn whose reasoning
|
||||
already ended at one; -1 for none. A fence or backtick is no reason to keep it: the reasoning may have left
|
||||
one open, and the answer's own tracking of code is wrong from there on."""
|
||||
k = self.buf.find(THINK_END)
|
||||
while k >= 0:
|
||||
before = self.buf[:k]
|
||||
if not (before.rsplit("\n", 1)[1] if "\n" in before else self.line + before).strip():
|
||||
return k
|
||||
k = self.buf.find(THINK_END, k + 1)
|
||||
return -1
|
||||
|
||||
def _release(self, deliver: bool) -> list[Event]:
|
||||
"""Settle the calls waiting in self.pending: events for real calls, or all of it back as reasoning text."""
|
||||
out = []
|
||||
@@ -1290,10 +1443,7 @@ class OutputParser:
|
||||
value = value[1:]
|
||||
if value.endswith("\n"):
|
||||
value = value[:-1]
|
||||
try:
|
||||
v = json.loads(value)
|
||||
except ValueError:
|
||||
v = value
|
||||
v = coerce_value(value, self.sdeclared.get(self.spname)) # the note comes with the final call (parse_tool_call)
|
||||
args(json.dumps(v, ensure_ascii=False))
|
||||
self.sp += end + len(PARAM_END)
|
||||
self.ss = "between"
|
||||
@@ -1342,22 +1492,26 @@ class OutputParser:
|
||||
# as a call) until it ends: a declared name is then a tool call, anything else stays reasoning text.
|
||||
body = self.buf[len(CALL_START):]
|
||||
end, think = call_end(body), body.find(THINK_END)
|
||||
drop = len(CALL_END)
|
||||
f = function_end(body) # #4427: as in the answer, a following call opener confirms an omitted closer
|
||||
if f >= 0 and (end < 0 or f < end) and re.match(r"<tool_call>\s*<function=\s*[^<>\s]+\s*>", body[f:].lstrip()):
|
||||
end, drop = f, 0
|
||||
if end >= 0 and (think < 0 or think >= end):
|
||||
call = None
|
||||
try:
|
||||
name = body[:end].strip()[len("<function="):].split(">", 1)[0].strip()
|
||||
name = self.aliases.get(name, name)
|
||||
if name in self.schemas:
|
||||
call = parse_tool_call(body[:end], self.schemas.get(name))
|
||||
call = parse_tool_call(body[:end], self.schemas.get(name), self.notes)
|
||||
call.name = name
|
||||
except ValueError:
|
||||
pass
|
||||
raw = self.buf[:len(CALL_START) + end + len(CALL_END)]
|
||||
raw = self.buf[:len(CALL_START) + end + drop]
|
||||
if call is not None or self.pending:
|
||||
self.pending.append([raw, call]) # settled by what follows it (see __init__)
|
||||
else:
|
||||
out.append(Event("reasoning", self._track(raw)))
|
||||
self.buf = body[end + len(CALL_END):]
|
||||
self.buf = body[end + drop:]
|
||||
self.state = "reasoning"
|
||||
elif think >= 0: # the thinking ended inside it: it never was a call
|
||||
out += self._release(False)
|
||||
@@ -1421,7 +1575,7 @@ class OutputParser:
|
||||
if i:
|
||||
out.append(Event("reasoning", self._track(self.buf[:i])))
|
||||
self.buf = self.buf[i + len(THINK_END):]
|
||||
self.state, self.lead = "content", True
|
||||
self.state, self.lead, self.after_think = "content", True, True
|
||||
self.fence, self.line, self.ticks = "", "", 0 # the answer's own text starts here
|
||||
elif self.state == "content":
|
||||
if self.lead: # newlines right after </think> or a call
|
||||
@@ -1431,6 +1585,19 @@ class OutputParser:
|
||||
return out
|
||||
self.buf, self.lead = stripped, False
|
||||
i = self.buf.find(CALL_START)
|
||||
stray = self._stray_think_at() if self.after_think else -1
|
||||
if stray >= 0 and (i < 0 or stray < i):
|
||||
# #4077: the reasoning ended at a `</think>` the model wrote while thinking (quoting the format on a
|
||||
# line of its own), and this is the one that ends it. What came between was reasoning, already sent
|
||||
# as the answer; the tag is dropped and the text after it starts over, so a call that follows is
|
||||
# not taken for a quote by the code fence or backtick the reasoning left open.
|
||||
if stray:
|
||||
out.append(Event("content", self._track(self.buf[:stray])))
|
||||
self.buf = self.buf[stray + len(THINK_END):]
|
||||
self.fence, self.line, self.ticks = "", "", 0
|
||||
self.lead = True
|
||||
self.stray_think += 1
|
||||
continue
|
||||
b, decided = self._bare_opener() if self.recover and self.schemas else (-1, False)
|
||||
if b >= 0 and (i < 0 or b < i):
|
||||
# recover: <function=NAME> alone at the start of a line - the call without its <tool_call>.
|
||||
@@ -1449,7 +1616,7 @@ class OutputParser:
|
||||
if i < 0:
|
||||
# Hold a partial tag AND the newlines before it: if a tool call follows, they are dropped,
|
||||
# so emitting them early would make streamed and whole outputs differ.
|
||||
j = len(self.buf) - self._hold(self.buf, (CALL_START,))
|
||||
j = len(self.buf) - self._hold(self.buf, (CALL_START, THINK_END) if self.after_think else (CALL_START,))
|
||||
if self.recover and self.schemas: # a line that may be becoming a bare <function=
|
||||
k = self.buf.rfind("\n") + 1
|
||||
tail = self.buf[k:]
|
||||
@@ -1610,6 +1777,21 @@ class OutputParser:
|
||||
how the turn ended: calls waiting from the reasoning become real calls only on a natural stop (None = stop);
|
||||
a turn cut by max tokens (or cancelled, or failed) keeps them as reasoning text (#1058)."""
|
||||
out = []
|
||||
if self.state == "rcall" and reason in (None, "stop"):
|
||||
# #4427: a call in the reasoning whose `</function>` is the end of the output (the `</tool_call>` left out)
|
||||
# is as whole as one in the answer: it waits for the turn's end like the others
|
||||
body = self.buf[len(CALL_START):]
|
||||
f = function_end(body)
|
||||
name = body.lstrip()[len(FUNC_START):].split(">", 1)[0].strip() if f >= 0 else ""
|
||||
name = self.aliases.get(name, name)
|
||||
if f >= 0 and not body[f:].strip() and name in self.schemas:
|
||||
try:
|
||||
call = parse_tool_call(body[:f], self.schemas.get(name), self.notes)
|
||||
call.name = name
|
||||
self.pending.append([self.buf, call])
|
||||
self.buf, self.state = "", "reasoning"
|
||||
except ValueError:
|
||||
pass
|
||||
if self.pending:
|
||||
out += self._release(reason in (None, "stop") and self.state == "reasoning" and not self.buf)
|
||||
if self.state == "call" and self.bare and function_end(self.buf) >= 0:
|
||||
@@ -1625,7 +1807,13 @@ class OutputParser:
|
||||
if self.ss == "function_done" and not self.buf[self.sp:].strip() and reason in (None, "stop"):
|
||||
# A natural end immediately after </function> retains native streaming recovery.
|
||||
out += self._close_scan()
|
||||
out.append(Event("tool_call", call=self.scall))
|
||||
call = self.scall
|
||||
try: # the final event carries the call's arguments, as the one _finish_call sends does
|
||||
call = parse_tool_call(self.buf, self.schemas.get(call.name), self.notes)
|
||||
call.name, call.id = self.scall.name, self.scall.id
|
||||
except ValueError:
|
||||
pass
|
||||
out.append(Event("tool_call", call=call))
|
||||
elif self.ss == "function_done":
|
||||
out.append(Event("content", self._track(CALL_START + self.buf)))
|
||||
self.buf = ""
|
||||
@@ -1651,7 +1839,7 @@ class OutputParser:
|
||||
try:
|
||||
if raw_name != raw_name.strip() and name not in self.schemas:
|
||||
raise ValueError("whitespace tool name is not declared")
|
||||
call = parse_tool_call(body, self.schemas.get(name))
|
||||
call = parse_tool_call(body, self.schemas.get(name), self.notes)
|
||||
call.name = name
|
||||
except ValueError:
|
||||
if not self.recover:
|
||||
|
||||
@@ -4813,6 +4813,8 @@ class Service:
|
||||
("strata:drafts_accepted_total", "counter", "Speculative draft tokens accepted", t["drafts_accepted"]),
|
||||
("strata:tool_calls_unread_total", "counter", "<tool_call> blocks returned as text (an undeclared name or a form the template does not use)",
|
||||
t.get("tool_calls_unread", 0)),
|
||||
("strata:tool_args_repaired_total", "counter", "Tool call arguments (a JSON array or object) that were damaged and read as written",
|
||||
t.get("tool_args_repaired", 0)),
|
||||
("strata:model_loaded", "gauge", "1 while the model is loaded", int(self.loaded())),
|
||||
]
|
||||
return "".join(f"# HELP {name} {help_}\n# TYPE {name} {kind}\n{name} {value}\n"
|
||||
@@ -5759,6 +5761,14 @@ class Service:
|
||||
self.totals["tool_calls_from_reasoning"] = self.totals.get("tool_calls_from_reasoning", 0) + parser.rescued
|
||||
print(f"[strata] tool calls inside the thinking: {parser.rescued} read as calls, {parser.refused} kept as "
|
||||
f"reasoning (quoted, or the turn did not end on a stop)", flush=True)
|
||||
if getattr(parser, "stray_think", 0):
|
||||
# #4077: the model wrote a closing think tag while thinking; the reasoning after it was sent as the answer
|
||||
self.totals["stray_think_closers"] = self.totals.get("stray_think_closers", 0) + parser.stray_think
|
||||
print(f"[strata] {parser.stray_think} extra closing think tag(s) in the answer were dropped (the model wrote one "
|
||||
"while thinking, so the rest of its reasoning went out as the answer; a call after it still counts)", flush=True)
|
||||
if getattr(parser, "notes", None):
|
||||
self.totals["tool_args_repaired"] = self.totals.get("tool_args_repaired", 0) + len(parser.notes)
|
||||
print(f"[strata] {len(parser.notes)} tool call argument(s) repaired: {'; '.join(sorted(set(parser.notes)))}", flush=True)
|
||||
rejected = []
|
||||
if parser.unread:
|
||||
named = getattr(parser, "unread_names", None)
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
"""A corpus of model outputs for the tool-call and reasoning parser (#4427, #4078, #4077).
|
||||
|
||||
Each specimen is the text a model wrote, the tools the request declared, and what a client must get: the calls (name and
|
||||
arguments), the answer text, the reasoning text. The sources are named in each entry: the reporters' own examples in
|
||||
the issues, the Qwen3.8-Flash-Next form (`<tool_call><function=NAME><parameter=P>`), the drift shapes of the 1,462-turn
|
||||
corpus the recovery switch was written for, and DeepSeek-V4.1's DSML form.
|
||||
|
||||
Every specimen is read whole, one byte at a time, in pieces of 3 and 7 bytes, and cut in two at every byte; with
|
||||
streaming of the arguments on and off. All of them must give the same events, and the streamed arguments of a call
|
||||
must be its final arguments.
|
||||
|
||||
python -m unittest serve.test_tool_corpus -v
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from serve.dsml import DsmlOutputParser # noqa: E402
|
||||
from serve.frontend import OutputParser # noqa: E402
|
||||
|
||||
|
||||
def tool(name, **props):
|
||||
return {"name": name, "parameters": {"type": "object", "properties": props}}
|
||||
|
||||
|
||||
S, I, A, O = {"type": "string"}, {"type": "integer"}, {"type": "array"}, {"type": "object"}
|
||||
TOOLS = [tool("todo_write", todos=A), tool("run_command", command=S, timeout=I), tool("write_file", path=S, content=S),
|
||||
tool("read", path=S), tool("bash", command=S), tool("edit", path=S, edits=A, options=O)]
|
||||
|
||||
|
||||
def qwen(name, **params):
|
||||
"""The call as Qwen3.8-Flash-Next writes it (the chat template's own form)."""
|
||||
body = "".join(f"<parameter={k}>\n{v}\n</parameter>\n" for k, v in params.items())
|
||||
return f"<tool_call>\n<function={name}>\n{body}</function>\n</tool_call>"
|
||||
|
||||
|
||||
def dsml(*calls):
|
||||
"""DeepSeek-V4.1's DSML block: calls are (name, [(parameter, value, is_string)])."""
|
||||
out = ["<｜DSML｜ calls>\n"]
|
||||
for name, params in calls:
|
||||
out.append(f'<｜DSML｜ invoke name="{name}">\n')
|
||||
for k, v, is_str in params:
|
||||
out.append(f'<｜DSML｜ parameter name="{k}" string="{str(is_str).lower()}">{v}</｜DSML｜ parameter>\n')
|
||||
out.append("</｜DSML｜ invoke>\n")
|
||||
return "".join(out) + "</｜DSML｜ calls>"
|
||||
|
||||
|
||||
NESTED = qwen("read", path="nested.txt")
|
||||
HEREDOC = f"python3 - <<'PY'\ntext = '''{NESTED}'''\nprint('tail that must not be lost')\nPY"
|
||||
REOPEN = "first line\n</parameter>\n<parameter=content>\nlast line"
|
||||
TODO_OK = ('[{"content": "Generate lorem ipsum page texture atlas (PIL) + inspect", "status": "in_progress"}, '
|
||||
'{"content": "Blender script: book geometry + materials", "status": "pending"}]')
|
||||
TODO_STRAY = ('[{"content": "Init git repo with LFS", "status": "in_progress", "pending": false, "completed": false}, '
|
||||
'{"content": "Write Blender build script", "status": "pending", "in_progress", "pending": false, '
|
||||
'"completed": false}]')
|
||||
TODO_STRAY_WANT = [{"content": "Init git repo with LFS", "status": "in_progress", "pending": False, "completed": False},
|
||||
{"content": "Write Blender build script", "status": "pending", "pending": False, "completed": False}]
|
||||
STRAY_VALUE = "reasoning sample:\n</think>\nanswer sample"
|
||||
|
||||
# name: dict(text, calls, content, reasoning, thinking, recover, finish, tools, unread, notes, source)
|
||||
CORPUS = {}
|
||||
|
||||
|
||||
def add(name, text, calls=(), content="", reasoning="", thinking=False, recover=False, finish="stop", tools=None,
|
||||
unread=0, notes=0, source="", parser="qwen", stream_only=False, announced=False):
|
||||
assert name not in CORPUS, name
|
||||
CORPUS[name] = dict(text=text, calls=[tuple(c) for c in calls], content=content, reasoning=reasoning,
|
||||
thinking=thinking, recover=recover, finish=finish, tools=tools or TOOLS, unread=unread,
|
||||
notes=notes, source=source, parser=parser, stream_only=stream_only, announced=announced)
|
||||
|
||||
|
||||
# ---- the reporters' examples ------------------------------------------------------------------------------------
|
||||
add("4427-todos-valid", qwen("todo_write", todos=TODO_OK), [("todo_write", {"todos": json.loads(TODO_OK)})],
|
||||
source="#4427 FraYoshi, first example (the Harness todo_write call)")
|
||||
add("4427-todos-stray-items", "Old tree project files are present but the task is fresh.\n\n" + qwen("todo_write", todos=TODO_STRAY),
|
||||
[("todo_write", {"todos": TODO_STRAY_WANT})], content="Old tree project files are present but the task is fresh.",
|
||||
notes=1, source="#4427 FraYoshi, second example: `\"status\": \"pending\", \"in_progress\", \"pending\": false`")
|
||||
add("4427-todos-single-quotes", qwen("todo_write", todos="[{'content': 'a', 'status': 'pending'}]"),
|
||||
[("todo_write", {"todos": [{"content": "a", "status": "pending"}]})], notes=1, source="#4427 shape: Python quotes")
|
||||
add("4427-todos-trailing-comma-and-open-bracket", qwen("todo_write", todos='[{"content": "a", "status": "pending"},'),
|
||||
[("todo_write", {"todos": [{"content": "a", "status": "pending"}]})], notes=1, source="#4427 shape: a cut array")
|
||||
add("4427-todos-json-string-of-an-array", qwen("todo_write", todos=json.dumps(TODO_OK)),
|
||||
[("todo_write", {"todos": json.loads(TODO_OK)})], source="#4427 shape: the array written as a JSON string")
|
||||
add("4427-todos-not-json-stays-text", qwen("todo_write", todos="buy milk"), [("todo_write", {"todos": "buy milk"})],
|
||||
source="#4427: text that is no array stays the text it is")
|
||||
add("4427-one-line", "Let me run ls and read the images.\n\n<tool_call> <function=run_command> <parameter=command> "
|
||||
"ls -la /home/user/imap-attachments/ 2>&1 | head -30 </parameter> </function> </tool_call>",
|
||||
[("run_command", {"command": " ls -la /home/user/imap-attachments/ 2>&1 | head -30 "})],
|
||||
content="Let me run ls and read the images.", source="#4427 miscdebris: the tags on one line")
|
||||
add("4427-one-line-in-thinking", "Let me run ls and read the images.\n\n<tool_call> <function=run_command> <parameter=command> "
|
||||
"ls -la 2>&1 | head -30 </parameter> </function> </tool_call>", [("run_command", {"command": " ls -la 2>&1 | head -30 "})],
|
||||
reasoning="Let me run ls and read the images.\n\n", thinking=True,
|
||||
source="#4427 miscdebris: written inside the thinking, the turn ends on it")
|
||||
add("4078-nested-call", qwen("write_file", path="fixture.txt", content="first line\n" + NESTED + "\nlast line"),
|
||||
[("write_file", {"path": "fixture.txt", "content": "first line\n" + NESTED + "\nlast line"})],
|
||||
source="#4078 LispEngineer / ackalker: a value holding a whole call")
|
||||
add("4078-reopened-parameter", qwen("write_file", path="fixture.txt", content=REOPEN),
|
||||
[("write_file", {"path": "fixture.txt", "content": REOPEN})], source="#4078: a value that reopens its own parameter")
|
||||
add("4078-heredoc", qwen("bash", command=HEREDOC), [("bash", {"command": HEREDOC})],
|
||||
source="#4078 Tsury: a heredoc that prints a quoted call")
|
||||
add("4078-call-format-in-a-file", qwen("write_file", path="DOC.md", content="Close a call with </function> and then </tool_call>.\nlike this."),
|
||||
[("write_file", {"path": "DOC.md", "content": "Close a call with </function> and then </tool_call>.\nlike this."})],
|
||||
source="#210: a file that documents the format")
|
||||
add("4077-stray-think-line", "The user wants a file with a literal.\n" + STRAY_VALUE + "\nSo I write it.\n</think>\n\n" +
|
||||
qwen("write_file", path="out.txt", content=STRAY_VALUE),
|
||||
[("write_file", {"path": "out.txt", "content": STRAY_VALUE})], thinking=True,
|
||||
reasoning="The user wants a file with a literal.\nreasoning sample:\n",
|
||||
content="answer sample\nSo I write it.", source="#4077 Tsury: the model quoted the closing tag at a line start")
|
||||
add("4077-stray-think-in-a-fence", "Content:\n```\n" + STRAY_VALUE + "\n```\nNow call it.\n</think>\n\n" +
|
||||
qwen("write_file", path="out.txt", content=STRAY_VALUE),
|
||||
[("write_file", {"path": "out.txt", "content": STRAY_VALUE})], thinking=True,
|
||||
reasoning="Content:\n```\nreasoning sample:\n", content="answer sample\n```\nNow call it.",
|
||||
source="#4077: the quote sat in a code fence, so the fence closes in the 'answer'")
|
||||
add("4077-stray-think-after-a-backtick", "The text is `" + STRAY_VALUE + "`. Call it.\n</think>\n\n" +
|
||||
qwen("write_file", path="out.txt", content=STRAY_VALUE),
|
||||
[("write_file", {"path": "out.txt", "content": STRAY_VALUE})], thinking=True,
|
||||
reasoning="The text is `reasoning sample:\n", content="answer sample`. Call it.", source="#4077: an open backtick")
|
||||
add("4077-quoted-think-stays-reasoning", "I will not write `</think>` yet, I keep thinking.\nDone thinking.\n</think>\n\nThe answer.",
|
||||
thinking=True, reasoning="I will not write `</think>` yet, I keep thinking.\nDone thinking.\n", content="The answer.",
|
||||
source="#537: a closing tag after a backtick is still reasoning (control)")
|
||||
add("4077-closer-in-the-answer-text-is-kept-mid-line", "Reasoning.\n</think>\n\nWrite </think> to end the thinking.",
|
||||
thinking=True, reasoning="Reasoning.\n", content="Write </think> to end the thinking.",
|
||||
source="a closing tag inside a sentence of the answer is text (control)")
|
||||
add("4077-no-second-closer-is-the-normal-case", "Reasoning.\n</think>\n\n" + qwen("read", path="a.txt"),
|
||||
[("read", {"path": "a.txt"})], thinking=True, reasoning="Reasoning.\n", source="control")
|
||||
|
||||
# ---- the Qwen3.8-Flash-Next form and its slips ------------------------------------------------------------------
|
||||
add("qwen-one-call", "I will read it.\n\n" + qwen("read", path="a.txt"), [("read", {"path": "a.txt"})],
|
||||
content="I will read it.", source="the template's own form")
|
||||
add("qwen-int-argument", qwen("run_command", command="make -j8", timeout="30"),
|
||||
[("run_command", {"command": "make -j8", "timeout": 30})], source="a declared integer")
|
||||
add("qwen-after-thinking", "Plan: read the file.\n</think>\n\n" + qwen("read", path="a.txt"), [("read", {"path": "a.txt"})],
|
||||
thinking=True, reasoning="Plan: read the file.\n", source="the normal agent turn")
|
||||
add("qwen-call-in-the-thinking", "Plan: read the file.\n\n" + qwen("read", path="a.txt"), [("read", {"path": "a.txt"})],
|
||||
thinking=True, reasoning="Plan: read the file.\n\n", source="#804: the turn ends inside the thinking, on the call")
|
||||
add("qwen-three-calls", "Reading all three.\n\n" + "\n".join(qwen("read", path=f"{n}.txt") for n in "abc"),
|
||||
[("read", {"path": "a.txt"}), ("read", {"path": "b.txt"}), ("read", {"path": "c.txt"})], content="Reading all three.",
|
||||
source="parallel calls, one wrapper each")
|
||||
add("qwen-calls-with-prose-between", qwen("read", path="a.txt") + "\nand now the other one:\n" + qwen("read", path="b.txt"),
|
||||
[("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], content="and now the other one:", source="prose between calls")
|
||||
add("qwen-three-calls-in-the-thinking", "Plan.\n\n" + "\n".join(qwen("read", path=f"{n}.txt") for n in "ab"),
|
||||
[("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], thinking=True, reasoning="Plan.\n\n\n",
|
||||
source="#804: two calls at the end of the thinking")
|
||||
add("qwen-function-close-left-out", "<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n</parameter>\n"
|
||||
"<parameter=content>\nhello\n</parameter>\n</tool_call>", [("write_file", {"path": "out.txt", "content": "hello"})],
|
||||
source="#4427 shape: no </function> (PR #3960's Coder-L slip)")
|
||||
add("qwen-function-close-left-out-in-thinking", "Writing.\n\n<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n"
|
||||
"</parameter>\n<parameter=content>\nhello\n</parameter>\n</tool_call>", [("write_file", {"path": "out.txt", "content": "hello"})],
|
||||
thinking=True, reasoning="Writing.\n\n", source="the same inside the thinking")
|
||||
add("qwen-wrapper-close-left-out", "<tool_call>\n<function=read>\n<parameter=path>\na.txt\n</parameter>\n</function>",
|
||||
[("read", {"path": "a.txt"})], stream_only=True, source="the output ends on </function> (the server always streams)")
|
||||
add("qwen-wrapper-close-left-out-in-thinking", "Plan.\n\n<tool_call>\n<function=read>\n<parameter=path>\na.txt\n</parameter>\n</function>",
|
||||
[("read", {"path": "a.txt"})], thinking=True, reasoning="Plan.\n\n", source="the same inside the thinking")
|
||||
add("qwen-wrapper-close-left-out-two-calls", qwen("read", path="a.txt").replace("\n</tool_call>", "") + "\n" + qwen("read", path="b.txt"),
|
||||
[("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], source="a following opener confirms the missing closer")
|
||||
add("qwen-wrapper-close-left-out-two-calls-in-thinking", "Plan.\n\n" + qwen("read", path="a.txt").replace("\n</tool_call>", "") +
|
||||
"\n" + qwen("read", path="b.txt"), [("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], thinking=True,
|
||||
reasoning="Plan.\n\n\n", source="the same inside the thinking")
|
||||
add("qwen-undeclared-name-is-text", qwen("format_disk", target="/"), content=qwen("format_disk", target="/"), unread=1,
|
||||
source="by design: a call for a tool the request did not declare comes back as text (and is counted)")
|
||||
add("qwen-cut-inside-a-value", "<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n</parameter>\n<parameter=content>\nhal",
|
||||
content="<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n</parameter>\n<parameter=content>\nhal",
|
||||
finish="length", announced=True, source="an unterminated tag: the output stopped at the token limit; no call")
|
||||
add("qwen-cut-inside-the-opener", "Now.\n\n<tool_call>\n<function=writ", content="Now.\n\n<tool_call>\n<function=writ",
|
||||
finish="length", source="cut inside the name")
|
||||
add("qwen-cut-after-function-close-on-length", "<tool_call>\n<function=read>\n<parameter=path>\na.txt\n</parameter>\n</function>",
|
||||
finish="length", content="<tool_call>\n<function=read>\n<parameter=path>\na.txt\n</parameter>\n</function>",
|
||||
source="a turn cut by the token limit is not a natural stop: no call")
|
||||
add("qwen-quoted-in-a-fence", "Example:\n```\n" + qwen("read", path="a.txt") + "\n```", content="Example:\n```\n" + qwen("read", path="a.txt") + "\n```",
|
||||
source="#1058: a call in a code fence is a quote")
|
||||
add("qwen-prose-names-the-format", "Calls are written as <tool_call> blocks.", content="Calls are written as <tool_call> blocks.",
|
||||
source="prose that names the tag")
|
||||
|
||||
# ---- the drift shapes (tool_call_recovery on; the corpus of 1,462 agent turns) ----------------------------------
|
||||
PW = "<parameter=path>\na.md\n</parameter>\n<parameter=content>\nhi\n</parameter>\n</function>"
|
||||
WRITE_TOOLS = [tool("write", path=S, content=S)]
|
||||
add("drift-parameter-as-opener", f"<tool_call>\n<parameter=write>\n{PW}\n</tool_call>", [("write", {"path": "a.md", "content": "hi"})],
|
||||
recover=True, tools=WRITE_TOOLS, source="corpus 300dfba2 (x11)")
|
||||
add("drift-json-in-the-wrapper", '<tool_call>\n{"name": "write", "arguments": {"path": "a.md", "content": "hi"}}\n</tool_call>',
|
||||
[("write", {"path": "a.md", "content": "hi"})], recover=True, tools=WRITE_TOOLS, source="corpus 5e847539 (x6)")
|
||||
add("drift-bare-function", f"Writing it now.\n\n<function=write>\n{PW}", [("write", {"path": "a.md", "content": "hi"})],
|
||||
content="Writing it now.", recover=True, tools=WRITE_TOOLS, source="corpus 395efd4c / 6e71eed4 (x5 each)")
|
||||
add("drift-batch-in-one-wrapper", f"<tool_call>\n<function=write>\n{PW}\n<function=write>\n{PW.replace('a.md', 'b.md')}\n</tool_call>",
|
||||
[("write", {"path": "a.md", "content": "hi"}), ("write", {"path": "b.md", "content": "hi"})], recover=True,
|
||||
tools=WRITE_TOOLS, source="corpus 0912870b / 51b155c9")
|
||||
|
||||
# ---- DeepSeek-V4.1 (DSML) ---------------------------------------------------------------------------------------
|
||||
add("dsml-one-call", dsml(("write_file", [("path", "out.txt", True), ("content", "hello\nworld", True)])),
|
||||
[("write_file", {"path": "out.txt", "content": "hello\nworld"})], parser="dsml", source="the model's own template")
|
||||
add("dsml-after-thinking", "Plan.</think>\n\n" + dsml(("read", [("path", "a.txt", True)])), [("read", {"path": "a.txt"})],
|
||||
thinking=True, reasoning="Plan.", parser="dsml", source="a thinking turn")
|
||||
add("dsml-typed-arguments", dsml(("run_command", [("command", "make", True), ("timeout", "30", False)]),
|
||||
("todo_write", [("todos", TODO_OK, False)])),
|
||||
[("run_command", {"command": "make", "timeout": 30}), ("todo_write", {"todos": json.loads(TODO_OK)})], parser="dsml",
|
||||
source="string=\"false\" values are JSON")
|
||||
add("dsml-damaged-array", dsml(("todo_write", [("todos", TODO_STRAY, False)])), [("todo_write", {"todos": TODO_STRAY_WANT})],
|
||||
notes=1, parser="dsml", source="#4427 shape inside DSML")
|
||||
add("dsml-several-calls-and-text", "Reading both.\n\n" + dsml(("read", [("path", "a.txt", True)]), ("read", [("path", "b.txt", True)])),
|
||||
[("read", {"path": "a.txt"}), ("read", {"path": "b.txt"})], content="Reading both.", parser="dsml", source="a batch")
|
||||
add("dsml-value-holds-a-qwen-call", dsml(("write_file", [("path", "x.txt", True), ("content", NESTED, True)])),
|
||||
[("write_file", {"path": "x.txt", "content": NESTED})], parser="dsml", source="#4078 shape in DSML")
|
||||
add("dsml-stray-think", "Quote:\n</think>\nmore thinking.\n</think>\n\n" + dsml(("read", [("path", "a.txt", True)])),
|
||||
[("read", {"path": "a.txt"})], thinking=True, reasoning="Quote:\n", content="more thinking.", parser="dsml",
|
||||
source="#4077 shape in DSML")
|
||||
add("dsml-cut-call", dsml(("read", [("path", "a.txt", True)]))[:60], finish="length", content=None,
|
||||
parser="dsml", source="a DSML call cut by the token limit is no call (its text is the translator's, not checked)")
|
||||
|
||||
|
||||
def run(spec, pieces, stream_tools):
|
||||
cls = DsmlOutputParser if spec["parser"] == "dsml" else OutputParser
|
||||
p = cls(thinking=spec["thinking"], tools=spec["tools"], stream_tools=stream_tools, recover=spec["recover"],
|
||||
declared_only=True)
|
||||
evs = []
|
||||
for piece in pieces:
|
||||
evs += p.feed(piece)
|
||||
evs += p.finish(spec["finish"])
|
||||
return evs, p
|
||||
|
||||
|
||||
def cut(text, size):
|
||||
return [text[i:i + size] for i in range(0, len(text), size)]
|
||||
|
||||
|
||||
class ToolCorpus(unittest.TestCase):
|
||||
def check(self, name, spec, pieces, stream_tools, label):
|
||||
evs, p = run(spec, pieces, stream_tools)
|
||||
calls = [(e.call.name, e.call.arguments) for e in evs if e.kind == "tool_call"]
|
||||
text = lambda kind: "".join(e.text for e in evs if e.kind == kind) # noqa: E731
|
||||
where = f"{name} [{label}, stream_tools={stream_tools}]"
|
||||
self.assertEqual(calls, spec["calls"], where)
|
||||
# a call announced to the client and then cut stays unfinished there (#211): its text is not repeated as content
|
||||
if spec["content"] is not None:
|
||||
self.assertEqual(text("content").strip(), "" if spec["announced"] and stream_tools else spec["content"], where)
|
||||
self.assertEqual(text("reasoning").strip(), spec["reasoning"].strip(), where)
|
||||
self.assertEqual(p.unread, spec["unread"], where)
|
||||
self.assertEqual(len(p.notes), spec["notes"], where)
|
||||
if stream_tools:
|
||||
for e in (x for x in evs if x.kind == "tool_call"):
|
||||
streamed = "".join(x.text for x in evs if x.kind == "tool_args" and x.call is not None and x.call.id == e.call.id)
|
||||
if streamed:
|
||||
self.assertEqual(json.loads(streamed), e.call.arguments, where)
|
||||
return evs
|
||||
|
||||
def test_every_specimen_whole_and_in_pieces(self):
|
||||
for name, spec in CORPUS.items():
|
||||
for stream_tools in (True,) if spec["stream_only"] else (False, True):
|
||||
self.check(name, spec, [spec["text"]], stream_tools, "whole")
|
||||
for size in (1, 3, 7):
|
||||
self.check(name, spec, cut(spec["text"], size), stream_tools, f"pieces of {size}")
|
||||
|
||||
def test_every_specimen_cut_in_two_at_every_byte(self):
|
||||
for name, spec in CORPUS.items():
|
||||
text = spec["text"]
|
||||
for at in range(1, len(text)):
|
||||
self.check(name, spec, [text[:at], text[at:]], True, f"cut at {at}")
|
||||
|
||||
def test_the_same_events_whatever_the_pieces(self):
|
||||
"""Text and calls of a specimen do not depend on how the stream was cut (the ids aside)."""
|
||||
for name, spec in CORPUS.items():
|
||||
whole, _ = run(spec, [spec["text"]], True)
|
||||
want = [(e.kind, e.text) for e in whole if e.kind in ("content", "reasoning")]
|
||||
joined = lambda evs, k: "".join(t for kind, t in evs if kind == k) # noqa: E731
|
||||
for size in (1, 5):
|
||||
got, _ = run(spec, cut(spec["text"], size), True)
|
||||
got = [(e.kind, e.text) for e in got if e.kind in ("content", "reasoning")]
|
||||
for kind in ("content", "reasoning"):
|
||||
self.assertEqual(joined(got, kind), joined(want, kind), f"{name} pieces of {size}")
|
||||
|
||||
def test_the_sources_are_named(self):
|
||||
for name, spec in CORPUS.items():
|
||||
self.assertTrue(spec["source"], name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,161 @@
|
||||
"""#4427, #4077, #4078 through the HTTP routes: what a client gets for the damaged calls and the stray closing think tag.
|
||||
|
||||
The model's text is replayed by the mock engine; the reply is read from Chat Completions, the Anthropic Messages API and
|
||||
the Responses API, each whole and streamed. Every route must hand over the same calls with the same arguments.
|
||||
|
||||
python -m unittest serve.test_tool_routes -v
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from serve.frontend import ChatTemplate # noqa: E402
|
||||
from serve.server import ByteTokenizer, MockEngine, Service, serve # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TEMPLATE = ChatTemplate(ROOT / "serve/chat_template.jinja")
|
||||
PROPS = {"todos": {"type": "array"}, "path": {"type": "string"}, "content": {"type": "string"}}
|
||||
NAMES = ("todo_write", "write_file")
|
||||
|
||||
|
||||
def call(name, **params):
|
||||
body = "".join(f"<parameter={k}>\n{v}\n</parameter>\n" for k, v in params.items())
|
||||
return f"<tool_call>\n<function={name}>\n{body}</function>\n</tool_call>"
|
||||
|
||||
|
||||
class Routes(unittest.TestCase):
|
||||
def read(self, script, **service):
|
||||
"""{route: (finish, [(name, arguments)], answer text)} for the model's `script`, on every route."""
|
||||
tok = ByteTokenizer()
|
||||
svc = Service(MockEngine(tok, script, max_context=16384), tok, TEMPLATE)
|
||||
for k, v in service.items():
|
||||
setattr(svc, k, v)
|
||||
httpd = serve(svc, port=0)
|
||||
port = httpd.server_address[1]
|
||||
out = {}
|
||||
|
||||
def post(path, body):
|
||||
c = http.client.HTTPConnection("127.0.0.1", port, timeout=60)
|
||||
try:
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
c.request("POST", path, body=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json", "anthropic-version": "2023-06-01"})
|
||||
return c.getresponse().read().decode()
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
def sse(raw):
|
||||
return [json.loads(line[6:]) for line in raw.splitlines() if line.startswith("data: {")]
|
||||
|
||||
chat_tools = [{"type": "function", "function": {"name": n, "parameters": {"type": "object", "properties": PROPS}}}
|
||||
for n in NAMES]
|
||||
try:
|
||||
for stream in (False, True):
|
||||
raw = post("/v1/chat/completions", {"model": "x", "max_tokens": 900, "stream": stream, "tools": chat_tools,
|
||||
"messages": [{"role": "user", "content": "go"}]})
|
||||
if stream:
|
||||
evs, calls, text, finish = sse(raw), {}, "", None
|
||||
for e in evs:
|
||||
ch = e["choices"][0]
|
||||
text += ch["delta"].get("content") or ""
|
||||
finish = ch.get("finish_reason") or finish
|
||||
for tc in ch["delta"].get("tool_calls") or []:
|
||||
slot = calls.setdefault(tc["index"], {"name": None, "args": ""})
|
||||
slot["name"] = (tc.get("function") or {}).get("name") or slot["name"]
|
||||
slot["args"] += (tc.get("function") or {}).get("arguments") or ""
|
||||
got = [(c["name"], json.loads(c["args"])) for _, c in sorted(calls.items())]
|
||||
else:
|
||||
d = json.loads(raw)["choices"][0]
|
||||
text, finish = d["message"].get("content") or "", d["finish_reason"]
|
||||
got = [(t["function"]["name"], json.loads(t["function"]["arguments"])) for t in d["message"].get("tool_calls") or []]
|
||||
out["chat", stream] = (finish, got, text.strip())
|
||||
|
||||
raw = post("/v1/messages", {"model": "x", "max_tokens": 900, "stream": stream,
|
||||
"tools": [{"name": n, "input_schema": {"type": "object", "properties": PROPS}} for n in NAMES],
|
||||
"messages": [{"role": "user", "content": "go"}]})
|
||||
if stream:
|
||||
evs, blocks, text, finish = sse(raw), {}, "", None
|
||||
for e in evs:
|
||||
if e["type"] == "content_block_start" and e["content_block"]["type"] == "tool_use":
|
||||
blocks[e["index"]] = {"name": e["content_block"]["name"], "json": ""}
|
||||
elif e["type"] == "content_block_delta":
|
||||
if e["delta"]["type"] == "input_json_delta":
|
||||
blocks[e["index"]]["json"] += e["delta"]["partial_json"]
|
||||
elif e["delta"]["type"] == "text_delta":
|
||||
text += e["delta"]["text"]
|
||||
elif e["type"] == "message_delta":
|
||||
finish = e["delta"]["stop_reason"]
|
||||
got = [(b["name"], json.loads(b["json"] or "{}")) for _, b in sorted(blocks.items())]
|
||||
else:
|
||||
d = json.loads(raw)
|
||||
finish = d["stop_reason"]
|
||||
text = "".join(b.get("text", "") for b in d["content"] if b["type"] == "text")
|
||||
got = [(b["name"], b["input"]) for b in d["content"] if b["type"] == "tool_use"]
|
||||
out["anthropic", stream] = (finish, got, text.strip())
|
||||
|
||||
raw = post("/v1/responses", {"model": "x", "input": "go", "stream": stream, "store": False,
|
||||
"tools": [{"type": "function", "name": n, "parameters": {"type": "object", "properties": PROPS}}
|
||||
for n in NAMES]})
|
||||
if stream:
|
||||
items = [e["item"] for e in sse(raw) if e["type"] == "response.output_item.done"]
|
||||
else:
|
||||
items = json.loads(raw)["output"]
|
||||
got = [(i["name"], json.loads(i["arguments"])) for i in items if i["type"] == "function_call"]
|
||||
text = "".join(c["text"] for i in items if i["type"] == "message" for c in i["content"] if c["type"] == "output_text")
|
||||
out["responses", stream] = ("-", got, text.strip())
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
return out
|
||||
|
||||
def every_route(self, script, calls, text="", finish=("tool_calls", "tool_use"), **service):
|
||||
got = self.read(script, **service)
|
||||
for (route, stream), (fin, got_calls, got_text) in got.items():
|
||||
with self.subTest(route=route, stream=stream):
|
||||
self.assertEqual(got_calls, calls)
|
||||
self.assertEqual(got_text, text)
|
||||
if route == "chat":
|
||||
self.assertEqual(fin, finish[0])
|
||||
elif route == "anthropic":
|
||||
self.assertEqual(fin, finish[1])
|
||||
|
||||
def test_a_damaged_array_argument_arrives_as_the_array(self):
|
||||
stray = ('[{"content": "Init", "status": "in_progress", "pending": false}, '
|
||||
'{"content": "Write", "status": "pending", "in_progress", "pending": false}]')
|
||||
want = [{"content": "Init", "status": "in_progress", "pending": False},
|
||||
{"content": "Write", "status": "pending", "pending": False}]
|
||||
self.every_route("Plan.\n</think>\n\n" + call("todo_write", todos=stray), [("todo_write", {"todos": want})])
|
||||
|
||||
def test_a_call_without_its_function_closer(self):
|
||||
text = ("Plan.\n</think>\n\n<tool_call>\n<function=write_file>\n<parameter=path>\nout.txt\n</parameter>\n"
|
||||
"<parameter=content>\nhello\n</parameter>\n</tool_call>")
|
||||
self.every_route(text, [("write_file", {"path": "out.txt", "content": "hello"})])
|
||||
|
||||
def test_a_call_without_its_wrapper_closer_at_the_end(self):
|
||||
text = "Plan.\n</think>\n\n" + call("write_file", path="out.txt", content="hello").removesuffix("\n</tool_call>")
|
||||
self.every_route(text, [("write_file", {"path": "out.txt", "content": "hello"})])
|
||||
|
||||
def test_a_value_that_quotes_a_whole_call(self):
|
||||
value = "first line\n" + call("write_file", path="nested.txt") + "\nlast line"
|
||||
self.every_route("Plan.\n</think>\n\n" + call("write_file", path="fixture.txt", content=value),
|
||||
[("write_file", {"path": "fixture.txt", "content": value})])
|
||||
|
||||
def test_a_stray_closing_think_tag_keeps_the_call(self):
|
||||
value = "reasoning sample:\n</think>\nanswer sample"
|
||||
script = ("The content is\n```\n" + value + "\n```\nCall it.\n</think>\n\n" + call("write_file", path="out.txt", content=value))
|
||||
got = self.read(script)
|
||||
for (route, stream), (fin, calls, text) in got.items():
|
||||
with self.subTest(route=route, stream=stream):
|
||||
self.assertEqual(calls, [("write_file", {"path": "out.txt", "content": value})])
|
||||
self.assertNotIn("</think>", text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user