From 209dcadb3f56e16832cfa7cbeedcdfed18b72c8d Mon Sep 17 00:00:00 2001 From: Niko1221 Date: Fri, 30 Oct 2026 19:06:11 +0100 Subject: [PATCH] Parser: a fence is closed only by one as long as it opened with, and an indented code block is a quote, so calls shown in them are not run (#4702 #4703) --- serve/frontend.py | 51 +++++++++++++++---- serve/test_parser_markdown_code.py | 80 ++++++++++++++++++++++++++++++ serve/test_parser_track.py | 20 ++++++-- 3 files changed, 135 insertions(+), 16 deletions(-) create mode 100644 serve/test_parser_markdown_code.py diff --git a/serve/frontend.py b/serve/frontend.py index 3b3cdeb17..4955193c2 100644 --- a/serve/frontend.py +++ b/serve/frontend.py @@ -1143,6 +1143,7 @@ class OutputParser: self.stray_think = 0 # ... and the strays of that kind dropped from the answer self.refused = 0 # declared calls kept as reasoning text (quoted, or the turn was cut) self.fence, self.line, self.ticks = "", "", 0 + self._prev_blank = self._indented = False # #4703: the line before was blank / this is inside an indented (4-space) code block # recover (on in the server since 0.2.8, off for a bare OutputParser; the config's "tool_call_recovery": false turns it off): a declared tool's call written next to the template's # form is the call - `` as the opener, JSON inside , a bare at the # start of a line outside code, or a second call in the same (which otherwise merges into one). @@ -1159,10 +1160,10 @@ class OutputParser: def _ok_at(self, p: int) -> bool: """self.buf[p] would open a call: at the start of a line, outside a code fence and inline code.""" - snap = (self.fence, self.line, self.ticks) + snap = self._snap() self._track(self.buf[:p]) ok = self._opener_ok() - self.fence, self.line, self.ticks = snap + self._restore(snap) return ok def _callable(self, name: str) -> bool: @@ -1270,15 +1271,29 @@ class OutputParser: self._line_parts.clear() self._line_blocks = None s = line.lstrip() + was_fenced = bool(self.fence) if self.fence: - if s.startswith(self.fence * 3): + # #4702: a fence is closed only by a line of at least as many of the same character as opened it (CommonMark), so a + # ``` example inside a ```` file stays inside it + run = len(s) - len(s.lstrip(self.fence[0])) + if run >= len(self.fence): self.fence = "" elif s[:3] in ("```", "~~~") and (s[0] * 3) not in s[3:]: - self.fence = s[0] + self.fence = s[0] * (len(s) - len(s.lstrip(s[0]))) elif not s: self.ticks = 0 else: self.ticks += line.count("`") + # #4703: an indented (4 spaces or a tab) code block starts after a blank line and goes on until a line that is not indented + if was_fenced or self.fence: + self._prev_blank = self._indented = False + elif not s: + self._prev_blank = True + elif line.startswith((" ", "\t")): + self._indented = self._prev_blank or self._indented + self._prev_blank = False + else: + self._prev_blank = self._indented = False else: if part: self._line_parts.append(part) @@ -1287,11 +1302,25 @@ class OutputParser: def _opener_ok(self) -> bool: """The reasoning text so far puts a `` at the start of a line, outside a code fence and outside inline code.""" - return not self.fence and not self.line.strip() and self.ticks % 2 == 0 + return not self.fence and not self.line.strip() and self.ticks % 2 == 0 and not self._in_indented_block() + + def _in_indented_block(self) -> bool: + """#4703: the current line is part of an indented code block: four spaces or a tab, after a blank line or another such line.""" + return (self._prev_blank or self._indented) and self.line.startswith((" ", "\t")) + + def _snap(self) -> tuple: + return (self.fence, self.line, self.ticks, self._prev_blank, self._indented) + + def _restore(self, snap: tuple) -> None: + self.fence, self.line, self.ticks, self._prev_blank, self._indented = snap + + def _reset_track(self) -> None: + self.fence, self.line, self.ticks = "", "", 0 + self._prev_blank = self._indented = False def _in_code(self) -> bool: - """The text so far leaves the next character inside a code fence or inline code.""" - return bool(self.fence) or (self.ticks + self.line.count("`")) % 2 == 1 + """The text so far leaves the next character inside a code fence, inline code or an indented code block.""" + return bool(self.fence) or (self.ticks + self.line.count("`")) % 2 == 1 or self._in_indented_block() def _quoted_think_end(self, i: int) -> bool | None: """#537: the `` at self.buf[i] is the model quoting the tag while it thinks, not the end of the @@ -1576,7 +1605,7 @@ class OutputParser: out.append(Event("reasoning", self._track(self.buf[:i]))) self.buf = self.buf[i + len(THINK_END):] self.state, self.lead, self.after_think = "content", True, True - self.fence, self.line, self.ticks = "", "", 0 # the answer's own text starts here + self._reset_track() # the answer's own text starts here elif self.state == "content": if self.lead: # newlines right after or a call stripped = self.buf.lstrip("\n") @@ -1594,7 +1623,7 @@ class OutputParser: if stray: out.append(Event("content", self._track(self.buf[:stray]))) self.buf = self.buf[stray + len(THINK_END):] - self.fence, self.line, self.ticks = "", "", 0 + self._reset_track() self.lead = True self.stray_think += 1 continue @@ -1630,10 +1659,10 @@ class OutputParser: return out # #1058: an opener inside a code fence or inline code is text (a quoted example), never a call. The # text before it decides, so streamed and whole outputs agree. Mid-sentence openers still count. - snap = (self.fence, self.line, self.ticks) + snap = self._snap() self._track(self.buf[:i]) in_code = self._in_code() - self.fence, self.line, self.ticks = snap + self._restore(snap) if in_code: out.append(Event("content", self._track(self.buf[:i + len(CALL_START)]))) self.buf = self.buf[i + len(CALL_START):] diff --git a/serve/test_parser_markdown_code.py b/serve/test_parser_markdown_code.py new file mode 100644 index 000000000..0d76bd159 --- /dev/null +++ b/serve/test_parser_markdown_code.py @@ -0,0 +1,80 @@ +"""#4702 and #4703: a call quoted in a Markdown code form is text, not a call. A four-backtick fence is closed only by a fence of +at least four backticks (an inner ``` block stays inside it), and an indented (4-space) code block after a blank line is a quote. +A call the model writes plainly, or indents inside a paragraph, is still a call. + + python -m unittest serve.test_parser_markdown_code +""" +import unittest + +from serve.frontend import OutputParser + + +def tool(name, **props): + return {"name": name, "parameters": {"type": "object", "properties": props}} + + +S = {"type": "string"} +TOOLS = [tool("read", path=S), tool("write_file", path=S, content=S)] + + +def qwen(name, **params): + body = "".join(f"\n{v}\n\n" for k, v in params.items()) + return f"\n\n{body}\n" + + +def parse(text, piece=0, recover=False): + p = OutputParser(thinking=False, tools=TOOLS, stream_tools=False, recover=recover, declared_only=True) + if piece: + evs = [] + for i in range(0, len(text), piece): + evs += p.feed(text[i:i + piece]) + evs += p.finish("stop") + else: + evs = p.feed(text) + p.finish("stop") + return ([(e.call.name, e.call.arguments) for e in evs if e.kind == "tool_call"], + "".join(e.text for e in evs if e.kind == "content")) + + +class MarkdownCode(unittest.TestCase): + def each(self, text): + for recover in (False, True): + for piece in (0, 1, 7): + yield parse(text, piece, recover) + + def test_an_inner_fence_does_not_close_a_longer_one(self): + text = ("Save this as docs/TOOLS.md:\n\n````markdown\n# Tool calls\n\nA call:\n\n```xml\n" + + qwen("read", path="EXAMPLE.txt") + "\n```\n````\n\nThat is the file.") + for calls, answer in self.each(text): + self.assertEqual(calls, []) + self.assertEqual(answer, text) + + def test_a_call_after_a_closed_longer_fence_is_a_call(self): + text = "Here:\n\n````md\nexample\n````\n\n" + qwen("read", path="a.txt") + for calls, _ in self.each(text): + self.assertEqual(calls, [("read", {"path": "a.txt"})]) + + def test_a_tilde_fence_is_not_closed_by_backticks(self): + text = "Quote:\n\n~~~\n```\n" + qwen("read", path="EXAMPLE.txt") + "\n```\n~~~\n" + for calls, _ in self.each(text): + self.assertEqual(calls, []) + + def test_an_indented_code_block_is_a_quote(self): + block = "\n".join(" " + line for line in qwen("read", path="EXAMPLE.txt").split("\n")) + text = "The format, indented as code:\n\n" + block + "\n\nIndentation makes it a code block in Markdown." + for calls, answer in self.each(text): + self.assertEqual(calls, []) + self.assertEqual(answer, text) + + def test_a_plain_call_after_a_paragraph_is_still_a_call(self): + text = "Let me read it.\n" + qwen("read", path="a.txt") + for calls, _ in self.each(text): + self.assertEqual(calls, [("read", {"path": "a.txt"})]) + + def test_a_call_after_an_indented_block_ends_it(self): + text = "Example:\n\n x = 1\n\nNow the real one:\n\n" + qwen("read", path="a.txt") + for calls, _ in self.each(text): + self.assertEqual(calls, [("read", {"path": "a.txt"})]) + + +if __name__ == "__main__": + unittest.main() diff --git a/serve/test_parser_track.py b/serve/test_parser_track.py index 53f7ef76c..a7e9dd7cf 100644 --- a/serve/test_parser_track.py +++ b/serve/test_parser_track.py @@ -16,15 +16,25 @@ class ReferenceParser(OutputParser): if k < len(parts) - 1: line, self.line = self.line + part, "" s = line.lstrip() + was_fenced = bool(self.fence) if self.fence: - if s.startswith(self.fence * 3): + if len(s) - len(s.lstrip(self.fence[0])) >= len(self.fence): # #4702: as many fence characters as opened it self.fence = "" elif s[:3] in ("```", "~~~") and (s[0] * 3) not in s[3:]: - self.fence = s[0] + self.fence = s[0] * (len(s) - len(s.lstrip(s[0]))) elif not s: self.ticks = 0 else: self.ticks += line.count("`") + if was_fenced or self.fence: # #4703: indented code blocks + self._prev_blank = self._indented = False + elif not s: + self._prev_blank = True + elif line.startswith((" ", "\t")): + self._indented = self._prev_blank or self._indented + self._prev_blank = False + else: + self._prev_blank = self._indented = False else: self.line += part return text @@ -32,14 +42,14 @@ class ReferenceParser(OutputParser): class TrackTests(unittest.TestCase): def assert_state(self, actual, expected): - self.assertEqual((actual.line, actual.fence, actual.ticks), - (expected.line, expected.fence, expected.ticks)) + self.assertEqual((actual.line, actual.fence, actual.ticks, actual._prev_blank, actual._indented), + (expected.line, expected.fence, expected.ticks, expected._prev_blank, expected._indented)) self.assertEqual(actual._opener_ok(), expected._opener_ok()) self.assertEqual(actual._in_code(), expected._in_code()) def test_fragment_state_from_varied_initial_states(self): fragments = ("", "a", "é🙂", "`", "``", "```", "~~~", "\r", "\n", "\r\n", - "\n\n", " ```python\n", "```quoted```\n", "~\ntext", "\t\nrest") + "\n\n", "\n code\n", " x\n", " ```python\n", "````md\n", "```quoted```\n", "~\ntext", "\t\nrest") for fence, line, ticks, text in itertools.product(("", "`", "~"), ("", " ", "``", "code `"), (0, 1, 2, 3), fragments):