Parser: a fence is closed only by one as long as it opened with, and an indented code block is a quote, so calls shown in them are not run (#4702 #4703)

serve/frontend.py

@@ -1143,6 +1143,7 @@ class OutputParser:
         self.stray_think = 0         # ... and the strays of that kind dropped from the answer
         self.refused = 0             # declared calls kept as reasoning text (quoted, or the turn was cut)
         self.fence, self.line, self.ticks = "", "", 0
+        self._prev_blank = self._indented = False   # #4703: the line before was blank / this is inside an indented (4-space) code block
         # recover (on in the server since 0.2.8, off for a bare OutputParser; the config's "tool_call_recovery": false turns it off): a declared tool's call written next to the template's
         # form is the call - `<parameter=NAME>` as the opener, JSON inside <tool_call>, a bare <function=NAME> at the
         # start of a line outside code, or a second call in the same <tool_call> (which otherwise merges into one).
@@ -1159,10 +1160,10 @@ class OutputParser:
 
     def _ok_at(self, p: int) -> bool:
         """self.buf[p] would open a call: at the start of a line, outside a code fence and inline code."""
-        snap = (self.fence, self.line, self.ticks)
+        snap = self._snap()
         self._track(self.buf[:p])
         ok = self._opener_ok()
-        self.fence, self.line, self.ticks = snap
+        self._restore(snap)
         return ok
 
     def _callable(self, name: str) -> bool:
@@ -1270,15 +1271,29 @@ class OutputParser:
                 self._line_parts.clear()
                 self._line_blocks = None
                 s = line.lstrip()
+                was_fenced = bool(self.fence)
                 if self.fence:
-                    if s.startswith(self.fence * 3):
+                    # #4702: a fence is closed only by a line of at least as many of the same character as opened it (CommonMark), so a
+                    # ``` example inside a ```` file stays inside it
+                    run = len(s) - len(s.lstrip(self.fence[0]))
+                    if run >= len(self.fence):
                         self.fence = ""
                 elif s[:3] in ("```", "~~~") and (s[0] * 3) not in s[3:]:
-                    self.fence = s[0]
+                    self.fence = s[0] * (len(s) - len(s.lstrip(s[0])))
                 elif not s:
                     self.ticks = 0
                 else:
                     self.ticks += line.count("`")
+                # #4703: an indented (4 spaces or a tab) code block starts after a blank line and goes on until a line that is not indented
+                if was_fenced or self.fence:
+                    self._prev_blank = self._indented = False
+                elif not s:
+                    self._prev_blank = True
+                elif line.startswith(("    ", "\t")):
+                    self._indented = self._prev_blank or self._indented
+                    self._prev_blank = False
+                else:
+                    self._prev_blank = self._indented = False
             else:
                 if part:
                     self._line_parts.append(part)
@@ -1287,11 +1302,25 @@ class OutputParser:
     def _opener_ok(self) -> bool:
         """The reasoning text so far puts a `<tool_call>` at the start of a line, outside a code fence and outside
         inline code."""
-        return not self.fence and not self.line.strip() and self.ticks % 2 == 0
+        return not self.fence and not self.line.strip() and self.ticks % 2 == 0 and not self._in_indented_block()
+
+    def _in_indented_block(self) -> bool:
+        """#4703: the current line is part of an indented code block: four spaces or a tab, after a blank line or another such line."""
+        return (self._prev_blank or self._indented) and self.line.startswith(("    ", "\t"))
+
+    def _snap(self) -> tuple:
+        return (self.fence, self.line, self.ticks, self._prev_blank, self._indented)
+
+    def _restore(self, snap: tuple) -> None:
+        self.fence, self.line, self.ticks, self._prev_blank, self._indented = snap
+
+    def _reset_track(self) -> None:
+        self.fence, self.line, self.ticks = "", "", 0
+        self._prev_blank = self._indented = False
 
     def _in_code(self) -> bool:
-        """The text so far leaves the next character inside a code fence or inline code."""
-        return bool(self.fence) or (self.ticks + self.line.count("`")) % 2 == 1
+        """The text so far leaves the next character inside a code fence, inline code or an indented code block."""
+        return bool(self.fence) or (self.ticks + self.line.count("`")) % 2 == 1 or self._in_indented_block()
 
     def _quoted_think_end(self, i: int) -> bool | None:
         """#537: the `</think>` at self.buf[i] is the model quoting the tag while it thinks, not the end of the
@@ -1576,7 +1605,7 @@ class OutputParser:
                     out.append(Event("reasoning", self._track(self.buf[:i])))
                 self.buf = self.buf[i + len(THINK_END):]
                 self.state, self.lead, self.after_think = "content", True, True
-                self.fence, self.line, self.ticks = "", "", 0     # the answer's own text starts here
+                self._reset_track()                               # the answer's own text starts here
             elif self.state == "content":
                 if self.lead:                                   # newlines right after </think> or a call
                     stripped = self.buf.lstrip("\n")
@@ -1594,7 +1623,7 @@ class OutputParser:
                     if stray:
                         out.append(Event("content", self._track(self.buf[:stray])))
                     self.buf = self.buf[stray + len(THINK_END):]
-                    self.fence, self.line, self.ticks = "", "", 0
+                    self._reset_track()
                     self.lead = True
                     self.stray_think += 1
                     continue
@@ -1630,10 +1659,10 @@ class OutputParser:
                     return out
                 # #1058: an opener inside a code fence or inline code is text (a quoted example), never a call.  The
                 # text before it decides, so streamed and whole outputs agree.  Mid-sentence openers still count.
-                snap = (self.fence, self.line, self.ticks)
+                snap = self._snap()
                 self._track(self.buf[:i])
                 in_code = self._in_code()
-                self.fence, self.line, self.ticks = snap
+                self._restore(snap)
                 if in_code:
                     out.append(Event("content", self._track(self.buf[:i + len(CALL_START)])))
                     self.buf = self.buf[i + len(CALL_START):]

serve/test_parser_markdown_code.py

@@ -0,0 +1,80 @@
+"""#4702 and #4703: a call quoted in a Markdown code form is text, not a call.  A four-backtick fence is closed only by a fence of
+at least four backticks (an inner ``` block stays inside it), and an indented (4-space) code block after a blank line is a quote.
+A call the model writes plainly, or indents inside a paragraph, is still a call.
+
+    python -m unittest serve.test_parser_markdown_code
+"""
+import unittest
+
+from serve.frontend import OutputParser
+
+
+def tool(name, **props):
+    return {"name": name, "parameters": {"type": "object", "properties": props}}
+
+
+S = {"type": "string"}
+TOOLS = [tool("read", path=S), tool("write_file", path=S, content=S)]
+
+
+def qwen(name, **params):
+    body = "".join(f"<parameter={k}>\n{v}\n</parameter>\n" for k, v in params.items())
+    return f"<tool_call>\n<function={name}>\n{body}</function>\n</tool_call>"
+
+
+def parse(text, piece=0, recover=False):
+    p = OutputParser(thinking=False, tools=TOOLS, stream_tools=False, recover=recover, declared_only=True)
+    if piece:
+        evs = []
+        for i in range(0, len(text), piece):
+            evs += p.feed(text[i:i + piece])
+        evs += p.finish("stop")
+    else:
+        evs = p.feed(text) + p.finish("stop")
+    return ([(e.call.name, e.call.arguments) for e in evs if e.kind == "tool_call"],
+            "".join(e.text for e in evs if e.kind == "content"))
+
+
+class MarkdownCode(unittest.TestCase):
+    def each(self, text):
+        for recover in (False, True):
+            for piece in (0, 1, 7):
+                yield parse(text, piece, recover)
+
+    def test_an_inner_fence_does_not_close_a_longer_one(self):
+        text = ("Save this as docs/TOOLS.md:\n\n````markdown\n# Tool calls\n\nA call:\n\n```xml\n"
+                + qwen("read", path="EXAMPLE.txt") + "\n```\n````\n\nThat is the file.")
+        for calls, answer in self.each(text):
+            self.assertEqual(calls, [])
+            self.assertEqual(answer, text)
+
+    def test_a_call_after_a_closed_longer_fence_is_a_call(self):
+        text = "Here:\n\n````md\nexample\n````\n\n" + qwen("read", path="a.txt")
+        for calls, _ in self.each(text):
+            self.assertEqual(calls, [("read", {"path": "a.txt"})])
+
+    def test_a_tilde_fence_is_not_closed_by_backticks(self):
+        text = "Quote:\n\n~~~\n```\n" + qwen("read", path="EXAMPLE.txt") + "\n```\n~~~\n"
+        for calls, _ in self.each(text):
+            self.assertEqual(calls, [])
+
+    def test_an_indented_code_block_is_a_quote(self):
+        block = "\n".join("    " + line for line in qwen("read", path="EXAMPLE.txt").split("\n"))
+        text = "The format, indented as code:\n\n" + block + "\n\nIndentation makes it a code block in Markdown."
+        for calls, answer in self.each(text):
+            self.assertEqual(calls, [])
+            self.assertEqual(answer, text)
+
+    def test_a_plain_call_after_a_paragraph_is_still_a_call(self):
+        text = "Let me read it.\n" + qwen("read", path="a.txt")
+        for calls, _ in self.each(text):
+            self.assertEqual(calls, [("read", {"path": "a.txt"})])
+
+    def test_a_call_after_an_indented_block_ends_it(self):
+        text = "Example:\n\n    x = 1\n\nNow the real one:\n\n" + qwen("read", path="a.txt")
+        for calls, _ in self.each(text):
+            self.assertEqual(calls, [("read", {"path": "a.txt"})])
+
+
+if __name__ == "__main__":
+    unittest.main()

serve/test_parser_track.py

@@ -16,15 +16,25 @@ class ReferenceParser(OutputParser):
             if k < len(parts) - 1:
                 line, self.line = self.line + part, ""
                 s = line.lstrip()
+                was_fenced = bool(self.fence)
                 if self.fence:
-                    if s.startswith(self.fence * 3):
+                    if len(s) - len(s.lstrip(self.fence[0])) >= len(self.fence):   # #4702: as many fence characters as opened it
                         self.fence = ""
                 elif s[:3] in ("```", "~~~") and (s[0] * 3) not in s[3:]:
-                    self.fence = s[0]
+                    self.fence = s[0] * (len(s) - len(s.lstrip(s[0])))
                 elif not s:
                     self.ticks = 0
                 else:
                     self.ticks += line.count("`")
+                if was_fenced or self.fence:                                       # #4703: indented code blocks
+                    self._prev_blank = self._indented = False
+                elif not s:
+                    self._prev_blank = True
+                elif line.startswith(("    ", "\t")):
+                    self._indented = self._prev_blank or self._indented
+                    self._prev_blank = False
+                else:
+                    self._prev_blank = self._indented = False
             else:
                 self.line += part
         return text
@@ -32,14 +42,14 @@ class ReferenceParser(OutputParser):
 
 class TrackTests(unittest.TestCase):
     def assert_state(self, actual, expected):
-        self.assertEqual((actual.line, actual.fence, actual.ticks),
-                         (expected.line, expected.fence, expected.ticks))
+        self.assertEqual((actual.line, actual.fence, actual.ticks, actual._prev_blank, actual._indented),
+                         (expected.line, expected.fence, expected.ticks, expected._prev_blank, expected._indented))
         self.assertEqual(actual._opener_ok(), expected._opener_ok())
         self.assertEqual(actual._in_code(), expected._in_code())
 
     def test_fragment_state_from_varied_initial_states(self):
         fragments = ("", "a", "é🙂", "`", "``", "```", "~~~", "\r", "\n", "\r\n",
-                     "\n\n", "  ```python\n", "```quoted```\n", "~\ntext", "\t\nrest")
+                     "\n\n", "\n    code\n", "    x\n", "  ```python\n", "````md\n", "```quoted```\n", "~\ntext", "\t\nrest")
         for fence, line, ticks, text in itertools.product(("", "`", "~"),
                                                          ("", "  ", "``", "code `"),
                                                          (0, 1, 2, 3), fragments):