mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 06:15:58 -10:00
Niko1221
43f32b3046
Service took the ids of "<|im_end|>" and "<|endoftext|>" as its stop ids whatever the tokenizer made of them. DeepSeek-V4.1's tokenizer has neither as one token: it cuts them into '<', '|', 'im', '_end', '|', '>' and 'end', 'of', 'text'. A server run without the family named (a hand-written config, a gateway) therefore ended the reply in front of the first "im", "|", ">", "end", "of" or "text" it wrote, and the real end token (id 1) did not end it. Nerazim is N, er, az, im: the reply stopped at 'im' and the client got "Neraz". The engine and its --eos 1 were fine; nothing was cut by the stop-string hold-back (StopMatcher gives its held tail back at the end). end_token_ids() keeps a marker only where it is a single token, and falls back to the tokenizer's own end token (special_ids "eos"). Qwen's tokenizers are unchanged. The family guess from the previous commit still names the right parser and DSML for DeepSeek. serve/test_stop_holdback.py: the end ids of both tokenizers; StopMatcher over every split of the text and one character at a time (Qwen and DeepSeek stop lists, words ending like the start of a stop string); the Detokenizer plus StopMatcher byte by byte; and the OpenAI and /v1/messages routes, streamed and whole, with and without the family named: no stop string (whole reply), a stop string that never comes (the held tail is given back), a stop string (cut exactly there).
279 lines
14 KiB
Python
279 lines
14 KiB
Python
"""The end of a reply and the text held back in front of a stop string (#4576: "Nerazim" came out as "Neraz").
|
||
|
||
DeepSeek-V4.1's tokenizer has no single token for Qwen's <|im_end|> / <|endoftext|>: it splits them into '<', '|', 'im', '_end',
|
||
'|', '>' and 'end', 'of', 'text'. A server that took the ids of those pieces as its stop ids (a config without "arch") ended the
|
||
reply in front of the first "im" it wrote: Nerazim = N, er, az, im. These tests keep three things true, for the OpenAI and the
|
||
Anthropic route, streamed and not, with and without the family named:
|
||
|
||
* the reply that has no stop string comes out whole (every word that ends in a piece of a stop string or of an end marker);
|
||
* a stop string cuts the reply exactly there, whatever the chunking;
|
||
* the text held back for a stop string that does not come is given back at the end.
|
||
|
||
python -m unittest serve.test_stop_holdback -v
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import sys
|
||
import unittest
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT))
|
||
sys.path.insert(0, str(ROOT / "tools"))
|
||
import strata_tokenizer as ST # noqa: E402
|
||
from serve import frontend, server # noqa: E402
|
||
from serve.frontend import ChatTemplate # noqa: E402
|
||
from serve.server import ByteTokenizer, Detokenizer, Service, StopMatcher, end_token_ids, serve # noqa: E402
|
||
from serve.test_dsv41 import EOS, SPECIALS, TEMPLATE, DsEngine # noqa: E402
|
||
|
||
QWEN_STOPS = ["<|im_end|>", "<|im_start|>", "<|endoftext|>", "\nUser:"]
|
||
DEEPSEEK_STOPS = ["<｜end▁of▁sentence｜>", "<｜User｜>", "</s>", "\n\nHuman:"]
|
||
# words and endings that look like the start of a stop string or a piece of an end marker
|
||
WORDS = ["Nerazim", "Zim", "swim", "Neraz", "im", "im_end", "Nerazimville", "a|b|c|", "x > y and a < b", "the end of the text",
|
||
"<", "<|", "<|im", "<|im_end", "|>", "friend", "legend", "\nUser", "\n\nHuman", "</", "</s", "<｜User", "<｜", "Nerazim<",
|
||
"Nerazim|", "Nerazim <|im_start|"]
|
||
|
||
|
||
def nerazim_tokenizer() -> ST.Tokenizer:
|
||
"""DeepSeek's specials, the 256 byte tokens and the pieces of the real vocabulary that matter here: N er az im ('im' is id
|
||
328 in the real one), and the pieces its tokenizer cuts Qwen's end markers into."""
|
||
b2u = ST.bytes_to_unicode()
|
||
extra = ["er", "az", "im", "_e", "_en", "_end", "en", "end", "of", "te", "tex", "text"]
|
||
merges = ["e r", "a z", "i m", "_ e", "_e n", "_en d", "e n", "en d", "o f", "t e", "te x", "tex t"]
|
||
tokens = [s for s, _ in SPECIALS] + [b2u[b] for b in range(256)] + extra
|
||
types = [t for _, t in SPECIALS] + [1] * (256 + len(extra))
|
||
return ST.Tokenizer(tokens, merges, types, "deepseek3", {"bos": 0, "eos": EOS})
|
||
|
||
|
||
class Fixture(unittest.TestCase):
|
||
@classmethod
|
||
def setUpClass(cls):
|
||
cls.tok = nerazim_tokenizer()
|
||
|
||
def test_the_fixture_has_the_real_tokenizer_shape(self):
|
||
self.assertEqual([self.tok.tokens[i] for i in self.tok.encode("Nerazim")], ["N", "er", "az", "im"])
|
||
pieces = [self.tok.tokens[i] for i in self.tok.encode("<|im_end|>", parse_special=True)]
|
||
self.assertEqual(pieces, ["<", "|", "im", "_end", "|", ">"])
|
||
pieces = [self.tok.tokens[i] for i in self.tok.encode("<|endoftext|>", parse_special=True)]
|
||
self.assertEqual(pieces, ["<", "|", "end", "of", "text", "|", ">"])
|
||
|
||
|
||
class EndTokenIds(Fixture):
|
||
def test_pieces_of_the_qwen_markers_are_not_stop_ids(self):
|
||
ids = end_token_ids(self.tok)
|
||
self.assertEqual(ids, {EOS})
|
||
pieces = set(self.tok.encode("<|im_end|>", parse_special=True)) | set(self.tok.encode("<|endoftext|>", parse_special=True))
|
||
self.assertFalse(ids & pieces)
|
||
|
||
def test_qwen_markers_that_are_one_token_each_are_the_stop_ids(self):
|
||
tok = ByteTokenizer()
|
||
self.assertEqual(end_token_ids(tok), set(tok.encode("<|im_end|>", parse_special=True))
|
||
| set(tok.encode("<|endoftext|>", parse_special=True)))
|
||
self.assertEqual(len(end_token_ids(tok)), 2)
|
||
|
||
def test_no_marker_and_no_eos_means_no_stop_id(self):
|
||
class Bare:
|
||
def encode(self, text, parse_special=False):
|
||
return list(text.encode())
|
||
self.assertEqual(end_token_ids(Bare()), set())
|
||
|
||
def test_a_service_without_a_family_does_not_stop_on_im(self):
|
||
svc = Service(DsEngine(self.tok, "x"), self.tok, ChatTemplate(TEMPLATE))
|
||
im = self.tok.encode("im")[0]
|
||
for piece in ("im", "<", "|", ">", "end", "of", "text", "_end"):
|
||
self.assertNotIn(self.tok.encode(piece)[0], svc.stop_ids, piece)
|
||
self.assertIn(im, set(self.tok.encode("<|im_end|>", parse_special=True)))
|
||
self.assertEqual(svc.stop_ids, {EOS})
|
||
|
||
|
||
class StopMatcherSplits(unittest.TestCase):
|
||
"""StopMatcher alone: every way to cut the text in two, and one character at a time."""
|
||
|
||
@staticmethod
|
||
def run_matcher(stops, chunks):
|
||
m = StopMatcher(stops)
|
||
out = "".join(m.push(c) for c in chunks)
|
||
if m.hit is None:
|
||
out += m.flush()
|
||
return out, m.hit
|
||
|
||
def chunkings(self, text):
|
||
yield [text]
|
||
for i in range(1, len(text)):
|
||
yield [text[:i], text[i:]]
|
||
yield list(text)
|
||
|
||
def test_text_without_a_stop_string_comes_out_whole_at_every_split(self):
|
||
for stops in (QWEN_STOPS, DEEPSEEK_STOPS, ["im_end"], ["im"], ["zim"], ["|", ">"], ["end"]):
|
||
for word in WORDS:
|
||
text = "My friend " + word
|
||
if any(s in text for s in stops):
|
||
continue
|
||
for chunks in self.chunkings(text):
|
||
with self.subTest(stops=stops, text=text, chunks=chunks):
|
||
self.assertEqual(self.run_matcher(stops, chunks), (text, None))
|
||
|
||
def test_a_stop_string_cuts_exactly_there_at_every_split(self):
|
||
for stops in (QWEN_STOPS, DEEPSEEK_STOPS):
|
||
for stop in stops:
|
||
for word in ("Nerazim", "swim"):
|
||
text = f"My friend {word}{stop}never seen"
|
||
for chunks in self.chunkings(text):
|
||
with self.subTest(stops=stops, text=text, chunks=chunks):
|
||
first = min((text.find(s), s) for s in stops if s in text)
|
||
self.assertEqual(self.run_matcher(stops, chunks), (text[:first[0]], first[1]))
|
||
|
||
def test_the_held_back_tail_is_given_back_at_the_end(self):
|
||
for stops in (QWEN_STOPS, DEEPSEEK_STOPS):
|
||
for text in ("Nerazim <|im", "Nerazim <｜User", "Nerazim\nUser", "Nerazim <", "ends with a pipe |"):
|
||
with self.subTest(stops=stops, text=text):
|
||
self.assertEqual(self.run_matcher(stops, list(text)), (text, None))
|
||
|
||
def test_a_stop_string_that_starts_like_the_word_costs_nothing_but_the_wait(self):
|
||
# "im_end" as a stop: "Nerazim" holds "im" until the next character says it is not the stop string
|
||
m = StopMatcher(["im_end"])
|
||
self.assertEqual(m.push("Neraz"), "Neraz")
|
||
self.assertEqual(m.push("im"), "")
|
||
self.assertEqual(m.held, "im")
|
||
self.assertEqual(m.flush(), "im")
|
||
|
||
|
||
class ByteWise(Fixture):
|
||
"""The text path as the server runs it: one token per byte through the Detokenizer (a UTF-8 character split across tokens is held
|
||
until it is whole), then the StopMatcher."""
|
||
|
||
def run_bytes(self, stops, text):
|
||
tok = ByteTokenizer()
|
||
detok, m, out = Detokenizer(tok), StopMatcher(stops), ""
|
||
for b in text.encode("utf-8"):
|
||
out += m.push(detok.push(b))
|
||
if m.hit is not None:
|
||
break
|
||
if m.hit is None:
|
||
out += m.flush()
|
||
return out, m.hit
|
||
|
||
def test_every_word_comes_out_whole_byte_by_byte(self):
|
||
for stops in (QWEN_STOPS, DEEPSEEK_STOPS):
|
||
for word in WORDS + ["Zażółć", "日本語の名前"]:
|
||
text = "Hello " + word
|
||
if any(s in text for s in stops):
|
||
continue
|
||
with self.subTest(stops=stops, text=text):
|
||
self.assertEqual(self.run_bytes(stops, text), (text, None))
|
||
|
||
def test_a_multibyte_stop_string_is_cut_byte_by_byte(self):
|
||
text = "Nerazim<｜end▁of▁sentence｜>not shown"
|
||
self.assertEqual(self.run_bytes(DEEPSEEK_STOPS, text), ("Nerazim", "<｜end▁of▁sentence｜>"))
|
||
|
||
|
||
class Routes(Fixture):
|
||
"""The four ways out of the server (OpenAI and Anthropic, streamed and whole), with the family named and without it."""
|
||
|
||
def start(self, script, named):
|
||
"""A server for `named` (one per test and setting, reused); the engine replays `script` for the next request."""
|
||
if named in self.servers:
|
||
engine = self.servers[named]
|
||
engine.script = self.tok.encode(script) + [EOS] # (a marker in the script is text the model wrote)
|
||
self.base = self.bases[named]
|
||
return engine
|
||
engine = self.servers[named] = DsEngine(self.tok, script)
|
||
engine.script = self.tok.encode(script) + [EOS]
|
||
svc = Service(engine, self.tok, ChatTemplate(TEMPLATE), model_name="deepseek-v4.1-flash")
|
||
if named:
|
||
svc.model_arch = "deepseek41"
|
||
svc.set_family("deepseek41")
|
||
httpd = serve(svc, port=0)
|
||
self.addCleanup(httpd.server_close)
|
||
self.addCleanup(httpd.shutdown)
|
||
self.addCleanup(frontend.use_family, None)
|
||
self.base = self.bases[named] = f"http://127.0.0.1:{httpd.server_address[1]}"
|
||
return engine
|
||
|
||
def send(self, route, stream, stops):
|
||
if route == "openai":
|
||
path = "/v1/chat/completions"
|
||
body = {"model": "m", "max_tokens": 200, "stream": stream, "chat_template_kwargs": {"enable_thinking": False},
|
||
"messages": [{"role": "user", "content": "Use the name Nerazim in a sentence."}]}
|
||
if stops:
|
||
body["stop"] = stops[:4]
|
||
else:
|
||
path = "/v1/messages"
|
||
body = {"model": "m", "max_tokens": 200, "stream": stream, "thinking": {"type": "disabled"},
|
||
"messages": [{"role": "user", "content": "Use the name Nerazim in a sentence."}]}
|
||
if stops:
|
||
body["stop_sequences"] = stops
|
||
req = urllib.request.Request(self.base + path, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"})
|
||
with urllib.request.urlopen(req, timeout=30) as r:
|
||
raw = r.read().decode("utf-8")
|
||
text, finish, stop_seq = "", None, None
|
||
if not stream:
|
||
d = json.loads(raw)
|
||
if route == "openai":
|
||
text, finish = d["choices"][0]["message"].get("content") or "", d["choices"][0]["finish_reason"]
|
||
else:
|
||
text = "".join(b.get("text", "") for b in d["content"] if b["type"] == "text")
|
||
finish, stop_seq = d["stop_reason"], d.get("stop_sequence")
|
||
return text, finish, stop_seq
|
||
for line in raw.splitlines():
|
||
if not line.startswith("data:") or line.strip() == "data: [DONE]":
|
||
continue
|
||
d = json.loads(line[5:])
|
||
if route == "openai":
|
||
for ch in d.get("choices", []):
|
||
text += (ch.get("delta") or {}).get("content") or ""
|
||
finish = ch.get("finish_reason") or finish
|
||
elif d.get("type") == "content_block_delta":
|
||
text += d["delta"].get("text", "")
|
||
elif d.get("type") == "message_delta":
|
||
finish, stop_seq = d["delta"].get("stop_reason"), d["delta"].get("stop_sequence")
|
||
return text, finish, stop_seq
|
||
|
||
def setUp(self):
|
||
self.servers = {}
|
||
self.bases = {}
|
||
|
||
def every_route(self):
|
||
for named in (False, True):
|
||
for route in ("openai", "anthropic"):
|
||
for stream in (False, True):
|
||
yield named, route, stream
|
||
|
||
def test_nerazim_and_its_kin_come_out_whole_with_no_stop_string(self):
|
||
for word in ("Nerazim", "Nerazimville and Nerazim", "swim", "Zim", "a|b|c|", "x > y", "the end of the text", "im_end", "<", "Nerazim<|im"):
|
||
for named, route, stream in self.every_route():
|
||
with self.subTest(word=word, named=named, route=route, stream=stream):
|
||
self.start(word, named)
|
||
text, finish, _ = self.send(route, stream, None)
|
||
self.assertEqual(text.strip(), word)
|
||
self.assertIn(finish, ("stop", "end_turn"))
|
||
|
||
def test_the_held_tail_is_given_back_when_the_stop_string_never_comes(self):
|
||
for stops in (QWEN_STOPS, DEEPSEEK_STOPS, ["im_end", "|>"]):
|
||
for named, route, stream in self.every_route():
|
||
for word in ("My friend Nerazim", "Nerazim <|im", "Nerazim <｜User", "ends with a pipe |"):
|
||
with self.subTest(stops=stops, word=word, named=named, route=route, stream=stream):
|
||
self.start(word, named)
|
||
text, finish, stop_seq = self.send(route, stream, stops)
|
||
self.assertEqual(text.strip(), word)
|
||
self.assertIn(finish, ("stop", "end_turn"))
|
||
self.assertIsNone(stop_seq)
|
||
|
||
def test_a_stop_string_cuts_the_reply_there(self):
|
||
for stops in (QWEN_STOPS, DEEPSEEK_STOPS):
|
||
for named, route, stream in self.every_route():
|
||
for stop in stops:
|
||
with self.subTest(stops=stops, stop=stop, named=named, route=route, stream=stream):
|
||
self.start("My friend Nerazim" + stop + "and more", named)
|
||
text, finish, stop_seq = self.send(route, stream, stops)
|
||
self.assertEqual(text.strip(), "My friend Nerazim")
|
||
self.assertIn(finish, ("stop", "end_turn", "stop_sequence"))
|
||
if route == "anthropic":
|
||
self.assertEqual(stop_seq, stop)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|