strata_tokenizer: the approved wording for an unknown pre-tokenizer

This commit is contained in:
Niko1221 committed 2026-11-06 19:19:52 +01:00
1 parent e17394fe1c
commit 37e91bfdbb
2 files changed
+5 -4

No files matched your search

+3 -1
View File
@@ -154,7 +154,9 @@ class UnknownPreTokenizerAtStart(unittest.TestCase):
server.main(["--engine", "mock", "--tokenizer", str(t), "--port", "0"])
self.assertEqual(cm.exception.code, 2)
self.assertIn("'qwen4'", err.getvalue())
self.assertIn("is not loaded", err.getvalue())
self.assertIn("so it isn't loaded", err.getvalue())
self.assertIn("Strata doesn't have yet ('qwen4')", err.getvalue())
self.assertIn("the model's tokenizer (", err.getvalue())
self.assertNotIn("Traceback", err.getvalue())
+2 -3
View File
@@ -120,9 +120,8 @@ class Tokenizer:
if pre_tokenizer_kind(pre) is None:
# Not a fallback: another model's pre-tokenizer splits text differently, so encoding with the qwen35 rule would
# change every prompt without any error (llama.cpp has one rule per `tokenizer.ggml.pre` type for the same reason).
raise UnknownPreTokenizer("this model's tokenizer uses the text-splitting rule %r, which this version of Strata does not have "
"(it has qwen35 and deepseek). Encoding with another rule would quietly change what the model reads, "
"so the model is not loaded." % (pre,))
raise UnknownPreTokenizer("This model splits text with a rule Strata doesn't have yet (%r), so it isn't loaded: "
"running it with the wrong rule would change what it reads." % (pre,))
self.token_types = token_types
self.special_ids = special_ids or {}
self.ids = {t: i for i, t in enumerate(tokens)}