mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-10 03:41:36 -10:00
strata_tokenizer: the approved wording for an unknown pre-tokenizer
This commit is contained in:
1 parent
e17394fe1c
commit
37e91bfdbb
2 files changed
+5
-4
No files matched your search
@@ -154,7 +154,9 @@ class UnknownPreTokenizerAtStart(unittest.TestCase):
|
||||
server.main(["--engine", "mock", "--tokenizer", str(t), "--port", "0"])
|
||||
self.assertEqual(cm.exception.code, 2)
|
||||
self.assertIn("'qwen4'", err.getvalue())
|
||||
self.assertIn("is not loaded", err.getvalue())
|
||||
self.assertIn("so it isn't loaded", err.getvalue())
|
||||
self.assertIn("Strata doesn't have yet ('qwen4')", err.getvalue())
|
||||
self.assertIn("the model's tokenizer (", err.getvalue())
|
||||
self.assertNotIn("Traceback", err.getvalue())
|
||||
|
||||
|
||||
|
||||
@@ -120,9 +120,8 @@ class Tokenizer:
|
||||
if pre_tokenizer_kind(pre) is None:
|
||||
# Not a fallback: another model's pre-tokenizer splits text differently, so encoding with the qwen35 rule would
|
||||
# change every prompt without any error (llama.cpp has one rule per `tokenizer.ggml.pre` type for the same reason).
|
||||
raise UnknownPreTokenizer("this model's tokenizer uses the text-splitting rule %r, which this version of Strata does not have "
|
||||
"(it has qwen35 and deepseek). Encoding with another rule would quietly change what the model reads, "
|
||||
"so the model is not loaded." % (pre,))
|
||||
raise UnknownPreTokenizer("This model splits text with a rule Strata doesn't have yet (%r), so it isn't loaded: "
|
||||
"running it with the wrong rule would change what it reads." % (pre,))
|
||||
self.token_types = token_types
|
||||
self.special_ids = special_ids or {}
|
||||
self.ids = {t: i for i, t in enumerate(tokens)}
|
||||
|
||||
Reference in new issue
Block a user