tokenizer: _bpe reads self once instead of per symbol pair, and a threads-vs-serial encode test (#1385; the 'int has no attribute ranks' fault could not be reproduced and looks like an interpreter bug)

serve/test_detok.py

@@ -191,6 +191,36 @@ def synthetic_tokenizer(seed=268, n_merges=1500):
     return ST.Tokenizer(tokens, merges, types)
 
 
+class ThreadedEncode(unittest.TestCase):
+    """#1385: 'int' object has no attribute 'ranks' in _bpe, seen once under load.  Could not be reproduced here (it
+    looks like an interpreter fault); this pins that many threads sharing one Tokenizer, as the server does, get
+    exactly the single-threaded ids, and that _bpe works with a tokenizer whose own state is only read."""
+
+    def test_threads_match_serial(self):
+        import threading
+        tok = synthetic_tokenizer()
+        rng = random.Random(1385)
+        texts = [random_text(rng, 6) for _ in range(40)]
+        want = [tok.encode(t) for t in texts]
+        errors = []
+
+        def work():
+            try:
+                for _ in range(15):
+                    for t, w in zip(texts, want):
+                        if tok.encode(t) != w:
+                            errors.append("mismatch")
+            except Exception as e:                  # noqa: BLE001
+                errors.append(repr(e))
+
+        ts = [threading.Thread(target=work) for _ in range(8)]
+        for t in ts:
+            t.start()
+        for t in ts:
+            t.join()
+        self.assertEqual(errors, [])
+
+
 class HeapBpe(unittest.TestCase):
     """#268: words longer than HEAP_MIN symbols merge with a heap; the ids must be exactly the scan's."""
 

tools/strata_tokenizer.py

@@ -150,11 +150,16 @@ class Tokenizer:
         """
         if len(word) > self.HEAP_MIN:
             return self._bpe_heap(word)
+        # `self` is read ONCE, here.  #1385: an interpreter (CPython 3.14.4) was seen handing this frame an int
+        # for `self` partway through the scan (`'int' object has no attribute 'ranks'`).  Nothing in this class
+        # can do that (no cache, decorator, slots or callback; a thread hammering test cannot make it happen),
+        # so the scan below works on a local and no longer re-reads `self` once per symbol pair.
+        ranks_get = self.ranks.get
         parts = list(word)
         while len(parts) > 1:
             best, best_rank = None, None
             for i in range(len(parts) - 1):
-                r = self.ranks.get((parts[i], parts[i + 1]))
+                r = ranks_get((parts[i], parts[i + 1]))
                 if r is not None and (best_rank is None or r < best_rank):
                     best, best_rank = i, r
             if best is None: