mirror of
https://github.com/Mintplex-Labs/tiktoken.git
synced 2026-07-01 18:48:04 -04:00
Shantanu Jain
23 lines
477 B
Python
23 lines
477 B
Python
import bisect
|
|
import functools
|
|
import os
|
|
|
|
import pytest
|
|
|
|
import tiktoken
|
|
|
|
MAX_EXAMPLES: int = int(os.environ.get("TIKTOKEN_MAX_EXAMPLES", "100"))
|
|
|
|
ENCODINGS = ["r50k_base", "cl100k_base"]
|
|
SOME_ENCODINGS = ["cl100k_base"]
|
|
|
|
|
|
ENCODING_FACTORIES = [
|
|
pytest.param(functools.partial(tiktoken.get_encoding, name), id=name) for name in ENCODINGS
|
|
]
|
|
SOME_ENCODING_FACTORIES = [
|
|
pytest.param(functools.partial(tiktoken.get_encoding, name), id=name) for name in SOME_ENCODINGS
|
|
]
|
|
|
|
|