mirror of
https://github.com/Mintplex-Labs/tiktoken.git
synced 2026-07-01 18:48:04 -04:00
Shantanu Jain
25 lines
625 B
Python
25 lines
625 B
Python
import subprocess
|
|
import sys
|
|
|
|
import tiktoken
|
|
|
|
|
|
def test_encoding_for_model():
|
|
enc = tiktoken.encoding_for_model("gpt2")
|
|
assert enc.name == "gpt2"
|
|
enc = tiktoken.encoding_for_model("text-davinci-003")
|
|
assert enc.name == "p50k_base"
|
|
enc = tiktoken.encoding_for_model("text-davinci-edit-001")
|
|
assert enc.name == "p50k_edit"
|
|
enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0301")
|
|
assert enc.name == "cl100k_base"
|
|
|
|
|
|
def test_optional_blobfile_dependency():
|
|
prog = """
|
|
import tiktoken
|
|
import sys
|
|
assert "blobfile" not in sys.modules
|
|
"""
|
|
subprocess.check_call([sys.executable, "-c", prog])
|