Mintplex-Labs/tiktoken
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/tiktoken.git synced 2026-07-01 18:48:04 -04:00
Code Issues Packages Projects Releases Wiki Activity

Correct other models as well

Browse Source
This commit is contained in:
Tat Dat Duong
2025-02-04 02:46:03 +01:00
parent 4dbb38e552
commit 9fc37ed57a
2 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tiktoken/model_to_encoding.json
+2 -2
View File
@@ -57,8 +57,8 @@
"gpt-4-vision-preview": "cl100k_base",
"gpt-4o": "o200k_base",
"gpt-4o-2024-05-13": "o200k_base",
"gpt-4o-2024-08-06":"o200k_base",
"gpt-4o-2024-11-20":"o200k_base",
"gpt-4o-2024-08-06": "o200k_base",
"gpt-4o-2024-11-20": "o200k_base",
"gpt-4o-mini-2024-07-18": "o200k_base",
"gpt-4o-mini": "o200k_base",
"o1": "o200k_base",
wasm/src/lib.rs
+4 -2
View File
@@ -460,9 +460,10 @@ pub fn encoding_for_model(
"text-babbage-001" => Ok("r50k_base"),
"text-ada-001" => Ok("r50k_base"),
"davinci" => Ok("r50k_base"),
"davinci-002" => Ok("cl100k_base"),
"curie" => Ok("r50k_base"),
"babbage" => Ok("r50k_base"),
"babbage-002" => Ok("r50k_base"),
"babbage-002" => Ok("cl100k_base"),
"ada" => Ok("r50k_base"),
"code-davinci-002" => Ok("p50k_base"),
"code-davinci-001" => Ok("p50k_base"),
@@ -491,7 +492,7 @@ pub fn encoding_for_model(
"gpt-3.5-turbo-0613" => Ok("cl100k_base"),
"gpt-3.5-turbo-16k" => Ok("cl100k_base"),
"gpt-3.5-turbo-16k-0613" => Ok("cl100k_base"),
"gpt-3.5-turbo-instruct" => Ok("clk100k_base"),
"gpt-3.5-turbo-instruct" => Ok("cl100k_base"),
"gpt-3.5-turbo-instruct-0914" => Ok("cl100k_base"),
"gpt-4" => Ok("cl100k_base"),
"gpt-4-0314" => Ok("cl100k_base"),
@@ -514,6 +515,7 @@ pub fn encoding_for_model(
"gpt-4o-2024-11-20" => Ok("o200k_base"),
"gpt-4o-mini-2024-07-18" => Ok("o200k_base"),
"gpt-4o-mini" => Ok("o200k_base"),
"o1" => Ok("o200k_base"),
"o1-2024-12-17" => Ok("o200k_base"),
"o1-mini" => Ok("o200k_base"),
"o1-preview" => Ok("o200k_base"),