From 9fc37ed57a1418a3012d4a7968a2d1fe62c7660e Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Tue, 4 Feb 2025 02:46:03 +0100 Subject: [PATCH] Correct other models as well --- tiktoken/model_to_encoding.json | 4 ++-- wasm/src/lib.rs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tiktoken/model_to_encoding.json b/tiktoken/model_to_encoding.json index 9a56d86..6b55f31 100644 --- a/tiktoken/model_to_encoding.json +++ b/tiktoken/model_to_encoding.json @@ -57,8 +57,8 @@ "gpt-4-vision-preview": "cl100k_base", "gpt-4o": "o200k_base", "gpt-4o-2024-05-13": "o200k_base", - "gpt-4o-2024-08-06":"o200k_base", - "gpt-4o-2024-11-20":"o200k_base", + "gpt-4o-2024-08-06": "o200k_base", + "gpt-4o-2024-11-20": "o200k_base", "gpt-4o-mini-2024-07-18": "o200k_base", "gpt-4o-mini": "o200k_base", "o1": "o200k_base", diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 53ef87a..65b29a0 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -460,9 +460,10 @@ pub fn encoding_for_model( "text-babbage-001" => Ok("r50k_base"), "text-ada-001" => Ok("r50k_base"), "davinci" => Ok("r50k_base"), + "davinci-002" => Ok("cl100k_base"), "curie" => Ok("r50k_base"), "babbage" => Ok("r50k_base"), - "babbage-002" => Ok("r50k_base"), + "babbage-002" => Ok("cl100k_base"), "ada" => Ok("r50k_base"), "code-davinci-002" => Ok("p50k_base"), "code-davinci-001" => Ok("p50k_base"), @@ -491,7 +492,7 @@ pub fn encoding_for_model( "gpt-3.5-turbo-0613" => Ok("cl100k_base"), "gpt-3.5-turbo-16k" => Ok("cl100k_base"), "gpt-3.5-turbo-16k-0613" => Ok("cl100k_base"), - "gpt-3.5-turbo-instruct" => Ok("clk100k_base"), + "gpt-3.5-turbo-instruct" => Ok("cl100k_base"), "gpt-3.5-turbo-instruct-0914" => Ok("cl100k_base"), "gpt-4" => Ok("cl100k_base"), "gpt-4-0314" => Ok("cl100k_base"), @@ -514,6 +515,7 @@ pub fn encoding_for_model( "gpt-4o-2024-11-20" => Ok("o200k_base"), "gpt-4o-mini-2024-07-18" => Ok("o200k_base"), "gpt-4o-mini" => Ok("o200k_base"), + "o1" => Ok("o200k_base"), "o1-2024-12-17" => Ok("o200k_base"), "o1-mini" => Ok("o200k_base"), "o1-preview" => Ok("o200k_base"),