diff --git a/js/src/core.ts b/js/src/core.ts index 6616e53..f413ccf 100644 --- a/js/src/core.ts +++ b/js/src/core.ts @@ -219,6 +219,7 @@ export function getEncodingNameForModel(model: TiktokenModel) { case "code-davinci-002": case "cushman-codex": case "davinci-codex": + case "davinci-002": case "text-davinci-002": case "text-davinci-003": { return "p50k_base"; @@ -229,6 +230,7 @@ export function getEncodingNameForModel(model: TiktokenModel) { } case "ada": case "babbage": + case "babbage-002": case "code-search-ada-code-001": case "code-search-babbage-code-001": case "curie": @@ -258,6 +260,10 @@ export function getEncodingNameForModel(model: TiktokenModel) { case "gpt-4-0613": case "gpt-4-0314": case "gpt-4": + case "gpt-3.5-turbo-1106": + case "gpt-35-turbo": + case "gpt-4-1106-preview": + case "gpt-4-vision-preview": case "text-embedding-ada-002": { return "cl100k_base"; } diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index ef8138f..aefe246 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -350,6 +350,8 @@ pub fn get_encoding(encoding: &str, extend_special_tokens: JsValue) -> Result Result { let encoding = match model { + "davinci" => Ok("p50k_base"), "text-davinci-003" => Ok("p50k_base"), "text-davinci-002" => Ok("p50k_base"), "text-davinci-001" => Ok("r50k_base"), @@ -416,6 +423,7 @@ pub fn encoding_for_model( "davinci" => Ok("r50k_base"), "curie" => Ok("r50k_base"), "babbage" => Ok("r50k_base"), + "babbage-002" => Ok("r50k_base"), "ada" => Ok("r50k_base"), "code-davinci-002" => Ok("p50k_base"), "code-davinci-001" => Ok("p50k_base"), @@ -448,6 +456,10 @@ pub fn encoding_for_model( "gpt-4-32k" => Ok("cl100k_base"), "gpt-4-32k-0314" => Ok("cl100k_base"), "gpt-4-32k-0613" => Ok("cl100k_base"), + "gpt-3.5-turbo-1106" => Ok("cl100k_base"), + "gpt-35-turbo" => Ok("cl100k_base"), + "gpt-4-1106-preview" => Ok("cl100k_base"), + "gpt-4-vision-preview" => Ok("cl100k_base"), model => Err(JsError::new( format!("Invalid model: {}", model.to_string()).as_str(), )),