From db9a80452e095fe07f0d14dfb301a840646ea4e9 Mon Sep 17 00:00:00 2001 From: Prince Mendiratta Date: Tue, 19 Sep 2023 10:46:33 +0530 Subject: [PATCH] feat: add gpt3.5-turbo-instruct model Signed-off-by: Prince Mendiratta --- js/src/core.ts | 2 ++ tiktoken/model_to_encoding.json | 2 ++ wasm/src/lib.rs | 2 ++ 3 files changed, 6 insertions(+) diff --git a/js/src/core.ts b/js/src/core.ts index 6616e53..4f75934 100644 --- a/js/src/core.ts +++ b/js/src/core.ts @@ -247,6 +247,8 @@ export function getEncodingNameForModel(model: TiktokenModel) { case "text-similarity-davinci-001": { return "r50k_base"; } + case "gpt-3.5-turbo-instruct-0914": + case "gpt-3.5-turbo-instruct": case "gpt-3.5-turbo-16k-0613": case "gpt-3.5-turbo-16k": case "gpt-3.5-turbo-0613": diff --git a/tiktoken/model_to_encoding.json b/tiktoken/model_to_encoding.json index a7a244b..7d17623 100644 --- a/tiktoken/model_to_encoding.json +++ b/tiktoken/model_to_encoding.json @@ -34,6 +34,8 @@ "gpt-3.5-turbo-0613": "cl100k_base", "gpt-3.5-turbo-16k": "cl100k_base", "gpt-3.5-turbo-16k-0613": "cl100k_base", + "gpt-3.5-turbo-instruct": "gpt-3.5-turbo-instruct", + "gpt-3.5-turbo-instruct-0914": "gpt-3.5-turbo-instruct-0914", "gpt-4": "cl100k_base", "gpt-4-0314": "cl100k_base", "gpt-4-0613": "cl100k_base", diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index ef8138f..6743f04 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -442,6 +442,8 @@ pub fn encoding_for_model( "gpt-3.5-turbo-0613" => Ok("cl100k_base"), "gpt-3.5-turbo-16k" => Ok("cl100k_base"), "gpt-3.5-turbo-16k-0613" => Ok("cl100k_base"), + "gpt-3.5-turbo-instruct" => Ok("clk100k_base") + "gpt-3.5-turbo-instruct-0914" => Ok("cl100k_base"), "gpt-4" => Ok("cl100k_base"), "gpt-4-0314" => Ok("cl100k_base"), "gpt-4-0613" => Ok("cl100k_base"),