Mintplex-Labs/tiktoken
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/tiktoken.git synced 2026-07-19 13:54:01 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat: add gpt3.5-turbo-instruct model

Browse Source 
Signed-off-by: Prince Mendiratta <prince.mendi@gmail.com>
This commit is contained in:
Prince Mendiratta
2023-09-19 10:46:33 +05:30
parent 072dd12962
commit db9a80452e
3 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/src/core.ts
+2
View File
@@ -247,6 +247,8 @@ export function getEncodingNameForModel(model: TiktokenModel) {
case "text-similarity-davinci-001": {
return "r50k_base";
}
case "gpt-3.5-turbo-instruct-0914":
case "gpt-3.5-turbo-instruct":
case "gpt-3.5-turbo-16k-0613":
case "gpt-3.5-turbo-16k":
case "gpt-3.5-turbo-0613":
tiktoken/model_to_encoding.json
+2
View File
@@ -34,6 +34,8 @@
"gpt-3.5-turbo-0613": "cl100k_base",
"gpt-3.5-turbo-16k": "cl100k_base",
"gpt-3.5-turbo-16k-0613": "cl100k_base",
"gpt-3.5-turbo-instruct": "gpt-3.5-turbo-instruct",
"gpt-3.5-turbo-instruct-0914": "gpt-3.5-turbo-instruct-0914",
"gpt-4": "cl100k_base",
"gpt-4-0314": "cl100k_base",
"gpt-4-0613": "cl100k_base",
wasm/src/lib.rs
+2
View File
@@ -442,6 +442,8 @@ pub fn encoding_for_model(
"gpt-3.5-turbo-0613" => Ok("cl100k_base"),
"gpt-3.5-turbo-16k" => Ok("cl100k_base"),
"gpt-3.5-turbo-16k-0613" => Ok("cl100k_base"),
"gpt-3.5-turbo-instruct" => Ok("clk100k_base")
"gpt-3.5-turbo-instruct-0914" => Ok("cl100k_base"),
"gpt-4" => Ok("cl100k_base"),
"gpt-4-0314" => Ok("cl100k_base"),
"gpt-4-0613" => Ok("cl100k_base"),