feat: use gpt-tokenizer (#1352)

.changeset/friendly-chicken-switch.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/env": patch
+"llamaindex": patch
+---
+
+feat: use `gpt-tokenizer`

packages/env/package.json

@@ -75,8 +75,8 @@
     "@swc/core": "^1.7.22",
     "@xenova/transformers": "^2.17.2",
     "concurrently": "^8.2.2",
+    "gpt-tokenizer": "^2.5.0",
     "pathe": "^1.1.2",
-    "tiktoken": "^1.0.16",
     "vitest": "^2.0.5"
   },
   "dependencies": {
@@ -85,9 +85,9 @@
   "peerDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@xenova/transformers": "^2.17.2",
+    "gpt-tokenizer": "^2.5.0",
     "js-tiktoken": "^1.0.12",
-    "pathe": "^1.1.2",
-    "tiktoken": "^1.0.15"
+    "pathe": "^1.1.2"
   },
   "peerDependenciesMeta": {
     "@aws-crypto/sha256-js": {

packages/env/src/tokenizers/node.ts

@@ -2,21 +2,18 @@
 import type { Tokenizer } from "./types.js";
 import { Tokenizers } from "./types.js";
 
-import { get_encoding } from "tiktoken";
+import cl100kBase from "gpt-tokenizer";
 
 class TokenizerSingleton {
-  private defaultTokenizer: Tokenizer;
+  #defaultTokenizer: Tokenizer;
 
   constructor() {
-    const encoding = get_encoding("cl100k_base");
-
-    this.defaultTokenizer = {
-      encode: (text: string) => {
-        return encoding.encode(text);
+    this.#defaultTokenizer = {
+      encode: (text: string): Uint32Array => {
+        return new Uint32Array(cl100kBase.encode(text));
       },
       decode: (tokens: Uint32Array) => {
-        const text = encoding.decode(tokens);
-        return new TextDecoder().decode(text);
+        return cl100kBase.decode(tokens);
       },
     };
   }
@@ -26,7 +23,7 @@ class TokenizerSingleton {
       throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
     }
 
-    return this.defaultTokenizer;
+    return this.#defaultTokenizer;
   }
 }
 

packages/env/tests/tokenizer.test.ts

@@ -0,0 +1,11 @@
+import { describe, expect, it } from "vitest";
+import { tokenizers } from "../src/tokenizers/node.js";
+
+describe("tokenizer", () => {
+  it("should tokenize text", () => {
+    const tokenizer = tokenizers.tokenizer();
+    expect(tokenizer.decode(tokenizer.encode("hello world"))).toBe(
+      "hello world",
+    );
+  });
+});

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -10,17 +10,16 @@
   },
   "dependencies": {
     "llamaindex": "workspace:*",
-    "react": "19.0.0-rc-7771d3a7-20240827",
-    "react-dom": "19.0.0-rc-7771d3a7-20240827",
-    "react-server-dom-webpack": "19.0.0-rc-7771d3a7-20240827",
-    "waku": "0.21.1"
+    "react": "19.0.0-rc-bf7e210c-20241017",
+    "react-dom": "19.0.0-rc-bf7e210c-20241017",
+    "react-server-dom-webpack": "19.0.0-rc-bf7e210c-20241017",
+    "waku": "0.21.4"
   },
   "devDependencies": {
-    "@types/react": "18.3.5",
-    "@types/react-dom": "18.3.0",
-    "autoprefixer": "10.4.20",
-    "tailwindcss": "3.4.10",
-    "typescript": "5.6.2",
-    "vite-plugin-wasm": "^3.3.0"
+    "@types/react": "18.3.11",
+    "@types/react-dom": "18.3.1",
+    "autoprefixer": "^10.4.20",
+    "tailwindcss": "^3.4.14",
+    "typescript": "5.6.2"
   }
 }

packages/llamaindex/e2e/examples/waku-query-engine/vite.config.ts

@@ -1,8 +0,0 @@
-import wasm from "vite-plugin-wasm";
-
-export default {
-  plugins: [wasm()],
-  ssr: {
-    external: ["tiktoken"],
-  },
-};

packages/llamaindex/package.json

@@ -56,6 +56,7 @@
     "chromadb": "1.9.2",
     "cohere-ai": "7.13.0",
     "discord-api-types": "^0.37.98",
+    "gpt-tokenizer": "^2.5.0",
     "groq-sdk": "^0.6.1",
     "js-tiktoken": "^1.0.14",
     "lodash": "^4.17.21",
@@ -69,7 +70,6 @@
     "pathe": "^1.1.2",
     "rake-modified": "^1.0.8",
     "string-strip-html": "^13.4.8",
-    "tiktoken": "^1.0.15",
     "unpdf": "^0.11.0",
     "weaviate-client": "^3.1.4",
     "wikipedia": "^2.1.2",

packages/llamaindex/src/next.ts

@@ -17,12 +17,6 @@
  */
 export default function withLlamaIndex(config: any) {
   config.experimental = config.experimental ?? {};
-  // copy tiktoken WASM files to the NextJS build
-  config.experimental.outputFileTracingIncludes =
-    config.experimental.outputFileTracingIncludes ?? {};
-  config.experimental.outputFileTracingIncludes["/**/*"] = [
-    "./node_modules/tiktoken/*.wasm",
-  ];
   // needed for transformers, see https://huggingface.co/docs/transformers.js/en/tutorials/next#step-2-install-and-configure-transformersjs
   config.experimental.serverComponentsExternalPackages =
     config.experimental.serverComponentsExternalPackages ?? [];