Merge main into sweep/vercel-edge-runtime-support

README.md

@@ -25,14 +25,15 @@ pnpm install llamaindex
 pnpm install @types/node
 ```
 
-Create the file example.ts
-
-```ts
 // example.ts
 import fs from "fs/promises";
 import { Document, VectorStoreIndex } from "llamaindex";
+import { Tiktoken } from "@dqbd/tiktoken";
 
 async function main() {
+  // Initialize Tiktoken with the WASM binary file
+  const tiktoken = new Tiktoken({ wasmURL: '/path/to/tiktoken_bg.wasm' });
+
   // Load essay from abramov.txt in Node
   const essay = await fs.readFile(
     "node_modules/llamaindex/examples/abramov.txt",
@@ -56,7 +57,6 @@ async function main() {
 }
 
 main();
-```
 
 Then you can run it using
 

apps/docs/docs/starter.md

@@ -20,8 +20,12 @@ Create the file `example.ts`. This code will load some example data, create a do
 // example.ts
 import fs from "fs/promises";
 import { Document, VectorStoreIndex } from "llamaindex";
+import { Tiktoken } from "@dqbd/tiktoken";
 
 async function main() {
+  // Initialize Tiktoken with the WASM binary file
+  const tiktoken = new Tiktoken({ wasmURL: '/path/to/tiktoken_bg.wasm' });
+
   // Load essay from abramov.txt in Node
   const essay = await fs.readFile(
     "node_modules/llamaindex/examples/abramov.txt",

packages/eslint-config-custom/package.json

@@ -8,7 +8,8 @@
     "eslint-config-next": "^13.4.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-config-turbo": "^1.9.3",
-    "eslint-plugin-react": "7.28.0"
+    "eslint-plugin-react": "7.28.0",
+    "@dqbd/tiktoken": "^1.0.7"
   },
   "publishConfig": {
     "access": "public"