fix: Add tiktoken WASM to withLlamaIndex (#1134)

Co-authored-by: Thuc Pham <51660321+thucpn@users.noreply.github.com>

.changeset/late-ties-design.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add tiktoken WASM to withLlamaIndex

apps/docs/docs/getting_started/environments.md

@@ -6,10 +6,17 @@ sidebar_position: 2
 
 We support Node.JS versions 18, 20 and 22, with experimental support for Deno, Bun and Vercel Edge functions.
 
-## NextJS App Router
+## NextJS
 
-If you're using NextJS App Router route handlers/serverless functions, you'll need to use the NodeJS mode:
+If you're using NextJS you'll need to add `withLlamaIndex` to your `next.config.js` file. This will add the necessary configuration for included 3rd-party libraries to your build:
 
 ```js
-export const runtime = "nodejs"; // default
+// next.config.js
+const withLlamaIndex = require("llamaindex/next");
+
+module.exports = withLlamaIndex({
+  // your next.js config
+});
 ```
+
+For details, check the latest [withLlamaIndex](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/llamaindex/src/next.ts) implementation.

packages/llamaindex/src/next.ts

@@ -17,6 +17,13 @@
  */
 export default function withLlamaIndex(config: any) {
   config.experimental = config.experimental ?? {};
+  // copy tiktoken WASM files to the NextJS build
+  config.experimental.outputFileTracingIncludes =
+    config.experimental.outputFileTracingIncludes ?? {};
+  config.experimental.outputFileTracingIncludes["/**/*"] = [
+    "./node_modules/tiktoken/*.wasm",
+  ];
+  // needed for transformers, see https://huggingface.co/docs/transformers.js/en/tutorials/next#step-2-install-and-configure-transformersjs
   config.experimental.serverComponentsExternalPackages =
     config.experimental.serverComponentsExternalPackages ?? [];
   config.experimental.serverComponentsExternalPackages.push(