Release 0.5.3 (#1038)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

apps/docs/CHANGELOG.md

@@ -1,5 +1,13 @@
 # docs
 
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
 ## 0.0.43
 
 ### Patch Changes

apps/docs/docs/modules/data_loaders/discord.mdx

@@ -0,0 +1,34 @@
+import CodeBlock from "@theme/CodeBlock";
+import CodeSource from "!raw-loader!../../../../../examples/readers/src/discord";
+
+# DiscordReader
+
+DiscordReader is a simple data loader that reads all messages in a given Discord channel and returns them as Document objects.
+It uses the [@discordjs/rest](https://github.com/discordjs/discord.js/tree/main/packages/rest) library to fetch the messages.
+
+## Usage
+
+First step is to create a Discord Application and generating a bot token [here](https://discord.com/developers/applications).
+In your Discord Application, go to the `OAuth2` tab and generate an invite URL by selecting `bot` and click `Read Messages/View Channels` as wells as `Read Message History`.
+This will invite the bot with the necessary permissions to read messages.
+Copy the URL in your browser and select the server you want your bot to join.
+
+<CodeBlock language="ts">{CodeSource}</CodeBlock>
+
+### Params
+
+#### DiscordReader()
+
+- `discordToken?`: The Discord bot token.
+- `makeRequest?`: Optionally provide a custom request function for edge environments, e.g. `fetch`. See discord.js for more info.
+
+#### DiscordReader.loadData
+
+- `channelIDs`: The ID(s) of discord channels as an array of strings.
+- `limit?`: Optionally limit the number of messages to read
+- `additionalInfo?`: An optional flag to include embedded messages and attachment urls in the document.
+- `oldestFirst?`: An optional flag to return the oldest messages first.
+
+## API Reference
+
+- [DiscordReader](../../api/classes/DiscordReader.md)

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.43",
+  "version": "0.0.44",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
@@ -34,10 +34,10 @@
     "@docusaurus/types": "3.4.0",
     "@tsconfig/docusaurus": "2.0.3",
     "@types/node": "^20.12.11",
-    "docusaurus-plugin-typedoc": "1.0.2",
-    "typedoc": "0.26.3",
+    "docusaurus-plugin-typedoc": "1.0.3",
+    "typedoc": "0.26.4",
     "typedoc-plugin-markdown": "4.1.2",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   },
   "browserslist": {
     "production": [

examples/package.json

@@ -9,7 +9,7 @@
     "@llamaindex/core": "^0.1.0",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^2.2.2",
-    "@zilliz/milvus2-sdk-node": "^2.4.2",
+    "@zilliz/milvus2-sdk-node": "^2.4.4",
     "chromadb": "^1.8.1",
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
@@ -21,7 +21,7 @@
   "devDependencies": {
     "@types/node": "^20.14.1",
     "tsx": "^4.15.6",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   },
   "scripts": {
     "lint": "eslint ."

examples/readers/package.json

@@ -12,7 +12,8 @@
     "start:llamaparse": "node --import tsx ./src/llamaparse.ts",
     "start:notion": "node --import tsx ./src/notion.ts",
     "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts",
-    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts"
+    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
+    "start:discord": "node --import tsx ./src/discord.ts"
   },
   "dependencies": {
     "llamaindex": "*"
@@ -20,6 +21,6 @@
   "devDependencies": {
     "@types/node": "^20.12.11",
     "tsx": "^4.15.6",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

examples/readers/src/discord.ts

@@ -0,0 +1,20 @@
+import { DiscordReader } from "llamaindex";
+
+async function main() {
+  // Create an instance of the DiscordReader. Set token here or DISCORD_TOKEN environment variable
+  const discordReader = new DiscordReader();
+
+  // Specify the channel IDs you want to read messages from as an arry of strings
+  const channelIds = ["721374320794009630", "719596376261918720"];
+
+  // Specify the number of messages to fetch per channel
+  const limit = 10;
+
+  // Load messages from the specified channel
+  const messages = await discordReader.loadData(channelIds, limit, true);
+
+  // Print out the messages
+  console.log(messages);
+}
+
+main().catch(console.error);

package.json

@@ -21,19 +21,19 @@
     "@changesets/cli": "^2.27.5",
     "@typescript-eslint/eslint-plugin": "^7.13.1",
     "eslint": "^8.57.0",
-    "eslint-config-next": "^14.2.4",
+    "eslint-config-next": "^14.2.5",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-turbo": "^2.0.5",
-    "eslint-plugin-react": "7.34.1",
+    "eslint-plugin-react": "7.34.3",
     "husky": "^9.0.11",
     "lint-staged": "^15.2.7",
     "madge": "^7.0.0",
     "prettier": "^3.3.2",
-    "prettier-plugin-organize-imports": "^3.2.4",
+    "prettier-plugin-organize-imports": "^4.0.0",
     "turbo": "^2.0.5",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   },
-  "packageManager": "pnpm@9.4.0",
+  "packageManager": "pnpm@9.5.0",
   "pnpm": {
     "overrides": {
       "trim": "1.0.1",

packages/autotool/examples/01_node/package.json

@@ -5,7 +5,7 @@
   "dependencies": {
     "@llamaindex/autotool": "workspace:*",
     "llamaindex": "workspace:*",
-    "openai": "^4.52.0"
+    "openai": "^4.52.5"
   },
   "devDependencies": {
     "tsx": "^4.15.6"

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.28
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+  - @llamaindex/autotool@2.0.0
+
 ## 0.1.27
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.27",
+  "version": "0.1.28",
   "scripts": {
     "dev": "next dev",
     "build": "next build",
@@ -14,7 +14,7 @@
     "class-variance-authority": "^0.7.0",
     "dotenv": "^16.3.1",
     "llamaindex": "workspace:*",
-    "lucide-react": "^0.378.0",
+    "lucide-react": "^0.407.0",
     "next": "14.3.0-canary.51",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
@@ -32,6 +32,6 @@
     "cross-env": "^7.0.3",
     "postcss": "^8.4.32",
     "tailwindcss": "^3.4.4",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

packages/autotool/package.json

@@ -47,11 +47,11 @@
   "dependencies": {
     "@swc/core": "^1.6.3",
     "jotai": "^2.8.3",
-    "typedoc": "^0.25.13",
+    "typedoc": "^0.26.4",
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.5.2",
+    "llamaindex": "^0.5.3",
     "openai": "^4",
     "typescript": "^4"
   },
@@ -72,11 +72,11 @@
     "@types/node": "^20.12.11",
     "bunchee": "5.3.0-beta.0",
     "llamaindex": "workspace:*",
-    "next": "14.2.3",
+    "next": "14.2.5",
     "rollup": "^4.18.0",
     "tsx": "^4.15.6",
-    "typescript": "^5.5.2",
-    "vitest": "^1.6.0",
+    "typescript": "^5.5.3",
+    "vitest": "^2.0.2",
     "webpack": "^5.92.1"
   }
 }

packages/community/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/community
 
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [b3681bf]
+  - @llamaindex/core@0.1.1
+
 ## 0.0.20
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.20",
+  "version": "0.0.21",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -46,7 +46,7 @@
     "bunchee": "5.3.0-beta.0"
   },
   "dependencies": {
-    "@aws-sdk/client-bedrock-runtime": "^3.600.0",
+    "@aws-sdk/client-bedrock-runtime": "^3.613.0",
     "@llamaindex/core": "workspace:*"
   }
 }

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/core
 
+## 0.1.1
+
+### Patch Changes
+
+- b3681bf: fix: DataCloneError when using FunctionTool
+
 ## 0.1.0
 
 ### Minor Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./llms": {

packages/core/src/global/settings/callback-manager.ts

@@ -105,9 +105,7 @@ export class CallbackManager {
     }
     queueMicrotask(() => {
       cbs.forEach((handler) =>
-        handler(
-          LlamaIndexCustomEvent.fromEvent(event, structuredClone(detail)),
-        ),
+        handler(LlamaIndexCustomEvent.fromEvent(event, { ...detail })),
       );
     });
   }

packages/core/tests/event-system.test.ts

@@ -6,6 +6,9 @@ declare module "@llamaindex/core/global" {
     test: {
       value: number;
     };
+    functionTest: {
+      fn: ({ x }: { x: number }) => string;
+    };
   }
 }
 
@@ -42,6 +45,25 @@ describe("event system", () => {
     expect(callback).toHaveBeenCalledTimes(1);
   });
 
+  test("dispatch function tool event", async () => {
+    const testFunction = ({ x }: { x: number }) => `${x * 2}`;
+    let callback;
+    Settings.callbackManager.on(
+      "functionTest",
+      (callback = vi.fn((event) => {
+        const data = event.detail;
+        expect(data.fn).toBe(testFunction);
+      })),
+    );
+
+    Settings.callbackManager.dispatchEvent("functionTest", {
+      fn: testFunction,
+    });
+    expect(callback).toHaveBeenCalledTimes(0);
+    await new Promise((resolve) => process.nextTick(resolve));
+    expect(callback).toHaveBeenCalledTimes(1);
+  });
+
   // rollup doesn't support decorators for now
   // test('wrap event caller', async () => {
   //   class A {

packages/core/tests/package.json

@@ -7,6 +7,6 @@
   },
   "devDependencies": {
     "@llamaindex/core": "workspace:*",
-    "vitest": "^1.6.0"
+    "vitest": "^2.0.2"
   }
 }

packages/env/package.json

@@ -68,11 +68,11 @@
   },
   "devDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@swc/cli": "^0.3.12",
+    "@swc/cli": "^0.4.0",
     "@swc/core": "^1.6.3",
     "concurrently": "^8.2.2",
     "pathe": "^1.1.2",
-    "vitest": "^1.6.0"
+    "vitest": "^2.0.2"
   },
   "dependencies": {
     "@types/lodash": "^4.17.5",

packages/experimental/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/experimental
 
+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
 ## 0.0.52
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.52",
+  "version": "0.0.53",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -56,7 +56,7 @@
   },
   "devDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@swc/cli": "^0.3.12",
+    "@swc/cli": "^0.4.0",
     "@swc/core": "^1.6.3",
     "@types/jsonpath": "^0.2.4",
     "concurrently": "^8.2.2",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,14 @@
 # llamaindex
 
+## 0.5.3
+
+### Patch Changes
+
+- 9bbbc67: feat: add a reader for Discord messages
+- b3681bf: fix: DataCloneError when using FunctionTool
+- Updated dependencies [b3681bf]
+  - @llamaindex/core@0.1.1
+
 ## 0.5.2
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.37
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
 ## 0.0.36
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.36",
+  "version": "0.0.37",
   "type": "module",
   "private": true,
   "scripts": {
@@ -12,13 +12,13 @@
     "cf-typegen": "wrangler types"
   },
   "devDependencies": {
-    "@cloudflare/vitest-pool-workers": "^0.4.3",
-    "@cloudflare/workers-types": "^4.20240605.0",
-    "@vitest/runner": "1.3.0",
-    "@vitest/snapshot": "1.3.0",
-    "typescript": "^5.5.2",
-    "vitest": "1.3.0",
-    "wrangler": "^3.60.1"
+    "@cloudflare/vitest-pool-workers": "^0.4.10",
+    "@cloudflare/workers-types": "^4.20240620.0",
+    "@vitest/runner": "1.5.3",
+    "@vitest/snapshot": "1.5.3",
+    "typescript": "^5.5.3",
+    "vitest": "1.5.3",
+    "wrangler": "^3.63.2"
   },
   "dependencies": {
     "llamaindex": "workspace:*"

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/next-agent-test
 
+## 0.1.37
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
 ## 0.1.36
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.36",
+  "version": "0.1.37",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -11,7 +11,7 @@
   "dependencies": {
     "ai": "^3.2.1",
     "llamaindex": "workspace:*",
-    "next": "14.2.4",
+    "next": "14.2.5",
     "react": "18.3.1",
     "react-dom": "18.3.1"
   },
@@ -20,9 +20,9 @@
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "eslint": "^8.57.0",
-    "eslint-config-next": "14.2.3",
+    "eslint-config-next": "14.2.5",
     "postcss": "^8",
     "tailwindcss": "^3.4.4",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,13 @@
 # test-edge-runtime
 
+## 0.1.36
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
 ## 0.1.35
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.35",
+  "version": "0.1.36",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -9,7 +9,7 @@
   },
   "dependencies": {
     "llamaindex": "workspace:*",
-    "next": "14.2.4",
+    "next": "14.2.5",
     "react": "^18.3.1",
     "react-dom": "^18.3.1"
   },
@@ -17,6 +17,6 @@
     "@types/node": "^20.12.11",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.18
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
 ## 0.0.17
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.17",
+  "version": "0.0.18",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -10,7 +10,7 @@
   },
   "dependencies": {
     "llamaindex": "workspace:*",
-    "next": "14.2.4",
+    "next": "14.2.5",
     "react": "18.3.1",
     "react-dom": "18.3.1"
   },
@@ -19,9 +19,9 @@
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "eslint": "^8.57.0",
-    "eslint-config-next": "14.2.3",
+    "eslint-config-next": "14.2.5",
     "postcss": "^8",
     "tailwindcss": "^3.4.4",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.37
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
 ## 0.0.36
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.36",
+  "version": "0.0.37",
   "type": "module",
   "private": true,
   "scripts": {
@@ -10,16 +10,16 @@
   },
   "dependencies": {
     "llamaindex": "workspace:*",
-    "react": "19.0.0-canary-e3ebcd54b-20240405",
-    "react-dom": "19.0.0-canary-e3ebcd54b-20240405",
-    "react-server-dom-webpack": "19.0.0-canary-e3ebcd54b-20240405",
-    "waku": "0.20.1"
+    "react": "19.0.0-beta-e7d213dfb0-20240507",
+    "react-dom": "19.0.0-beta-e7d213dfb0-20240507",
+    "react-server-dom-webpack": "19.0.0-beta-e7d213dfb0-20240507",
+    "waku": "0.20.2"
   },
   "devDependencies": {
-    "@types/react": "18.3.1",
+    "@types/react": "18.3.3",
     "@types/react-dom": "18.3.0",
     "autoprefixer": "10.4.19",
-    "tailwindcss": "3.4.3",
-    "typescript": "5.4.5"
+    "tailwindcss": "3.4.4",
+    "typescript": "5.5.3"
   }
 }

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -20,18 +20,20 @@
     "llamaindex"
   ],
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.21.1",
+    "@anthropic-ai/sdk": "0.21.1",
     "@aws-crypto/sha256-js": "^5.2.0",
     "@azure/identity": "^4.2.1",
     "@datastax/astra-db-ts": "^1.2.1",
+    "@discordjs/rest": "^2.3.0",
     "@google-cloud/vertexai": "^1.2.0",
-    "@google/generative-ai": "^0.12.0",
-    "@grpc/grpc-js": "^1.10.8",
+    "@google/generative-ai": "0.12.0",
+    "@grpc/grpc-js": "^1.10.11",
     "@huggingface/inference": "^2.7.0",
     "@llamaindex/cloud": "workspace:*",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*",
-    "@mistralai/mistralai": "^0.4.0",
+    "@mistralai/mistralai": "^0.5.0",
+    "@mixedbread-ai/sdk": "^2.2.11",
     "@pinecone-database/pinecone": "^2.2.2",
     "@qdrant/js-client-rest": "^1.9.0",
     "@types/lodash": "^4.17.4",
@@ -39,13 +41,13 @@
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.6",
     "@xenova/transformers": "^2.17.2",
-    "@zilliz/milvus2-sdk-node": "^2.4.2",
+    "@zilliz/milvus2-sdk-node": "^2.4.4",
     "ajv": "^8.16.0",
-    "assemblyai": "^4.4.5",
+    "assemblyai": "^4.6.0",
     "chromadb": "1.8.1",
-    "cohere-ai": "7.9.5",
+    "cohere-ai": "7.10.6",
+    "discord-api-types": "^0.37.92",
     "groq-sdk": "^0.5.0",
-    "@mixedbread-ai/sdk": "^2.2.11",
     "js-tiktoken": "^1.0.12",
     "lodash": "^4.17.21",
     "magic-bytes.js": "^1.10.0",
@@ -53,16 +55,16 @@
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.7.0",
     "notion-md-crawler": "^1.0.0",
-    "openai": "^4.52.0",
+    "openai": "^4.52.5",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
     "pg": "^8.12.0",
-    "pgvector": "^0.1.8",
-    "portkey-ai": "^0.1.16",
+    "pgvector": "^0.2.0",
+    "portkey-ai": "0.1.16",
     "rake-modified": "^1.0.8",
     "string-strip-html": "^13.4.8",
     "tiktoken": "^1.0.15",
-    "unpdf": "^0.10.1",
+    "unpdf": "^0.11.0",
     "wikipedia": "^2.1.2",
     "wink-nlp": "^2.3.0",
     "zod": "^3.23.8"
@@ -77,11 +79,11 @@
   },
   "devDependencies": {
     "@notionhq/client": "^2.2.15",
-    "@swc/cli": "^0.3.12",
+    "@swc/cli": "^0.4.0",
     "@swc/core": "^1.6.3",
     "concurrently": "^8.2.2",
-    "glob": "^10.4.2",
-    "typescript": "^5.5.2"
+    "glob": "^11.0.0",
+    "typescript": "^5.5.3"
   },
   "engines": {
     "node": ">=18.0.0"

packages/llamaindex/src/readers/DiscordReader.ts

@@ -0,0 +1,137 @@
+import { REST, type RESTOptions } from "@discordjs/rest";
+import { Document } from "@llamaindex/core/schema";
+import { getEnv } from "@llamaindex/env";
+import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
+
+/**
+ * Represents a reader for Discord messages using @discordjs/rest
+ * See https://github.com/discordjs/discord.js/tree/main/packages/rest
+ */
+export class DiscordReader {
+  private client: REST;
+
+  constructor(
+    discordToken?: string,
+    requestHandler?: RESTOptions["makeRequest"],
+  ) {
+    const token = discordToken ?? getEnv("DISCORD_TOKEN");
+    if (!token) {
+      throw new Error(
+        "Must specify `discordToken` or set environment variable `DISCORD_TOKEN`.",
+      );
+    }
+
+    const restOptions: Partial<RESTOptions> = { version: "10" };
+
+    // Use the provided request handler if specified
+    if (requestHandler) {
+      restOptions.makeRequest = requestHandler;
+    }
+
+    this.client = new REST(restOptions).setToken(token);
+  }
+
+  // Read all messages in a channel given a channel ID
+  private async readChannel(
+    channelId: string,
+    limit?: number,
+    additionalInfo?: boolean,
+    oldestFirst?: boolean,
+  ): Promise<Document[]> {
+    const params = new URLSearchParams();
+    if (limit) params.append("limit", limit.toString());
+    if (oldestFirst) params.append("after", "0");
+
+    try {
+      const endpoint =
+        `${Routes.channelMessages(channelId)}?${params}` as `/channels/${string}/messages`;
+      const messages = (await this.client.get(endpoint)) as APIMessage[];
+      return messages.map((msg) =>
+        this.createDocumentFromMessage(msg, additionalInfo),
+      );
+    } catch (err) {
+      console.error(err);
+      return [];
+    }
+  }
+
+  private createDocumentFromMessage(
+    msg: APIMessage,
+    additionalInfo?: boolean,
+  ): Document {
+    let content = msg.content || "";
+
+    // Include information from embedded messages
+    if (additionalInfo && msg.embeds.length > 0) {
+      content +=
+        "\n" + msg.embeds.map((embed) => this.embedToString(embed)).join("\n");
+    }
+
+    // Include URL from attachments
+    if (additionalInfo && msg.attachments.length > 0) {
+      content +=
+        "\n" +
+        msg.attachments
+          .map((attachment) => `Attachment: ${attachment.url}`)
+          .join("\n");
+    }
+
+    return new Document({
+      text: content,
+      id_: msg.id,
+      metadata: {
+        messageId: msg.id,
+        username: msg.author.username,
+        createdAt: new Date(msg.timestamp).toISOString(),
+        editedAt: msg.edited_timestamp
+          ? new Date(msg.edited_timestamp).toISOString()
+          : undefined,
+      },
+    });
+  }
+
+  // Create a string representation of an embedded message
+  private embedToString(embed: APIEmbed): string {
+    let result = "***Embedded Message***\n";
+    if (embed.title) result += `**${embed.title}**\n`;
+    if (embed.description) result += `${embed.description}\n`;
+    if (embed.url) result += `${embed.url}\n`;
+    if (embed.fields) {
+      result += embed.fields
+        .map((field) => `**${field.name}**: ${field.value}`)
+        .join("\n");
+    }
+    return result.trim();
+  }
+
+  /**
+   * Loads messages from multiple discord channels and returns an array of Document Objects.
+   *
+   * @param {string[]} channelIds - An array of channel IDs from which to load data.
+   * @param {number} [limit] - An optional limit on the number of messages to load per channel.
+   * @param {boolean} [additionalInfo] - An optional flag to include content from embedded messages and attachments urls as text.
+   * @param {boolean} [oldestFirst] - An optional flag to load oldest messages first.
+   * @return {Promise<Document[]>} A promise that resolves to an array of loaded documents.
+   */
+  async loadData(
+    channelIds: string[],
+    limit?: number,
+    additionalInfo?: boolean,
+    oldestFirst?: boolean,
+  ): Promise<Document[]> {
+    let results: Document[] = [];
+    for (const channelId of channelIds) {
+      if (typeof channelId !== "string") {
+        throw new Error(`Channel id ${channelId} must be a string.`);
+      }
+      const channelDocuments = await this.readChannel(
+        channelId,
+        limit,
+        additionalInfo,
+        oldestFirst,
+      );
+      results = results.concat(channelDocuments);
+    }
+    return results;
+  }
+}

packages/llamaindex/src/readers/index.ts

@@ -1,5 +1,6 @@
 export * from "./AssemblyAIReader.js";
 export * from "./CSVReader.js";
+export * from "./DiscordReader.js";
 export * from "./DocxReader.js";
 export * from "./HTMLReader.js";
 export * from "./ImageReader.js";

packages/llamaindex/tests/package.json

@@ -8,6 +8,6 @@
   },
   "devDependencies": {
     "llamaindex": "workspace:*",
-    "vitest": "^1.6.0"
+    "vitest": "^2.0.2"
   }
 }

packages/llamaindex/tests/readers/.snap/basic.pdf.snap

@@ -1,95 +1,36 @@
 Sample PDF
-
 This is a simple PDF file. Fun fun fun.
-
 Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Phasellus facilisis odio sed mi.
-
 Curabitur suscipit. Nullam vel nisi. Etiam semper ipsum ut lectus. Proin aliquam, erat eget
-
 pharetra commodo, eros mi condimentum quam, sed commodo justo quam ut velit.
-
-Integer
- 
-a
- 
-erat.
- 
-Cras
- 
-laoreet
- 
-ligula
- 
-cursus
- 
-enim.
- 
-Aenean
- 
-scelerisque
- 
-velit
- 
-et
- 
-tellus.
-
+Integer a erat. Cras laoreet ligula cursus enim. Aenean scelerisque velit et tellus.
 Vestibulum dictum aliquet sem. Nulla facilisi. Vestibulum accumsan ante vitae elit. Nulla
-
 erat dolor, blandit in, rutrum quis, semper pulvinar, enim. Nullam varius congue risus.
-
 Vivamus sollicitudin, metus ut interdum eleifend, nisi tellus pellentesque elit, tristique
-
 accumsan eros quam et risus. Suspendisse libero odio, mattis sit amet, aliquet eget,
-
 hendrerit vel, nulla. Sed vitae augue. Aliquam erat volutpat. Aliquam feugiat vulputate nisl.
-
 Suspendisse quis nulla pretium ante pretium mollis. Proin velit ligula, sagittis at, egestas a,
-
 pulvinar quis, nisl.
-
 Pellentesque sit amet lectus. Praesent pulvinar, nunc quis iaculis sagittis, justo quam
-
 lobortis tortor, sed vestibulum dui metus venenatis est. Nunc cursus ligula. Nulla facilisi.
-
 Phasellus ullamcorper consectetuer ante. Duis tincidunt, urna id condimentum luctus, nibh
-
 ante vulputate sapien, id sagittis massa orci ut enim. Pellentesque vestibulum convallis
-
 sem. Nulla consequat quam ut nisl. Nullam est. Curabitur tincidunt dapibus lorem. Proin
-
 velit turpis, scelerisque sit amet, iaculis nec, rhoncus ac, ipsum. Phasellus lorem arcu,
-
 feugiat eu, gravida eu, consequat molestie, ipsum. Nullam vel est ut ipsum volutpat
-
 feugiat. Aenean pellentesque.
-
 In mauris. Pellentesque dui nisi, iaculis eu, rhoncus in, venenatis ac, ante. Ut odio justo,
-
 scelerisque vel, facilisis non, commodo a, pede. Cras nec massa sit amet tortor volutpat
-
 varius. Donec lacinia, neque a luctus aliquet, pede massa imperdiet ante, at varius lorem
-
 pede sed sapien. Fusce erat nibh, aliquet in, eleifend eget, commodo eget, erat. Fusce
-
 consectetuer. Cras risus tortor, porttitor nec, tristique sed, convallis semper, eros. Fusce
-
 vulputate ipsum a mauris. Phasellus mollis. Curabitur sed urna. Aliquam nec sapien non
-
 nibh pulvinar convallis. Vivamus facilisis augue quis quam. Proin cursus aliquet metus.
-
 Suspendisse lacinia. Nulla at tellus ac turpis eleifend scelerisque. Maecenas a pede vitae
-
 enim commodo interdum. Donec odio. Sed sollicitudin dui vitae justo.
-
 Morbi elit nunc, facilisis a, mollis a, molestie at, lectus. Suspendisse eget mauris eu tellus
-
 molestie cursus. Duis ut magna at justo dignissim condimentum. Cum sociis natoque
-
 penatibus et magnis dis parturient montes, nascetur ridiculus mus. Vivamus varius. Ut sit
-
 amet diam suscipit mauris ornare aliquam. Sed varius. Duis arcu. Etiam tristique massa
-
 eget dui. Phasellus congue. Aenean est erat, tincidunt eget, venenatis quis, commodo at,
-
 quam.

packages/llamaindex/tests/readers/pdf-reader.test.ts

@@ -20,7 +20,7 @@ describe("pdf reader", () => {
     const documents = await reader.loadData(
       "../../../examples/data/brk-2022.pdf",
     );
-    expect(documents.length).toBe(140);
+    expect(documents.length).toBe(144);
   });
   test("manga.pdf", async () => {
     const documents = await reader.loadData("../../../examples/data/manga.pdf");

packages/wasm-tools/package.json

@@ -8,10 +8,10 @@
     "@types/node": "^20.12.11"
   },
   "devDependencies": {
-    "@swc/cli": "^0.3.12",
+    "@swc/cli": "^0.4.0",
     "@swc/core": "^1.6.3",
     "assemblyscript": "^0.27.27",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   },
   "engines": {
     "node": ">=18.0.0"