feat: add a reader for Discord messages (#1040)

2026-07-19 18:43:34 -04:00 · 2024-07-16 19:19:48 +02:00
parent b3681bf681
commit 9bbbc67c8e
8 changed files with 269 additions and 1 deletions
@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat: add a reader for Discord messages
@@ -0,0 +1,34 @@
+import CodeBlock from "@theme/CodeBlock";
+import CodeSource from "!raw-loader!../../../../../examples/readers/src/discord";
+
+# DiscordReader
+
+DiscordReader is a simple data loader that reads all messages in a given Discord channel and returns them as Document objects.
+It uses the [@discordjs/rest](https://github.com/discordjs/discord.js/tree/main/packages/rest) library to fetch the messages.
+
+## Usage
+
+First step is to create a Discord Application and generating a bot token [here](https://discord.com/developers/applications).
+In your Discord Application, go to the `OAuth2` tab and generate an invite URL by selecting `bot` and click `Read Messages/View Channels` as wells as `Read Message History`.
+This will invite the bot with the necessary permissions to read messages.
+Copy the URL in your browser and select the server you want your bot to join.
+
+<CodeBlock language="ts">{CodeSource}</CodeBlock>
+
+### Params
+
+#### DiscordReader()
+
+- `discordToken?`: The Discord bot token.
+- `makeRequest?`: Optionally provide a custom request function for edge environments, e.g. `fetch`. See discord.js for more info.
+
+#### DiscordReader.loadData
+
+- `channelIDs`: The ID(s) of discord channels as an array of strings.
+- `limit?`: Optionally limit the number of messages to read
+- `additionalInfo?`: An optional flag to include embedded messages and attachment urls in the document.
+- `oldestFirst?`: An optional flag to return the oldest messages first.
+
+## API Reference
+
+- [DiscordReader](../../api/classes/DiscordReader.md)
@@ -12,7 +12,8 @@
    "start:llamaparse": "node --import tsx ./src/llamaparse.ts",
    "start:notion": "node --import tsx ./src/notion.ts",
    "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts",
-    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts"
+    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
+    "start:discord": "node --import tsx ./src/discord.ts"
  },
  "dependencies": {
    "llamaindex": "*"
@@ -0,0 +1,20 @@
+import { DiscordReader } from "llamaindex";
+
+async function main() {
+  // Create an instance of the DiscordReader. Set token here or DISCORD_TOKEN environment variable
+  const discordReader = new DiscordReader();
+
+  // Specify the channel IDs you want to read messages from as an arry of strings
+  const channelIds = ["721374320794009630", "719596376261918720"];
+
+  // Specify the number of messages to fetch per channel
+  const limit = 10;
+
+  // Load messages from the specified channel
+  const messages = await discordReader.loadData(channelIds, limit, true);
+
+  // Print out the messages
+  console.log(messages);
+}
+
+main().catch(console.error);
@@ -24,6 +24,7 @@
    "@aws-crypto/sha256-js": "^5.2.0",
    "@azure/identity": "^4.2.1",
    "@datastax/astra-db-ts": "^1.2.1",
+    "@discordjs/rest": "^2.3.0",
    "@google-cloud/vertexai": "^1.2.0",
    "@google/generative-ai": "0.12.0",
    "@grpc/grpc-js": "^1.10.11",
@@ -45,6 +46,7 @@
    "assemblyai": "^4.6.0",
    "chromadb": "1.8.1",
    "cohere-ai": "7.10.6",
+    "discord-api-types": "^0.37.92",
    "groq-sdk": "^0.5.0",
    "js-tiktoken": "^1.0.12",
    "lodash": "^4.17.21",
@@ -0,0 +1,137 @@
+import { REST, type RESTOptions } from "@discordjs/rest";
+import { Document } from "@llamaindex/core/schema";
+import { getEnv } from "@llamaindex/env";
+import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
+
+/**
+ * Represents a reader for Discord messages using @discordjs/rest
+ * See https://github.com/discordjs/discord.js/tree/main/packages/rest
+ */
+export class DiscordReader {
+  private client: REST;
+
+  constructor(
+    discordToken?: string,
+    requestHandler?: RESTOptions["makeRequest"],
+  ) {
+    const token = discordToken ?? getEnv("DISCORD_TOKEN");
+    if (!token) {
+      throw new Error(
+        "Must specify `discordToken` or set environment variable `DISCORD_TOKEN`.",
+      );
+    }
+
+    const restOptions: Partial<RESTOptions> = { version: "10" };
+
+    // Use the provided request handler if specified
+    if (requestHandler) {
+      restOptions.makeRequest = requestHandler;
+    }
+
+    this.client = new REST(restOptions).setToken(token);
+  }
+
+  // Read all messages in a channel given a channel ID
+  private async readChannel(
+    channelId: string,
+    limit?: number,
+    additionalInfo?: boolean,
+    oldestFirst?: boolean,
+  ): Promise<Document[]> {
+    const params = new URLSearchParams();
+    if (limit) params.append("limit", limit.toString());
+    if (oldestFirst) params.append("after", "0");
+
+    try {
+      const endpoint =
+        `${Routes.channelMessages(channelId)}?${params}` as `/channels/${string}/messages`;
+      const messages = (await this.client.get(endpoint)) as APIMessage[];
+      return messages.map((msg) =>
+        this.createDocumentFromMessage(msg, additionalInfo),
+      );
+    } catch (err) {
+      console.error(err);
+      return [];
+    }
+  }
+
+  private createDocumentFromMessage(
+    msg: APIMessage,
+    additionalInfo?: boolean,
+  ): Document {
+    let content = msg.content || "";
+
+    // Include information from embedded messages
+    if (additionalInfo && msg.embeds.length > 0) {
+      content +=
+        "\n" + msg.embeds.map((embed) => this.embedToString(embed)).join("\n");
+    }
+
+    // Include URL from attachments
+    if (additionalInfo && msg.attachments.length > 0) {
+      content +=
+        "\n" +
+        msg.attachments
+          .map((attachment) => `Attachment: ${attachment.url}`)
+          .join("\n");
+    }
+
+    return new Document({
+      text: content,
+      id_: msg.id,
+      metadata: {
+        messageId: msg.id,
+        username: msg.author.username,
+        createdAt: new Date(msg.timestamp).toISOString(),
+        editedAt: msg.edited_timestamp
+          ? new Date(msg.edited_timestamp).toISOString()
+          : undefined,
+      },
+    });
+  }
+
+  // Create a string representation of an embedded message
+  private embedToString(embed: APIEmbed): string {
+    let result = "***Embedded Message***\n";
+    if (embed.title) result += `**${embed.title}**\n`;
+    if (embed.description) result += `${embed.description}\n`;
+    if (embed.url) result += `${embed.url}\n`;
+    if (embed.fields) {
+      result += embed.fields
+        .map((field) => `**${field.name}**: ${field.value}`)
+        .join("\n");
+    }
+    return result.trim();
+  }
+
+  /**
+   * Loads messages from multiple discord channels and returns an array of Document Objects.
+   *
+   * @param {string[]} channelIds - An array of channel IDs from which to load data.
+   * @param {number} [limit] - An optional limit on the number of messages to load per channel.
+   * @param {boolean} [additionalInfo] - An optional flag to include content from embedded messages and attachments urls as text.
+   * @param {boolean} [oldestFirst] - An optional flag to load oldest messages first.
+   * @return {Promise<Document[]>} A promise that resolves to an array of loaded documents.
+   */
+  async loadData(
+    channelIds: string[],
+    limit?: number,
+    additionalInfo?: boolean,
+    oldestFirst?: boolean,
+  ): Promise<Document[]> {
+    let results: Document[] = [];
+    for (const channelId of channelIds) {
+      if (typeof channelId !== "string") {
+        throw new Error(`Channel id ${channelId} must be a string.`);
+      }
+      const channelDocuments = await this.readChannel(
+        channelId,
+        limit,
+        additionalInfo,
+        oldestFirst,
+      );
+      results = results.concat(channelDocuments);
+    }
+    return results;
+  }
+}
@@ -1,5 +1,6 @@
 export * from "./AssemblyAIReader.js";
 export * from "./CSVReader.js";
+export * from "./DiscordReader.js";
 export * from "./DocxReader.js";
 export * from "./HTMLReader.js";
 export * from "./ImageReader.js";
@@ -473,6 +473,9 @@ importers:
      '@datastax/astra-db-ts':
        specifier: ^1.2.1
        version: 1.3.0
+      '@discordjs/rest':
+        specifier: ^2.3.0
+        version: 2.3.0
      '@google-cloud/vertexai':
        specifier: ^1.2.0
        version: 1.3.0(encoding@0.1.13)
@@ -536,6 +539,9 @@ importers:
      cohere-ai:
        specifier: 7.10.6
        version: 7.10.6(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))(encoding@0.1.13)
+      discord-api-types:
+        specifier: ^0.37.92
+        version: 0.37.92
      groq-sdk:
        specifier: ^0.5.0
        version: 0.5.0(encoding@0.1.13)
@@ -1977,6 +1983,18 @@ packages:
    resolution: {integrity: sha512-KrkT6qO5NxqNfy68sBl6CTSoJ4SNDIS5iQArkibhlbGU4LaDukZ3q2HIkh8aUKDio6o4itU4xDR7t82Y2eP1Bg==}
    engines: {node: '>=14'}

+  '@discordjs/collection@2.1.0':
+    resolution: {integrity: sha512-mLcTACtXUuVgutoznkh6hS3UFqYirDYAg5Dc1m8xn6OvPjetnUlf/xjtqnnc47OwWdaoCQnHmHh9KofhD6uRqw==}
+    engines: {node: '>=18'}
+
+  '@discordjs/rest@2.3.0':
+    resolution: {integrity: sha512-C1kAJK8aSYRv3ZwMG8cvrrW4GN0g5eMdP8AuN8ODH5DyOCbHgJspze1my3xHOAgwLJdKUbWNVyAeJ9cEdduqIg==}
+    engines: {node: '>=16.11.0'}
+
+  '@discordjs/util@1.1.0':
+    resolution: {integrity: sha512-IndcI5hzlNZ7GS96RV3Xw1R2kaDuXEp7tRIy/KlhidpN/BQ1qh1NZt3377dMLTa44xDUNKT7hnXkA/oUAzD/lg==}
+    engines: {node: '>=16.11.0'}
+
  '@discoveryjs/json-ext@0.5.7':
    resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==}
    engines: {node: '>=10.0.0'}
@@ -3230,6 +3248,14 @@ packages:
  '@rushstack/eslint-patch@1.10.3':
    resolution: {integrity: sha512-qC/xYId4NMebE6w/V33Fh9gWxLgURiNYgVNObbJl2LZv0GUUItCcCqC5axQSwRaAgaxl2mELq1rMzlswaQ0Zxg==}

+  '@sapphire/async-queue@1.5.2':
+    resolution: {integrity: sha512-7X7FFAA4DngXUl95+hYbUF19bp1LGiffjJtu7ygrZrbdCSsdDDBaSjB7Akw0ZbOu6k0xpXyljnJ6/RZUvLfRdg==}
+    engines: {node: '>=v14.0.0', npm: '>=7.0.0'}
+
+  '@sapphire/snowflake@3.5.3':
+    resolution: {integrity: sha512-jjmJywLAFoWeBi1W7994zZyiNWPIiqRRNAmSERxyg93xRGzNYvGjlZ0gR6x0F4gPRi2+0O6S71kOZYyr3cxaIQ==}
+    engines: {node: '>=v14.0.0', npm: '>=7.0.0'}
+
  '@sevinf/maybe@0.5.0':
    resolution: {integrity: sha512-ARhyoYDnY1LES3vYI0fiG6e9esWfTNcXcO6+MPJJXcnyMV3bim4lnFt45VXouV7y82F4x3YH8nOQ6VztuvUiWg==}

@@ -4121,6 +4147,10 @@ packages:
  '@vitest/utils@2.0.2':
    resolution: {integrity: sha512-pxCY1v7kmOCWYWjzc0zfjGTA3Wmn8PKnlPvSrsA643P1NHl1fOyXj2Q9SaNlrlFE+ivCsxM80Ov3AR82RmHCWQ==}

+  '@vladfrangu/async_event_emitter@2.4.3':
+    resolution: {integrity: sha512-wn15EJHUk2RDtCw6wVJndUhFbfJYwyNhSD9s+yiQi5c2MmXDy3KLWvZ5LrcOzt3CHdscoAnaJrxIRKBI1QQgGw==}
+    engines: {node: '>=v14.0.0', npm: '>=7.0.0'}
+
  '@vue/compiler-core@3.4.31':
    resolution: {integrity: sha512-skOiodXWTV3DxfDhB4rOf3OGalpITLlgCeOwb+Y9GJpfQ8ErigdBUHomBzvG78JoVE8MJoQsb+qhZiHfKeNeEg==}

@@ -5460,6 +5490,12 @@ packages:
    resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==}
    engines: {node: '>=8'}

+  discord-api-types@0.37.83:
+    resolution: {integrity: sha512-urGGYeWtWNYMKnYlZnOnDHm8fVRffQs3U0SpE8RHeiuLKb/u92APS8HoQnPTFbnXmY1vVnXjXO4dOxcAn3J+DA==}
+
+  discord-api-types@0.37.92:
+    resolution: {integrity: sha512-7xnedbQRLRef/O+4jKPyIFwl6YqoyihOG3OSneiRmVJMBk30ph2YuZGcHjeX1Kk/a3yQWeyCKe4RZJB3iECcxg==}
+
  dlv@1.1.3:
    resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==}

@@ -10174,6 +10210,10 @@ packages:
    resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==}
    engines: {node: '>=14.0'}

+  undici@6.13.0:
+    resolution: {integrity: sha512-Q2rtqmZWrbP8nePMq7mOJIN98M0fYvSgV89vwl/BQRT4mDOeY2GXZngfGpcBBhtky3woM7G24wZV3Q304Bv6cw==}
+    engines: {node: '>=18.0'}
+
  unenv-nightly@1.10.0-1717606461.a117952:
    resolution: {integrity: sha512-u3TfBX02WzbHTpaEfWEKwDijDSFAHcgXkayUZ+MVDrjhLFvgAJzFGTSTmwlEhwWi2exyRQey23ah9wELMM6etg==}

@@ -12661,6 +12701,22 @@ snapshots:
      gonzales-pe: 4.3.0
      node-source-walk: 6.0.2

+  '@discordjs/collection@2.1.0': {}
+
+  '@discordjs/rest@2.3.0':
+    dependencies:
+      '@discordjs/collection': 2.1.0
+      '@discordjs/util': 1.1.0
+      '@sapphire/async-queue': 1.5.2
+      '@sapphire/snowflake': 3.5.3
+      '@vladfrangu/async_event_emitter': 2.4.3
+      discord-api-types: 0.37.83
+      magic-bytes.js: 1.10.0
+      tslib: 2.6.3
+      undici: 6.13.0
+
+  '@discordjs/util@1.1.0': {}
+
  '@discoveryjs/json-ext@0.5.7': {}

  '@docsearch/css@3.6.0': {}
@@ -14173,6 +14229,10 @@ snapshots:

  '@rushstack/eslint-patch@1.10.3': {}

+  '@sapphire/async-queue@1.5.2': {}
+
+  '@sapphire/snowflake@3.5.3': {}
+
  '@sevinf/maybe@0.5.0': {}

  '@shikijs/core@1.10.3':
@@ -15274,6 +15334,8 @@ snapshots:
      loupe: 3.1.1
      tinyrainbow: 1.2.0

+  '@vladfrangu/async_event_emitter@2.4.3': {}
+
  '@vue/compiler-core@3.4.31':
    dependencies:
      '@babel/parser': 7.24.7
@@ -16840,6 +16902,10 @@ snapshots:
    dependencies:
      path-type: 4.0.0

+  discord-api-types@0.37.83: {}
+
+  discord-api-types@0.37.92: {}
+
  dlv@1.1.3: {}

  dns-packet@5.6.1:
@@ -22613,6 +22679,8 @@ snapshots:
    dependencies:
      '@fastify/busboy': 2.1.1

+  undici@6.13.0: {}
+
  unenv-nightly@1.10.0-1717606461.a117952:
    dependencies:
      consola: 3.2.3