rolled back notion package and changeset

Create OutputParser.test.ts [Needs Merge]

.changeset/little-melons-shake.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-OpenAI 4.3.1 and Anthropic 0.6.2

.changeset/quiet-months-flow.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Typesafe metadata (thanks @TomPenguin)

.changeset/serious-jokes-crash.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Update READMEs (thanks @andfk)

.changeset/seven-adults-doubt.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+MongoReader (thanks @kkang2097)

.changeset/tasty-melons-jog.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Bug: missing exports from storage (thanks @aashutoshrathi)

.changeset/two-dogs-flow.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Make OutputParser less strict and add tests (Thanks @kkang2097)

apps/docs/docs/end_to_end.md

@@ -14,7 +14,7 @@ Read a file and chat about it with the LLM.
 
 Create a vector index and query it. The vector index will use embeddings to fetch the top k most relevant nodes. By default, the top k is 2.
 
-## [Summary Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/summarIndex.ts)
+## [Summary Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/summaryIndex.ts)
 
 Create a list index and query it. This example also use the `LLMRetriever`, which will use the LLM to select the best nodes to use when generating answer.
 

apps/simple/CHANGELOG.md

@@ -1,5 +1,30 @@
 # simple
 
+## 0.0.25
+
+### Patch Changes
+
+- Updated dependencies [4a5591b]
+- Updated dependencies [4a5591b]
+- Updated dependencies [4a5591b]
+  - llamaindex@0.0.27
+
+## 0.0.24
+
+### Patch Changes
+
+- Updated dependencies [5bb55bc]
+  - llamaindex@0.0.26
+
+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [e21eca2]
+- Updated dependencies [40a8f07]
+- Updated dependencies [40a8f07]
+  - llamaindex@0.0.25
+
 ## 0.0.22
 
 ### Patch Changes

apps/simple/keywordIndex.ts

@@ -0,0 +1,32 @@
+import {
+  Document,
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "llamaindex";
+import essay from "./essay";
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+  const index = await KeywordTableIndex.fromDocuments([document]);
+
+  const allModes: KeywordTableRetrieverMode[] = [
+    KeywordTableRetrieverMode.DEFAULT,
+    KeywordTableRetrieverMode.SIMPLE,
+    KeywordTableRetrieverMode.RAKE,
+  ];
+  allModes.forEach(async (mode) => {
+    const queryEngine = index.asQueryEngine({
+      retriever: index.asRetriever({
+        mode,
+      }),
+    });
+    const response = await queryEngine.query(
+      "What did the author do growing up?",
+    );
+    console.log(response.toString());
+  });
+}
+
+main().catch((e: Error) => {
+  console.error(e, e.stack);
+});

apps/simple/mongo.ts

@@ -0,0 +1,68 @@
+import { MongoClient } from "mongodb";
+import { VectorStoreIndex } from "../../packages/core/src/indices";
+import { Document } from "../../packages/core/src/Node";
+import { SimpleMongoReader } from "../../packages/core/src/readers/SimpleMongoReader";
+
+import { stdin as input, stdout as output } from "node:process";
+import readline from "node:readline/promises";
+
+async function main() {
+  //Dummy test code
+  const query: object = { _id: "waldo" };
+  const options: object = {};
+  const projections: object = { embedding: 0 };
+  const limit: number = Infinity;
+  const uri: string = process.env.MONGODB_URI ?? "fake_uri";
+  const client: MongoClient = new MongoClient(uri);
+
+  //Where the real code starts
+  const MR = new SimpleMongoReader(client);
+  const documents: Document[] = await MR.loadData(
+    "data",
+    "posts",
+    1,
+    {},
+    options,
+    projections,
+  );
+
+  //
+  //If you need to look at low-level details of
+  // a queryEngine (for example, needing to check each individual node)
+  //
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  // var storageContext = await storageContextFromDefaults({});
+  // var serviceContext = serviceContextFromDefaults({});
+  // const docStore = storageContext.docStore;
+
+  // for (const doc of documents) {
+  //   docStore.setDocumentHash(doc.id_, doc.hash);
+  // }
+  // const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
+  // console.log(nodes);
+
+  //
+  //Making Vector Store from documents
+  //
+
+  const index = await VectorStoreIndex.fromDocuments(documents);
+  // Create query engine
+  const queryEngine = index.asQueryEngine();
+
+  const rl = readline.createInterface({ input, output });
+  while (true) {
+    const query = await rl.question("Query: ");
+
+    if (!query) {
+      break;
+    }
+
+    const response = await queryEngine.query(query);
+
+    // Output response
+    console.log(response.toString());
+  }
+}
+
+main();

apps/simple/notion.ts

@@ -0,0 +1,89 @@
+import { Client } from "@notionhq/client";
+import { program } from "commander";
+import { NotionReader, VectorStoreIndex } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+
+program
+  .argument("[page]", "Notion page id (must be provided)")
+  .action(async (page, _options, command) => {
+    // Initializing a client
+
+    if (!process.env.NOTION_TOKEN) {
+      console.log(
+        "No NOTION_TOKEN found in environment variables. You will need to register an integration https://www.notion.com/my-integrations and put it in your NOTION_TOKEN environment variable.",
+      );
+      return;
+    }
+
+    const notion = new Client({
+      auth: process.env.NOTION_TOKEN,
+    });
+
+    if (!page) {
+      const response = await notion.search({
+        filter: {
+          value: "page",
+          property: "object",
+        },
+        sort: {
+          direction: "descending",
+          timestamp: "last_edited_time",
+        },
+      });
+
+      const { results } = response;
+
+      if (results.length === 0) {
+        console.log(
+          "No pages found. You will need to share it with your integration. (tap the three dots on the top right, find Add connections, and add your integration)",
+        );
+        return;
+      } else {
+        const pages = results
+          .map((result) => {
+            if (!("url" in result)) {
+              return null;
+            }
+
+            return {
+              id: result.id,
+              url: result.url,
+            };
+          })
+          .filter((page) => page !== null);
+        console.log("Found pages:");
+        console.table(pages);
+        console.log(`To run, run ts-node ${command.name()} [page id]`);
+        return;
+      }
+    }
+
+    const reader = new NotionReader({ client: notion });
+    const documents = await reader.loadData(page);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Query: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      // Output response
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

apps/simple/package.json

@@ -1,12 +1,14 @@
 {
-  "version": "0.0.22",
+  "version": "0.0.25",
   "private": true,
   "name": "simple",
   "dependencies": {
+    "@notionhq/client": "^2.2.12",
+    "commander": "^11.0.0",
     "llamaindex": "workspace:*"
   },
   "devDependencies": {
-    "@types/node": "^18.17.6"
+    "@types/node": "^18.17.12"
   },
   "scripts": {
     "lint": "eslint ."

examples/keywordIndex.ts

@@ -0,0 +1,32 @@
+import {
+  Document,
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "llamaindex";
+import essay from "./essay";
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+  const index = await KeywordTableIndex.fromDocuments([document]);
+
+  const allModes: KeywordTableRetrieverMode[] = [
+    KeywordTableRetrieverMode.DEFAULT,
+    KeywordTableRetrieverMode.SIMPLE,
+    KeywordTableRetrieverMode.RAKE,
+  ];
+  allModes.forEach(async (mode) => {
+    const queryEngine = index.asQueryEngine({
+      retriever: index.asRetriever({
+        mode,
+      }),
+    });
+    const response = await queryEngine.query(
+      "What did the author do growing up?",
+    );
+    console.log(response.toString());
+  });
+}
+
+main().catch((e: Error) => {
+  console.error(e, e.stack);
+});

examples/notion.ts

@@ -0,0 +1,89 @@
+import { Client } from "@notionhq/client";
+import { program } from "commander";
+import { NotionReader, VectorStoreIndex } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+
+program
+  .argument("[page]", "Notion page id (must be provided)")
+  .action(async (page, _options, command) => {
+    // Initializing a client
+
+    if (!process.env.NOTION_TOKEN) {
+      console.log(
+        "No NOTION_TOKEN found in environment variables. You will need to register an integration https://www.notion.com/my-integrations and put it in your NOTION_TOKEN environment variable.",
+      );
+      return;
+    }
+
+    const notion = new Client({
+      auth: process.env.NOTION_TOKEN,
+    });
+
+    if (!page) {
+      const response = await notion.search({
+        filter: {
+          value: "page",
+          property: "object",
+        },
+        sort: {
+          direction: "descending",
+          timestamp: "last_edited_time",
+        },
+      });
+
+      const { results } = response;
+
+      if (results.length === 0) {
+        console.log(
+          "No pages found. You will need to share it with your integration. (tap the three dots on the top right, find Add connections, and add your integration)",
+        );
+        return;
+      } else {
+        const pages = results
+          .map((result) => {
+            if (!("url" in result)) {
+              return null;
+            }
+
+            return {
+              id: result.id,
+              url: result.url,
+            };
+          })
+          .filter((page) => page !== null);
+        console.log("Found pages:");
+        console.table(pages);
+        console.log(`To run, run ts-node ${command.name()} [page id]`);
+        return;
+      }
+    }
+
+    const reader = new NotionReader({ client: notion });
+    const documents = await reader.loadData(page);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Query: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      // Output response
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

packages/core/CHANGELOG.md

@@ -1,5 +1,27 @@
 # llamaindex
 
+## 0.0.27
+
+### Patch Changes
+
+- 4a5591b: Chat History summarization (thanks @marcusschiesser)
+- 4a5591b: Notion database support (thanks @TomPenguin)
+- 4a5591b: KeywordIndex (thanks @swk777)
+
+## 0.0.26
+
+### Patch Changes
+
+- 5bb55bc: Add notion loader (thank you @TomPenguin!)
+
+## 0.0.25
+
+### Patch Changes
+
+- e21eca2: OpenAI 4.3.1 and Anthropic 0.6.2
+- 40a8f07: Update READMEs (thanks @andfk)
+- 40a8f07: Bug: missing exports from storage (thanks @aashutoshrathi)
+
 ## 0.0.24
 
 ### Patch Changes

packages/core/package.json

@@ -1,23 +1,28 @@
 {
   "name": "llamaindex",
-  "version": "0.0.24",
+  "version": "0.0.27",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.6.2",
+    "@notionhq/client": "^2.2.13",
     "lodash": "^4.17.21",
-    "openai": "^4.3.1",
+    "md-utils-ts": "^2.0.0",
+    "mongodb": "^6.0.0",
+    "notion-md-crawler": "^0.0.2",
+    "openai": "^4.7.0",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
-    "replicate": "^0.16.1",
+    "rake-modified": "^1.0.8",
+    "replicate": "^0.18.0",
     "tiktoken-node": "^0.0.6",
-    "uuid": "^9.0.0",
+    "uuid": "^9.0.1",
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
-    "@types/lodash": "^4.14.197",
-    "@types/node": "^18.17.12",
+    "@types/lodash": "^4.14.198",
+    "@types/node": "^18.17.15",
     "@types/papaparse": "^5.3.8",
     "@types/pdf-parse": "^1.1.1",
-    "@types/uuid": "^9.0.2",
+    "@types/uuid": "^9.0.4",
     "node-stdlib-browser": "^1.2.0",
     "tsup": "^7.2.0"
   },

packages/core/src/ChatEngine.ts

@@ -1,4 +1,7 @@
 import { v4 as uuidv4 } from "uuid";
+import { Event } from "./callbacks/CallbackManager";
+import { ChatHistory, SimpleChatHistory } from "./ChatHistory";
+import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 import { TextNode } from "./Node";
 import {
   CondenseQuestionPrompt,
@@ -11,8 +14,6 @@ import { BaseQueryEngine } from "./QueryEngine";
 import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { Event } from "./callbacks/CallbackManager";
-import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 
 /**
  * A ChatEngine is used to handle back and forth chats between the application and the LLM.
@@ -188,3 +189,28 @@ export class ContextChatEngine implements ChatEngine {
     this.chatHistory = [];
   }
 }
+
+/**
+ * HistoryChatEngine is a ChatEngine that uses a ChatHistory to keep track of the chat history. This is an example with the same behavior as SimpleChatEngine
+ * TODO: generally use the ChatHistory instead of ChatMessage[] - breaking change
+ */
+export class HistoryChatEngine implements ChatEngine {
+  chatHistory: ChatHistory;
+  llm: LLM;
+
+  constructor(init?: Partial<HistoryChatEngine>) {
+    this.chatHistory = init?.chatHistory ?? new SimpleChatHistory();
+    this.llm = init?.llm ?? new OpenAI();
+  }
+
+  async chat(message: string): Promise<Response> {
+    this.chatHistory.addMessage({ content: message, role: "user" });
+    const response = await this.llm.chat(this.chatHistory.messages);
+    this.chatHistory.addMessage(response.message);
+    return new Response(response.message.content);
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+}

packages/core/src/ChatHistory.ts

@@ -0,0 +1,73 @@
+import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
+import {
+  defaultSummaryPrompt,
+  messagesToHistoryStr,
+  SummaryPrompt,
+} from "./Prompt";
+
+/**
+ * A ChatHistory is used to keep the state of back and forth chat messages
+ */
+export interface ChatHistory {
+  messages: ChatMessage[];
+  /**
+   * Adds a message to the chat history.
+   * @param message
+   */
+  addMessage(message: ChatMessage): Promise<void>;
+
+  /**
+   * Resets the chat history so that it's empty.
+   */
+  reset(): void;
+}
+
+export class SimpleChatHistory implements ChatHistory {
+  messages: ChatMessage[];
+
+  constructor(init?: Partial<SimpleChatHistory>) {
+    this.messages = init?.messages ?? [];
+  }
+
+  async addMessage(message: ChatMessage) {
+    this.messages.push(message);
+  }
+
+  reset() {
+    this.messages = [];
+  }
+}
+
+export class SummaryChatHistory implements ChatHistory {
+  messages: ChatMessage[];
+  summaryPrompt: SummaryPrompt;
+  llm: LLM;
+
+  constructor(init?: Partial<SummaryChatHistory>) {
+    this.messages = init?.messages ?? [];
+    this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
+    this.llm = init?.llm ?? new OpenAI();
+  }
+
+  private async summarize() {
+    const chatHistoryStr = messagesToHistoryStr(this.messages);
+
+    const response = await this.llm.complete(
+      this.summaryPrompt({ context: chatHistoryStr }),
+    );
+
+    this.messages = [{ content: response.message.content, role: "system" }];
+  }
+
+  async addMessage(message: ChatMessage) {
+    // TODO: check if summarization is necessary
+    // TBD what are good conditions, e.g. depending on the context length of the LLM?
+    // for now we just have a dummy implementation at always summarizes the messages
+    await this.summarize();
+    this.messages.push(message);
+  }
+
+  reset() {
+    this.messages = [];
+  }
+}

packages/core/src/Node.ts

@@ -23,19 +23,23 @@ export enum MetadataMode {
   NONE = "NONE",
 }
 
-export interface RelatedNodeInfo {
+export type Metadata = Record<string, any>;
+
+export interface RelatedNodeInfo<T extends Metadata = Metadata> {
   nodeId: string;
   nodeType?: ObjectType;
-  metadata: Record<string, any>;
+  metadata: T;
   hash?: string;
 }
 
-export type RelatedNodeType = RelatedNodeInfo | RelatedNodeInfo[];
+export type RelatedNodeType<T extends Metadata = Metadata> =
+  | RelatedNodeInfo<T>
+  | RelatedNodeInfo<T>[];
 
 /**
  * Generic abstract class for retrievable nodes
  */
-export abstract class BaseNode {
+export abstract class BaseNode<T extends Metadata = Metadata> {
   /**
    * The unique ID of the Node/Document. The trailing underscore is here
    * to avoid collisions with the id keyword in Python.
@@ -46,13 +50,13 @@ export abstract class BaseNode {
   embedding?: number[];
 
   // Metadata fields
-  metadata: Record<string, any> = {};
+  metadata: T = {} as T;
   excludedEmbedMetadataKeys: string[] = [];
   excludedLlmMetadataKeys: string[] = [];
-  relationships: Partial<Record<NodeRelationship, RelatedNodeType>> = {};
+  relationships: Partial<Record<NodeRelationship, RelatedNodeType<T>>> = {};
   hash: string = "";
 
-  constructor(init?: Partial<BaseNode>) {
+  constructor(init?: Partial<BaseNode<T>>) {
     Object.assign(this, init);
   }
 
@@ -62,7 +66,7 @@ export abstract class BaseNode {
   abstract getMetadataStr(metadataMode: MetadataMode): string;
   abstract setContent(value: any): void;
 
-  get sourceNode(): RelatedNodeInfo | undefined {
+  get sourceNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.SOURCE];
 
     if (Array.isArray(relationship)) {
@@ -72,7 +76,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get prevNode(): RelatedNodeInfo | undefined {
+  get prevNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.PREVIOUS];
 
     if (Array.isArray(relationship)) {
@@ -84,7 +88,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get nextNode(): RelatedNodeInfo | undefined {
+  get nextNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.NEXT];
 
     if (Array.isArray(relationship)) {
@@ -94,7 +98,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get parentNode(): RelatedNodeInfo | undefined {
+  get parentNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.PARENT];
 
     if (Array.isArray(relationship)) {
@@ -104,7 +108,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get childNodes(): RelatedNodeInfo[] | undefined {
+  get childNodes(): RelatedNodeInfo<T>[] | undefined {
     const relationship = this.relationships[NodeRelationship.CHILD];
 
     if (!Array.isArray(relationship)) {
@@ -126,7 +130,7 @@ export abstract class BaseNode {
     return this.embedding;
   }
 
-  asRelatedNodeInfo(): RelatedNodeInfo {
+  asRelatedNodeInfo(): RelatedNodeInfo<T> {
     return {
       nodeId: this.id_,
       metadata: this.metadata,
@@ -146,7 +150,7 @@ export abstract class BaseNode {
 /**
  * TextNode is the default node type for text. Most common node type in LlamaIndex.TS
  */
-export class TextNode extends BaseNode {
+export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
   text: string = "";
   startCharIdx?: number;
   endCharIdx?: number;
@@ -154,7 +158,7 @@ export class TextNode extends BaseNode {
   // metadataTemplate: NOTE write your own formatter if needed
   metadataSeparator: string = "\n";
 
-  constructor(init?: Partial<TextNode>) {
+  constructor(init?: Partial<TextNode<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -233,10 +237,10 @@ export class TextNode extends BaseNode {
 //   }
 // }
 
-export class IndexNode extends TextNode {
+export class IndexNode<T extends Metadata = Metadata> extends TextNode<T> {
   indexId: string = "";
 
-  constructor(init?: Partial<IndexNode>) {
+  constructor(init?: Partial<IndexNode<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -253,8 +257,8 @@ export class IndexNode extends TextNode {
 /**
  * A document is just a special text node with a docId.
  */
-export class Document extends TextNode {
-  constructor(init?: Partial<Document>) {
+export class Document<T extends Metadata = Metadata> extends TextNode<T> {
+  constructor(init?: Partial<Document<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -292,7 +296,7 @@ export function jsonToNode(json: any) {
 /**
  * A node with a similarity score
  */
-export interface NodeWithScore {
-  node: BaseNode;
-  score: number;
+export interface NodeWithScore<T extends Metadata = Metadata> {
+  node: BaseNode<T>;
+  score?: number;
 }

packages/core/src/OutputParser.ts

@@ -53,30 +53,31 @@ class OutputParserError extends Error {
  * @param text A markdown block with JSON
  * @returns parsed JSON object
  */
-function parseJsonMarkdown(text: string) {
+export function parseJsonMarkdown(text: string) {
   text = text.trim();
 
-  const beginDelimiter = "```json";
-  const endDelimiter = "```";
+  const left_square = text.indexOf("[");
+  const left_brace = text.indexOf("{");
 
-  const beginIndex = text.indexOf(beginDelimiter);
-  const endIndex = text.indexOf(
-    endDelimiter,
-    beginIndex + beginDelimiter.length,
-  );
-  if (beginIndex === -1 || endIndex === -1) {
-    throw new OutputParserError("Not a json markdown", { output: text });
+  var left: number;
+  var right: number;
+  if (left_square < left_brace && left_square != -1) {
+    left = left_square;
+    right = text.lastIndexOf("]");
+  } else {
+    left = left_brace;
+    right = text.lastIndexOf("}");
   }
-
-  const jsonText = text.substring(beginIndex + beginDelimiter.length, endIndex);
-
+  const jsonText = text.substring(left, right + 1);
   try {
+    //Single JSON object case
+    if (left_square === -1) {
+      return [JSON.parse(jsonText)];
+    }
+    //Multiple JSON object case.
     return JSON.parse(jsonText);
   } catch (e) {
-    throw new OutputParserError("Not a valid json", {
-      cause: e as Error,
-      output: text,
-    });
+    throw new OutputParserError("Not a json markdown", { output: text });
   }
 }
 

packages/core/src/Prompt.ts

@@ -356,3 +356,34 @@ ${context}
 };
 
 export type ContextSystemPrompt = typeof defaultContextSystemPrompt;
+
+export const defaultKeywordExtractPrompt = ({
+  context = "",
+  maxKeywords = 10,
+}) => {
+  return `
+Some text is provided below. Given the text, extract up to ${maxKeywords} keywords from the text. Avoid stopwords.
+---------------------
+${context}
+---------------------
+Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'
+`;
+};
+
+export type KeywordExtractPrompt = typeof defaultKeywordExtractPrompt;
+
+export const defaultQueryKeywordExtractPrompt = ({
+  question = "",
+  maxKeywords = 10,
+}) => {
+  return `(
+  "A question is provided below. Given the question, extract up to ${maxKeywords} "
+  "keywords from the text. Focus on extracting the keywords that we can use "
+  "to best lookup answers to the question. Avoid stopwords."
+  "---------------------"
+  "${question}"
+  "---------------------"
+  "Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'"
+)`;
+};
+export type QueryKeywordExtractPrompt = typeof defaultQueryKeywordExtractPrompt;

packages/core/src/index.ts

@@ -22,6 +22,7 @@ export * from "./callbacks/CallbackManager";
 
 export * from "./readers/CSVReader";
 export * from "./readers/MarkdownReader";
+export * from "./readers/NotionReader";
 export * from "./readers/PDFReader";
 export * from "./readers/SimpleDirectoryReader";
 export * from "./readers/base";

packages/core/src/indices/BaseIndex.ts

@@ -39,6 +39,7 @@ export abstract class IndexStruct {
 export enum IndexStructType {
   SIMPLE_DICT = "simple_dict",
   LIST = "list",
+  KEYWORD_TABLE = "keyword_table",
 }
 
 export class IndexDict extends IndexStruct {
@@ -106,6 +107,36 @@ export class IndexList extends IndexStruct {
   }
 }
 
+// A table of keywords mapping keywords to text chunks.
+export class KeywordTable extends IndexStruct {
+  table: Map<string, Set<string>> = new Map();
+  type: IndexStructType = IndexStructType.KEYWORD_TABLE;
+  addNode(keywords: string[], nodeId: string): void {
+    keywords.forEach((keyword) => {
+      if (!this.table.has(keyword)) {
+        this.table.set(keyword, new Set());
+      }
+      this.table.get(keyword)!.add(nodeId);
+    });
+  }
+
+  deleteNode(keywords: string[], nodeId: string) {
+    keywords.forEach((keyword) => {
+      if (this.table.has(keyword)) {
+        this.table.get(keyword)!.delete(nodeId);
+      }
+    });
+  }
+
+  toJson(): Record<string, unknown> {
+    return {
+      ...super.toJson(),
+      table: this.table,
+      type: this.type,
+    };
+  }
+}
+
 export interface BaseIndexInit<T> {
   serviceContext: ServiceContext;
   storageContext: StorageContext;

packages/core/src/indices/index.ts

@@ -1,3 +1,4 @@
 export * from "./BaseIndex";
+export * from "./keyword";
 export * from "./summary";
 export * from "./vectorStore";

packages/core/src/indices/keyword/KeywordTableIndex.ts

@@ -0,0 +1,269 @@
+import { BaseNode, Document, MetadataMode } from "../../Node";
+import { defaultKeywordExtractPrompt } from "../../Prompt";
+import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
+import { ResponseSynthesizer } from "../../ResponseSynthesizer";
+import { BaseRetriever } from "../../Retriever";
+import {
+  ServiceContext,
+  serviceContextFromDefaults,
+} from "../../ServiceContext";
+import { StorageContext, storageContextFromDefaults } from "../../storage";
+import { BaseDocumentStore } from "../../storage/docStore/types";
+import {
+  BaseIndex,
+  BaseIndexInit,
+  IndexStructType,
+  KeywordTable,
+} from "../BaseIndex";
+import {
+  KeywordTableLLMRetriever,
+  KeywordTableRAKERetriever,
+  KeywordTableSimpleRetriever,
+} from "./KeywordTableIndexRetriever";
+import { extractKeywordsGivenResponse } from "./utils";
+
+export interface KeywordIndexOptions {
+  nodes?: BaseNode[];
+  indexStruct?: KeywordTable;
+  indexId?: string;
+  serviceContext?: ServiceContext;
+  storageContext?: StorageContext;
+}
+export enum KeywordTableRetrieverMode {
+  DEFAULT = "DEFAULT",
+  SIMPLE = "SIMPLE",
+  RAKE = "RAKE",
+}
+
+const KeywordTableRetrieverMap = {
+  [KeywordTableRetrieverMode.DEFAULT]: KeywordTableLLMRetriever,
+  [KeywordTableRetrieverMode.SIMPLE]: KeywordTableSimpleRetriever,
+  [KeywordTableRetrieverMode.RAKE]: KeywordTableRAKERetriever,
+};
+
+/**
+ * The KeywordTableIndex, an index that extracts keywords from each Node and builds a mapping from each keyword to the corresponding Nodes of that keyword.
+ */
+export class KeywordTableIndex extends BaseIndex<KeywordTable> {
+  constructor(init: BaseIndexInit<KeywordTable>) {
+    super(init);
+  }
+
+  static async init(options: KeywordIndexOptions): Promise<KeywordTableIndex> {
+    const storageContext =
+      options.storageContext ?? (await storageContextFromDefaults({}));
+    const serviceContext =
+      options.serviceContext ?? serviceContextFromDefaults({});
+    const { docStore, indexStore } = storageContext;
+
+    // Setup IndexStruct from storage
+    let indexStructs = (await indexStore.getIndexStructs()) as KeywordTable[];
+    let indexStruct: KeywordTable | null;
+
+    if (options.indexStruct && indexStructs.length > 0) {
+      throw new Error(
+        "Cannot initialize index with both indexStruct and indexStore",
+      );
+    }
+
+    if (options.indexStruct) {
+      indexStruct = options.indexStruct;
+    } else if (indexStructs.length == 1) {
+      indexStruct = indexStructs[0];
+    } else if (indexStructs.length > 1 && options.indexId) {
+      indexStruct = (await indexStore.getIndexStruct(
+        options.indexId,
+      )) as KeywordTable;
+    } else {
+      indexStruct = null;
+    }
+
+    // check indexStruct type
+    if (indexStruct && indexStruct.type !== IndexStructType.KEYWORD_TABLE) {
+      throw new Error(
+        "Attempting to initialize KeywordTableIndex with non-keyword table indexStruct",
+      );
+    }
+
+    if (indexStruct) {
+      if (options.nodes) {
+        throw new Error(
+          "Cannot initialize KeywordTableIndex with both nodes and indexStruct",
+        );
+      }
+    } else {
+      if (!options.nodes) {
+        throw new Error(
+          "Cannot initialize KeywordTableIndex without nodes or indexStruct",
+        );
+      }
+      indexStruct = await KeywordTableIndex.buildIndexFromNodes(
+        options.nodes,
+        storageContext.docStore,
+        serviceContext,
+      );
+
+      await indexStore.addIndexStruct(indexStruct);
+    }
+
+    return new KeywordTableIndex({
+      storageContext,
+      serviceContext,
+      docStore,
+      indexStore,
+      indexStruct,
+    });
+  }
+
+  asRetriever(options?: any): BaseRetriever {
+    const { mode = KeywordTableRetrieverMode.DEFAULT, ...otherOptions } =
+      options ?? {};
+    const KeywordTableRetriever =
+      KeywordTableRetrieverMap[mode as KeywordTableRetrieverMode];
+    if (KeywordTableRetriever) {
+      return new KeywordTableRetriever({ index: this, ...otherOptions });
+    }
+    throw new Error(`Unknown retriever mode: ${mode}`);
+  }
+
+  asQueryEngine(options?: {
+    retriever?: BaseRetriever;
+    responseSynthesizer?: ResponseSynthesizer;
+  }): BaseQueryEngine {
+    const { retriever, responseSynthesizer } = options ?? {};
+    return new RetrieverQueryEngine(
+      retriever ?? this.asRetriever(),
+      responseSynthesizer,
+    );
+  }
+
+  static async extractKeywords(
+    text: string,
+    serviceContext: ServiceContext,
+  ): Promise<Set<string>> {
+    const response = await serviceContext.llm.complete(
+      defaultKeywordExtractPrompt({
+        context: text,
+      }),
+    );
+    return extractKeywordsGivenResponse(response.message.content, "KEYWORDS:");
+  }
+
+  /**
+   * High level API: split documents, get keywords, and build index.
+   * @param documents
+   * @param storageContext
+   * @param serviceContext
+   * @returns
+   */
+  static async fromDocuments(
+    documents: Document[],
+    args: {
+      storageContext?: StorageContext;
+      serviceContext?: ServiceContext;
+    } = {},
+  ): Promise<KeywordTableIndex> {
+    let { storageContext, serviceContext } = args;
+    storageContext = storageContext ?? (await storageContextFromDefaults({}));
+    serviceContext = serviceContext ?? serviceContextFromDefaults({});
+    const docStore = storageContext.docStore;
+
+    docStore.addDocuments(documents, true);
+    for (const doc of documents) {
+      docStore.setDocumentHash(doc.id_, doc.hash);
+    }
+
+    const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
+    const index = await KeywordTableIndex.init({
+      nodes,
+      storageContext,
+      serviceContext,
+    });
+    return index;
+  }
+
+  /**
+   * Get keywords for nodes and place them into the index.
+   * @param nodes
+   * @param serviceContext
+   * @param vectorStore
+   * @returns
+   */
+  static async buildIndexFromNodes(
+    nodes: BaseNode[],
+    docStore: BaseDocumentStore,
+    serviceContext: ServiceContext,
+  ): Promise<KeywordTable> {
+    const indexStruct = new KeywordTable();
+    await docStore.addDocuments(nodes, true);
+    for (const node of nodes) {
+      const keywords = await KeywordTableIndex.extractKeywords(
+        node.getContent(MetadataMode.LLM),
+        serviceContext,
+      );
+      indexStruct.addNode([...keywords], node.id_);
+    }
+    return indexStruct;
+  }
+
+  async insertNodes(nodes: BaseNode[]) {
+    for (let node of nodes) {
+      const keywords = await KeywordTableIndex.extractKeywords(
+        node.getContent(MetadataMode.LLM),
+        this.serviceContext,
+      );
+      this.indexStruct.addNode([...keywords], node.id_);
+    }
+  }
+
+  deleteNode(nodeId: string): void {
+    const keywordsToDelete: Set<string> = new Set();
+    for (const [keyword, existingNodeIds] of Object.entries(
+      this.indexStruct.table,
+    )) {
+      const index = existingNodeIds.indexOf(nodeId);
+      if (index !== -1) {
+        existingNodeIds.splice(index, 1);
+
+        // Delete keywords that have zero nodes
+        if (existingNodeIds.length === 0) {
+          keywordsToDelete.add(keyword);
+        }
+      }
+    }
+    this.indexStruct.deleteNode([...keywordsToDelete], nodeId);
+  }
+
+  async deleteNodes(nodeIds: string[], deleteFromDocStore: boolean) {
+    nodeIds.forEach((nodeId) => {
+      this.deleteNode(nodeId);
+    });
+
+    if (deleteFromDocStore) {
+      for (const nodeId of nodeIds) {
+        await this.docStore.deleteDocument(nodeId, false);
+      }
+    }
+
+    await this.storageContext.indexStore.addIndexStruct(this.indexStruct);
+  }
+
+  async deleteRefDoc(
+    refDocId: string,
+    deleteFromDocStore?: boolean,
+  ): Promise<void> {
+    const refDocInfo = await this.docStore.getRefDocInfo(refDocId);
+
+    if (!refDocInfo) {
+      return;
+    }
+
+    await this.deleteNodes(refDocInfo.nodeIds, false);
+
+    if (deleteFromDocStore) {
+      await this.docStore.deleteRefDoc(refDocId, false);
+    }
+
+    return;
+  }
+}

packages/core/src/indices/keyword/KeywordTableIndexRetriever.ts

@@ -0,0 +1,119 @@
+import { NodeWithScore } from "../../Node";
+import {
+  defaultKeywordExtractPrompt,
+  defaultQueryKeywordExtractPrompt,
+  KeywordExtractPrompt,
+  QueryKeywordExtractPrompt,
+} from "../../Prompt";
+import { BaseRetriever } from "../../Retriever";
+import { ServiceContext } from "../../ServiceContext";
+import { BaseDocumentStore } from "../../storage/docStore/types";
+import { KeywordTable } from "../BaseIndex";
+import { KeywordTableIndex } from "./KeywordTableIndex";
+import {
+  extractKeywordsGivenResponse,
+  rakeExtractKeywords,
+  simpleExtractKeywords,
+} from "./utils";
+
+// Base Keyword Table Retriever
+abstract class BaseKeywordTableRetriever implements BaseRetriever {
+  protected index: KeywordTableIndex;
+  protected indexStruct: KeywordTable;
+  protected docstore: BaseDocumentStore;
+  protected serviceContext: ServiceContext;
+
+  protected maxKeywordsPerQuery: number; // Maximum number of keywords to extract from query.
+  protected numChunksPerQuery: number; // Maximum number of text chunks to query.
+  protected keywordExtractTemplate: KeywordExtractPrompt; // A Keyword Extraction Prompt
+  protected queryKeywordExtractTemplate: QueryKeywordExtractPrompt; // A Query Keyword Extraction Prompt
+
+  constructor({
+    index,
+    keywordExtractTemplate,
+    queryKeywordExtractTemplate,
+    maxKeywordsPerQuery = 10,
+    numChunksPerQuery = 10,
+  }: {
+    index: KeywordTableIndex;
+    keywordExtractTemplate?: KeywordExtractPrompt;
+    queryKeywordExtractTemplate?: QueryKeywordExtractPrompt;
+    maxKeywordsPerQuery: number;
+    numChunksPerQuery: number;
+  }) {
+    this.index = index;
+    this.indexStruct = index.indexStruct;
+    this.docstore = index.docStore;
+    this.serviceContext = index.serviceContext;
+
+    this.maxKeywordsPerQuery = maxKeywordsPerQuery;
+    this.numChunksPerQuery = numChunksPerQuery;
+    this.keywordExtractTemplate =
+      keywordExtractTemplate || defaultKeywordExtractPrompt;
+    this.queryKeywordExtractTemplate =
+      queryKeywordExtractTemplate || defaultQueryKeywordExtractPrompt;
+  }
+
+  abstract getKeywords(query: string): Promise<string[]>;
+
+  async retrieve(query: string): Promise<NodeWithScore[]> {
+    const keywords = await this.getKeywords(query);
+    const chunkIndicesCount: { [key: string]: number } = {};
+    const filteredKeywords = keywords.filter((keyword) =>
+      this.indexStruct.table.has(keyword),
+    );
+
+    for (let keyword of filteredKeywords) {
+      for (let nodeId of this.indexStruct.table.get(keyword) || []) {
+        chunkIndicesCount[nodeId] = (chunkIndicesCount[nodeId] ?? 0) + 1;
+      }
+    }
+
+    const sortedChunkIndices = Object.keys(chunkIndicesCount)
+      .sort((a, b) => chunkIndicesCount[b] - chunkIndicesCount[a])
+      .slice(0, this.numChunksPerQuery);
+
+    const sortedNodes = await this.docstore.getNodes(sortedChunkIndices);
+
+    return sortedNodes.map((node) => ({ node }));
+  }
+
+  getServiceContext(): ServiceContext {
+    return this.index.serviceContext;
+  }
+}
+
+// Extracts keywords using LLMs.
+export class KeywordTableLLMRetriever extends BaseKeywordTableRetriever {
+  async getKeywords(query: string): Promise<string[]> {
+    const response = await this.serviceContext.llm.complete(
+      this.queryKeywordExtractTemplate({
+        question: query,
+        maxKeywords: this.maxKeywordsPerQuery,
+      }),
+    );
+    const keywords = extractKeywordsGivenResponse(
+      response.message.content,
+      "KEYWORDS:",
+    );
+    return [...keywords];
+  }
+}
+
+// Extracts keywords using simple regex-based keyword extractor.
+export class KeywordTableSimpleRetriever extends BaseKeywordTableRetriever {
+  getKeywords(query: string): Promise<string[]> {
+    return Promise.resolve([
+      ...simpleExtractKeywords(query, this.maxKeywordsPerQuery),
+    ]);
+  }
+}
+
+// Extracts keywords using RAKE keyword extractor
+export class KeywordTableRAKERetriever extends BaseKeywordTableRetriever {
+  getKeywords(query: string): Promise<string[]> {
+    return Promise.resolve([
+      ...rakeExtractKeywords(query, this.maxKeywordsPerQuery),
+    ]);
+  }
+}

packages/core/src/indices/keyword/index.ts

@@ -0,0 +1,9 @@
+export {
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "./KeywordTableIndex";
+export {
+  KeywordTableLLMRetriever,
+  KeywordTableRAKERetriever,
+  KeywordTableSimpleRetriever,
+} from "./KeywordTableIndexRetriever";

packages/core/src/indices/keyword/utils.ts

@@ -0,0 +1,81 @@
+// @ts-ignore
+import rake from "rake-modified";
+
+// Get subtokens from a list of tokens., filtering for stopwords.
+export function expandTokensWithSubtokens(tokens: Set<string>): Set<string> {
+  const results: Set<string> = new Set();
+  const regex: RegExp = /\w+/g;
+
+  for (let token of tokens) {
+    results.add(token);
+    const subTokens: RegExpMatchArray | null = token.match(regex);
+    if (subTokens && subTokens.length > 1) {
+      for (let w of subTokens) {
+        results.add(w);
+      }
+    }
+  }
+  return results;
+}
+
+export function extractKeywordsGivenResponse(
+  response: string,
+  startToken: string = "",
+  lowercase: boolean = true,
+): Set<string> {
+  const results: string[] = [];
+  response = response.trim();
+
+  if (response.startsWith(startToken)) {
+    response = response.substring(startToken.length);
+  }
+
+  const keywords: string[] = response.split(",");
+  for (let k of keywords) {
+    let rk: string = k;
+    if (lowercase) {
+      rk = rk.toLowerCase();
+    }
+    results.push(rk.trim());
+  }
+
+  return expandTokensWithSubtokens(new Set(results));
+}
+
+export function simpleExtractKeywords(
+  textChunk: string,
+  maxKeywords?: number,
+): Set<string> {
+  const regex: RegExp = /\w+/g;
+  let tokens: string[] = [...textChunk.matchAll(regex)].map((token) =>
+    token[0].toLowerCase().trim(),
+  );
+
+  // Creating a frequency map
+  const valueCounts: { [key: string]: number } = {};
+  for (let token of tokens) {
+    valueCounts[token] = (valueCounts[token] || 0) + 1;
+  }
+
+  // Sorting tokens by frequency
+  const sortedTokens: string[] = Object.keys(valueCounts).sort(
+    (a, b) => valueCounts[b] - valueCounts[a],
+  );
+
+  const keywords: string[] = maxKeywords
+    ? sortedTokens.slice(0, maxKeywords)
+    : sortedTokens;
+
+  return new Set(keywords);
+}
+
+export function rakeExtractKeywords(
+  textChunk: string,
+  maxKeywords?: number,
+): Set<string> {
+  const keywords = Object.keys(rake(textChunk));
+  const limitedKeywords = maxKeywords
+    ? keywords.slice(0, maxKeywords)
+    : keywords;
+  return new Set(limitedKeywords);
+}

packages/core/src/readers/NotionReader.ts

@@ -0,0 +1,67 @@
+import { Client } from "@notionhq/client";
+import { crawler, Crawler, Pages, pageToString } from "notion-md-crawler";
+import { Document } from "../Node";
+import { BaseReader } from "./base";
+
+type OptionalSerializers = Parameters<Crawler>[number]["serializers"];
+
+/**
+ * Options for initializing the NotionReader class
+ * @typedef {Object} NotionReaderOptions
+ * @property {Client} client - The Notion Client object for API interactions
+ * @property {OptionalSerializers} [serializers] - Option to customize serialization. See [the url](https://github.com/TomPenguin/notion-md-crawler/tree/main) for details.
+ */
+type NotionReaderOptions = {
+  client: Client;
+  serializers?: OptionalSerializers;
+};
+
+/**
+ * Notion pages are retrieved recursively and converted to Document objects.
+ * Notion Database can also be loaded, and [the serialization method can be customized](https://github.com/TomPenguin/notion-md-crawler/tree/main).
+ *
+ * [Note] To use this reader, must be created the Notion integration must be created in advance
+ * Please refer to [this document](https://www.notion.so/help/create-integrations-with-the-notion-api) for details.
+ */
+export class NotionReader implements BaseReader {
+  private crawl: ReturnType<Crawler>;
+
+  /**
+   * Constructor for the NotionReader class
+   * @param {NotionReaderOptions} options - Configuration options for the reader
+   */
+  constructor({ client, serializers }: NotionReaderOptions) {
+    this.crawl = crawler({ client, serializers });
+  }
+
+  /**
+   * Converts Pages to an array of Document objects
+   * @param {Pages} pages - The Notion pages to convert (Return value of `loadPages`)
+   * @returns {Document[]} An array of Document objects
+   */
+  toDocuments(pages: Pages): Document[] {
+    return Object.values(pages).map((page) => {
+      const text = pageToString(page);
+      return new Document({ text, metadata: page.metadata });
+    });
+  }
+
+  /**
+   * Loads recursively the Notion page with the specified root page ID.
+   * @param {string} rootPageId - The root Notion page ID
+   * @returns {Promise<Pages>} A Promise that resolves to a Pages object(Convertible with the `toDocuments` method)
+   */
+  async loadPages(rootPageId: string): Promise<Pages> {
+    return this.crawl(rootPageId);
+  }
+
+  /**
+   * Loads recursively Notion pages and converts them to an array of Document objects
+   * @param {string} rootPageId - The root Notion page ID
+   * @returns {Promise<Document[]>} A Promise that resolves to an array of Document objects
+   */
+  async loadData(rootPageId: string): Promise<Document[]> {
+    const pages = await this.loadPages(rootPageId);
+    return this.toDocuments(pages);
+  }
+}

packages/core/src/readers/SimpleMongoReader.ts

@@ -0,0 +1,51 @@
+import { MongoClient } from "mongodb";
+import { Document } from "../Node";
+import { BaseReader } from "./base";
+
+/**
+ * Read in from MongoDB
+ */
+export class SimpleMongoReader implements BaseReader {
+  private client: MongoClient;
+
+  constructor(client: MongoClient) {
+    this.client = client;
+  }
+
+  /**
+   * Loads data from MongoDB collection
+   * @param {string} db_name - The name of the database to load.
+   * @param {string} collection_name - The name of the collection to load.
+   * @param {Number} [max_docs = 0] - Maximum number of documents to return. 0 means no limit.
+   * @param {Record<string, any>} [query_dict={}] - Specific query, as specified by MongoDB NodeJS documentation.
+   * @param {Record<string, any>} [query_options={}] - Specific query options, as specified by MongoDB NodeJS documentation.
+   * @param {Record<string, any>} [projection = {}] - Projection options, as specified by MongoDB NodeJS documentation.
+   * @returns {Promise<Document[]>}
+   */
+  async loadData(
+    db_name: string,
+    collection_name: string,
+    max_docs = 0,
+    //For later: Think about whether we want to pass generic objects in...
+    query_dict: Record<string, any> = {},
+    query_options: Record<string, any> = {},
+    projection: Record<string, any> = {},
+  ): Promise<Document[]> {
+    //Get items from collection using built-in functions
+    const cursor: Partial<Document>[] = await this.client
+      .db(db_name)
+      .collection(collection_name)
+      .find(query_dict, query_options)
+      .limit(max_docs)
+      .project(projection)
+      .toArray();
+
+    //Aggregate results and return
+    const documents: Document[] = [];
+    cursor.forEach((element: Partial<Document>) => {
+      //For later: Metadata filtering
+      documents.push(new Document({ text: JSON.stringify(element) }));
+    });
+    return documents;
+  }
+}

packages/core/src/tests/Keyword.test.ts

@@ -0,0 +1,84 @@
+import {
+  rakeExtractKeywords,
+  simpleExtractKeywords,
+} from "../indices/keyword/utils";
+describe("SimpleExtractKeywords", () => {
+  test("should extract unique keywords", () => {
+    const text = "apple banana apple cherry";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set(["apple", "banana", "cherry"]));
+  });
+
+  test("should handle empty string", () => {
+    const text = "";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set());
+  });
+
+  test("should handle case sensitivity", () => {
+    const text = "Apple apple";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set(["apple"]));
+  });
+
+  test("should order keywords by frequency", () => {
+    const text = "apple banana apple cherry banana apple";
+    const result = simpleExtractKeywords(text);
+    expect([...result]).toEqual(["apple", "banana", "cherry"]);
+  });
+
+  test("should respect the maxKeywords parameter", () => {
+    const text = "apple banana apple cherry banana apple orange";
+    const result = simpleExtractKeywords(text, 2);
+    expect(result).toEqual(new Set(["apple", "banana"]));
+  });
+
+  test("should handle non-alphabetic characters", () => {
+    const text = "apple! banana... apple? cherry, orange;";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set(["apple", "banana", "cherry", "orange"]));
+  });
+});
+
+describe("RakeExtractKeywords", () => {
+  const sampleText = `Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.`;
+  test("should return all keywords if maxKeywords is not provided", () => {
+    const result = rakeExtractKeywords(sampleText);
+    expect(result).toEqual(
+      new Set([
+        "strong feelings",
+        "beginning writers",
+        "short stories",
+        "write essays",
+        "stories",
+        "write",
+        "deep",
+        "imagined",
+        "characters",
+        "plot",
+        "hardly",
+        "awful",
+        "probably",
+        "supposed",
+        "wrote",
+        "didn",
+        "programming",
+        "writing",
+        "school",
+        "outside",
+        "main",
+        "college",
+      ]),
+    );
+  });
+
+  test("should respect the maxKeywords parameter", () => {
+    const result = rakeExtractKeywords(sampleText, 2);
+    expect(result).toEqual(new Set(["strong feelings", "beginning writers"]));
+  });
+
+  test("should handle empty return from rake", () => {
+    const result = rakeExtractKeywords("");
+    expect(result).toEqual(new Set());
+  });
+});

packages/core/src/tests/OutputParser.test.ts

@@ -0,0 +1,90 @@
+import { SubQuestionOutputParser } from "../OutputParser";
+
+//This parser is really important, so make sure to add tests
+// as the parser sees through more iterations.
+describe("SubQuestionOutputParser", () => {
+  test("parses expected", () => {
+    const parser = new SubQuestionOutputParser();
+
+    const data = [
+      {
+        name: "uber_10k",
+        description: "Provides information about Uber financials for year 2021",
+      },
+      {
+        name: "lyft_10k",
+        description: "Provides information about Lyft financials for year 2021",
+      },
+    ];
+
+    const data_str: string = JSON.stringify(data);
+    const full_string = `\`\`\`json
+${data_str}
+\`\`\``;
+
+    const real_answer = { parsedOutput: data, rawOutput: full_string };
+
+    expect(parser.parse(full_string)).toEqual(real_answer);
+  });
+
+  //This is in case our LLM outputs a list response, but without ```json.
+  test("parses without ```json", () => {
+    const parser = new SubQuestionOutputParser();
+
+    const data = [
+      {
+        name: "uber_10k",
+        description: "Provides information about Uber financials for year 2021",
+      },
+      {
+        name: "lyft_10k",
+        description: "Provides information about Lyft financials for year 2021",
+      },
+    ];
+
+    const data_str: string = JSON.stringify(data);
+    const full_string = `${data_str}`;
+
+    const real_answer = { parsedOutput: data, rawOutput: full_string };
+
+    expect(parser.parse(JSON.stringify(data))).toEqual(real_answer);
+  });
+
+  test("parses null single response", () => {
+    const parser = new SubQuestionOutputParser();
+    const data_str =
+      "[\n" +
+      "    {\n" +
+      `        "subQuestion": "Sorry, I don't have any relevant information to answer your question",\n` +
+      '        "toolName": ""\n' +
+      "    }\n" +
+      "]";
+    const data = [
+      {
+        subQuestion:
+          "Sorry, I don't have any relevant information to answer your question",
+        toolName: "",
+      },
+    ];
+    const real_answer = { parsedOutput: data, rawOutput: data_str };
+    expect(parser.parse(data_str)).toEqual(real_answer);
+  });
+
+  test("Single JSON object case", () => {
+    const parser = new SubQuestionOutputParser();
+    const data_str =
+      "    {\n" +
+      `        "subQuestion": "Sorry, I don't have any relevant information to answer your question",\n` +
+      '        "toolName": ""\n' +
+      "    }\n";
+    const data = [
+      {
+        subQuestion:
+          "Sorry, I don't have any relevant information to answer your question",
+        toolName: "",
+      },
+    ];
+    const real_answer = { parsedOutput: data, rawOutput: data_str };
+    expect(parser.parse(data_str)).toEqual(real_answer);
+  });
+});

packages/eslint-config-custom/index.js

@@ -26,6 +26,9 @@ module.exports = {
           "DEBUG",
           "no_proxy",
           "NO_PROXY",
+
+          "NOTION_TOKEN",
+          "MONGODB_URI",
         ],
       },
     ],

sweep.yaml

@@ -7,5 +7,17 @@ branch: "main"
 gha_enabled: False
 # This is the description of your project. It will be used by sweep when creating PRs. You can tell Sweep what's unique about your project, what frameworks you use, or anything else you want.
 # Here's an example: sweepai/sweep is a python project. The main api endpoints are in sweepai/api.py. Write code that adheres to PEP8.
-description: ""
+description: "LlamaIndexTS is a data framework in TypeScript for your LLM applications"
+
+sandbox:
+  install:
+    - npm install -g pnpm
+    - pnpm i
+    - pnpm add --save-dev prettier -w
+  check:
+    - pnpx prettier --write {file_path}
+    - pnpm eslint --fix {file_path}
+    - pnpx ts-node --type-check {file_path}
+    - pnpm test
+
 # Default Values: https://github.com/sweepai/sweep/blob/main/sweep.yaml