rolled back notion package and changeset

Create OutputParser.test.ts [Needs Merge]

.changeset/quiet-months-flow.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Typesafe metadata (thanks @TomPenguin)

.changeset/seven-adults-doubt.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+MongoReader (thanks @kkang2097)

.changeset/two-dogs-flow.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Make OutputParser less strict and add tests (Thanks @kkang2097)

.github/ISSUE_TEMPLATE/sweep-bugfix.yml

@@ -1,5 +1,5 @@
 name: Bugfix
-title: 'Sweep: '
+title: "Sweep: "
 description: Write something like "We notice ... behavior when ... happens instead of ...""
 labels: sweep
 body:
@@ -8,4 +8,4 @@ body:
     attributes:
       label: Details
       description: More details about the bug
-      placeholder: The bug might be in ... file
+      placeholder: The bug might be in ... file

.github/ISSUE_TEMPLATE/sweep-feature.yml

@@ -1,5 +1,5 @@
 name: Feature Request
-title: 'Sweep: '
+title: "Sweep: "
 description: Write something like "Write an api endpoint that does "..." in the "..." file"
 labels: sweep
 body:
@@ -8,4 +8,4 @@ body:
     attributes:
       label: Details
       description: More details for Sweep
-      placeholder: The new endpoint should use the ... class from ... file because it contains ... logic
+      placeholder: The new endpoint should use the ... class from ... file because it contains ... logic

.github/ISSUE_TEMPLATE/sweep-refactor.yml

@@ -1,5 +1,5 @@
 name: Refactor
-title: 'Sweep: '
+title: "Sweep: "
 description: Write something like "Modify the ... api endpoint to use ... version and ... framework"
 labels: sweep
 body:
@@ -8,4 +8,4 @@ body:
     attributes:
       label: Details
       description: More details for Sweep
-      placeholder: We are migrating this function to ... version because ...
+      placeholder: We are migrating this function to ... version because ...

.github/workflows/lint_on_push_or_pull.yml

@@ -22,4 +22,4 @@ jobs:
         run: pnpm install
 
       - name: Run lint
-        run: pnpm run lint
+        run: pnpm run lint

.github/workflows/test.yml

@@ -7,18 +7,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v2
 
-    - name: Setup Node.js
-      uses: actions/setup-node@v2
-      with:
-        node-version: '18'
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: "18"
 
-    - name: Install dependencies
-      run: |
-        npm i -g pnpm
-        pnpm install
+      - name: Install dependencies
+        run: |
+          npm i -g pnpm
+          pnpm install
 
-    - name: Run tests
-      run: pnpm run test
+      - name: Run tests
+        run: pnpm run test

README.md

@@ -20,7 +20,7 @@ In a new folder:
 export OPENAI_API_KEY="sk-......" # Replace with your key from https://platform.openai.com/account/api-keys
 pnpm init
 pnpm install typescript
-pnpm exec tsc –-init # if needed
+pnpm exec tsc --init # if needed
 pnpm install llamaindex
 pnpm install @types/node
 ```
@@ -36,7 +36,7 @@ async function main() {
   // Load essay from abramov.txt in Node
   const essay = await fs.readFile(
     "node_modules/llamaindex/examples/abramov.txt",
-    "utf-8"
+    "utf-8",
   );
 
   // Create Document object with essay
@@ -48,7 +48,7 @@ async function main() {
   // Query the index
   const queryEngine = index.asQueryEngine();
   const response = await queryEngine.query(
-    "What did the author do in college?"
+    "What did the author do in college?",
   );
 
   // Output response
@@ -61,7 +61,7 @@ main();
 Then you can run it using
 
 ```bash
-pnpm dlx ts-node example.ts
+pnpx ts-node example.ts
 ```
 
 ## Playground

apps/docs/babel.config.js

@@ -1,3 +1,3 @@
 module.exports = {
-  presets: [require.resolve('@docusaurus/core/lib/babel/preset')],
+  presets: [require.resolve("@docusaurus/core/lib/babel/preset")],
 };

apps/docs/docs/concepts.md

@@ -8,40 +8,42 @@ LlamaIndex.TS helps you build LLM-powered applications (e.g. Q&A, chatbot) over
 
 In this high-level concepts guide, you will learn:
 
-* how an LLM can answer questions using your own data.
-* key concepts and modules in LlamaIndex.TS for composing your own query pipeline.
+- how an LLM can answer questions using your own data.
+- key concepts and modules in LlamaIndex.TS for composing your own query pipeline.
 
 ## Answering Questions Across Your Data
 
 LlamaIndex uses a two stage method when using an LLM with your data:
 
-1) **indexing stage**: preparing a knowledge base, and
-2) **querying stage**: retrieving relevant context from the knowledge to assist the LLM in responding to a question
+1. **indexing stage**: preparing a knowledge base, and
+2. **querying stage**: retrieving relevant context from the knowledge to assist the LLM in responding to a question
 
 ![](./_static/concepts/rag.jpg)
 
 This process is also known as Retrieval Augmented Generation (RAG).
 
-LlamaIndex.TS provides the essential toolkit for making both steps super easy. 
+LlamaIndex.TS provides the essential toolkit for making both steps super easy.
 
 Let's explore each stage in detail.
 
 ### Indexing Stage
+
 LlamaIndex.TS help you prepare the knowledge base with a suite of data connectors and indexes.
 
-![](./_static/concepts/indexing.jpg) 
+![](./_static/concepts/indexing.jpg)
 
 [**Data Loaders**](./modules/high_level/data_loader.md):
 A data connector (i.e. `Reader`) ingest data from different data sources and data formats into a simple `Document` representation (text and simple metadata).
 
 [**Documents / Nodes**](./modules/high_level/documents_and_nodes.md): A `Document` is a generic container around any data source - for instance, a PDF, an API output, or retrieved data from a database. A `Node` is the atomic unit of data in LlamaIndex and represents a "chunk" of a source `Document`. It's a rich representation that includes metadata and relationships (to other nodes) to enable accurate and expressive retrieval operations.
 
-[**Data Indexes**](./modules/high_level/data_index.md): 
+[**Data Indexes**](./modules/high_level/data_index.md):
 Once you've ingested your data, LlamaIndex helps you index data into a format that's easy to retrieve.
 
 Under the hood, LlamaIndex parses the raw documents into intermediate representations, calculates vector embeddings, and stores your data in-memory or to disk.
 
 ### Querying Stage
+
 In the querying stage, the query pipeline retrieves the most relevant context given a user query,
 and pass that to the LLM (along with the query) to synthesize a response.
 
@@ -57,12 +59,13 @@ These building blocks can be customized to reflect ranking preferences, as well
 ![](./_static/concepts/querying.jpg)
 
 #### Building Blocks
-[**Retrievers**](./modules/low_level/retriever.md): 
+
+[**Retrievers**](./modules/low_level/retriever.md):
 A retriever defines how to efficiently retrieve relevant context from a knowledge base (i.e. index) when given a query.
 The specific retrieval logic differs for difference indices, the most popular being dense retrieval against a vector index.
 
 [**Response Synthesizers**](./modules/low_level/response_synthesizer.md):
-A response synthesizer generates a response from an LLM, using a user query and a given set of retrieved text chunks.  
+A response synthesizer generates a response from an LLM, using a user query and a given set of retrieved text chunks.
 
 #### Pipelines
 
@@ -70,7 +73,6 @@ A response synthesizer generates a response from an LLM, using a user query and
 A query engine is an end-to-end pipeline that allow you to ask question over your data.
 It takes in a natural language query, and returns a response, along with reference context retrieved and passed to the LLM.
 
-
-[**Chat Engines**](./modules/high_level/chat_engine.md): 
+[**Chat Engines**](./modules/high_level/chat_engine.md):
 A chat engine is an end-to-end pipeline for having a conversation with your data
 (multiple back-and-forth instead of a single question & answer).

apps/docs/docs/end_to_end.md

@@ -10,14 +10,14 @@ We include several end-to-end examples using LlamaIndex.TS in the repository
 
 Read a file and chat about it with the LLM.
 
-## [List Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/listIndex.ts)
-
-Create a list index and query it. This example also use the `LLMRetriever`, which will use the LLM to select the best nodes to use when generating answer.
-
 ## [Vector Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/vectorIndex.ts)
 
 Create a vector index and query it. The vector index will use embeddings to fetch the top k most relevant nodes. By default, the top k is 2.
 
+## [Summary Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/summaryIndex.ts)
+
+Create a list index and query it. This example also use the `LLMRetriever`, which will use the LLM to select the best nodes to use when generating answer.
+
 ## [Save / Load an Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/storageContext.ts)
 
 Create and load a vector index. Persistance to disk in LlamaIndex.TS happens automatically once a storage context object is created.
@@ -28,7 +28,7 @@ Create a vector index and query it, while also configuring the the `LLM`, the `S
 
 ## [OpenAI LLM](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/openai.ts)
 
-Create an OpenAI LLM and directly use it for chat. 
+Create an OpenAI LLM and directly use it for chat.
 
 ## [Llama2 DeuceLLM](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/llamadeuce.ts)
 
@@ -40,4 +40,4 @@ Uses the `SubQuestionQueryEngine`, which breaks complex queries into multiple qu
 
 ## [Low Level Modules](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/lowlevel.ts)
 
-This example uses several low-level components, which removes the need for an actual query engine. These components can be used anywhere, in any application, or customized and sub-classed to meet your own needs.
+This example uses several low-level components, which removes the need for an actual query engine. These components can be used anywhere, in any application, or customized and sub-classed to meet your own needs.

apps/docs/docs/modules/_category_.yml

@@ -1,3 +1,3 @@
 label: "Modules"
 collapsed: false
-position: 5
+position: 5

apps/docs/docs/modules/high_level/_category_.yml

@@ -1 +1 @@
-label: High-Level Modules
+label: High-Level Modules

apps/docs/docs/modules/high_level/data_index.md

@@ -6,23 +6,18 @@ sidebar_position: 2
 
 An index is the basic container and organization for your data. LlamaIndex.TS supports two indexes:
 
-- `ListIndex` - will send every `Node` in the index to the LLM in order to generate a response
 - `VectorStoreIndex` - will send the top-k `Node`s to the LLM when generating a response. The default top-k is 2.
+- `SummaryIndex` - will send every `Node` in the index to the LLM in order to generate a response
 
 ```typescript
-import {
-  Document,
-  VectorStoreIndex,
-} from "llamaindex";
+import { Document, VectorStoreIndex } from "llamaindex";
 
 const document = new Document({ text: "test" });
 
-const index = await VectorStoreIndex.fromDocuments(
-  [document]
-);
+const index = await VectorStoreIndex.fromDocuments([document]);
 ```
 
 ## API Reference
 
-- [ListIndex](../../api/classes/ListIndex.md)
-- [VectorStoreIndex](../../api/classes/VectorStoreIndex.md)
+- [SummaryIndex](../../api/classes/SummaryIndex.md)
+- [VectorStoreIndex](../../api/classes/VectorStoreIndex.md)

apps/docs/docs/modules/high_level/documents_and_nodes.md

@@ -9,7 +9,7 @@ sidebar_position: 0
 ```typescript
 import { Document } from "llamaindex";
 
-document = new Document({ text: "text", metadata: { "key": "val" }});
+document = new Document({ text: "text", metadata: { key: "val" } });
 ```
 
 ## API Reference

apps/docs/docs/modules/low_level/_category_.yml

@@ -1 +1 @@
-label: Low-Level Modules
+label: Low-Level Modules

apps/docs/docs/modules/low_level/embedding.md

@@ -4,7 +4,7 @@ sidebar_position: 1
 
 # Embedding
 
-The embedding model in LlamaIndex is responsible for creating numerical representations of text. By default, LlamaIndex will use the `text-embedding-ada-002` model from OpenAI. 
+The embedding model in LlamaIndex is responsible for creating numerical representations of text. By default, LlamaIndex will use the `text-embedding-ada-002` model from OpenAI.
 
 This can be explicitly set in the `ServiceContext` object.
 

apps/docs/docs/modules/low_level/llm.md

@@ -4,7 +4,7 @@ sidebar_position: 0
 
 # LLM
 
-The LLM is responsible for reading text and generating natural language responses to queries. By default, LlamaIndex.TS uses `gpt-3.5-turbo`. 
+The LLM is responsible for reading text and generating natural language responses to queries. By default, LlamaIndex.TS uses `gpt-3.5-turbo`.
 
 The LLM can be explicitly set in the `ServiceContext` object.
 
@@ -19,4 +19,4 @@ const serviceContext = serviceContextFromDefaults({ llm: openaiLLM });
 ## API Reference
 
 - [OpenAI](../../api/classes/OpenAI.md)
-- [ServiceContext](../../api/interfaces/ServiceContext.md)
+- [ServiceContext](../../api/interfaces/ServiceContext.md)

apps/docs/docs/modules/low_level/node_parser.md

@@ -7,10 +7,7 @@ sidebar_position: 3
 The `NodeParser` in LlamaIndex is responbile for splitting `Document` objects into more manageable `Node` objects. When you call `.fromDocuments()`, the `NodeParser` from the `ServiceContext` is used to do this automatically for you. Alternatively, you can use it to split documents ahead of time.
 
 ```typescript
-import {
-  Document,
-  SimpleNodeParser,
-} from "llamaindex";
+import { Document, SimpleNodeParser } from "llamaindex";
 
 const nodeParser = new SimpleNodeParser();
 const nodes = nodeParser.getNodesFromDocuments([
@@ -25,7 +22,7 @@ The underlying text splitter will split text by sentences. It can also be used a
 ```typescript
 import { SentenceSplitter } from "llamaindex";
 
-const splitter = new SentenceSplitter({ chunkSize: 1, });
+const splitter = new SentenceSplitter({ chunkSize: 1 });
 
 const textSplits = splitter.splitText("Hello World");
 ```

apps/docs/docs/modules/low_level/response_synthesizer.md

@@ -6,26 +6,21 @@ sidebar_position: 6
 
 The ResponseSynthesizer is responsible for sending the query, nodes, and prompt templates to the LLM to generate a response. There are a few key modes for generating a response:
 
-- `Refine`: "create and refine" an answer by sequentially going through each retrieved text chunk. 
-    This makes a separate LLM call per Node. Good for more detailed answers.
-- `CompactAndRefine` (default): "compact" the prompt during each LLM call by stuffing as 
-    many text chunks that can fit within the maximum prompt size. If there are 
-    too many chunks to stuff in one prompt, "create and refine" an answer by going through
-    multiple compact prompts. The same as `refine`, but should result in less LLM calls.
-- `TreeSummarize`: Given a set of text chunks and the query, recursively construct a tree 
-    and return the root node as the response. Good for summarization purposes.
+- `Refine`: "create and refine" an answer by sequentially going through each retrieved text chunk.
+  This makes a separate LLM call per Node. Good for more detailed answers.
+- `CompactAndRefine` (default): "compact" the prompt during each LLM call by stuffing as
+  many text chunks that can fit within the maximum prompt size. If there are
+  too many chunks to stuff in one prompt, "create and refine" an answer by going through
+  multiple compact prompts. The same as `refine`, but should result in less LLM calls.
+- `TreeSummarize`: Given a set of text chunks and the query, recursively construct a tree
+  and return the root node as the response. Good for summarization purposes.
 - `SimpleResponseBuilder`: Given a set of text chunks and the query, apply the query to each text
-    chunk while accumulating the responses into an array. Returns a concatenated string of all
-    responses. Good for when you need to run the same query separately against each text
-    chunk.
+  chunk while accumulating the responses into an array. Returns a concatenated string of all
+  responses. Good for when you need to run the same query separately against each text
+  chunk.
 
 ```typescript
-import {
-  TextNode,
-  NodeWithScore,
-  ResponseSynthesizer,
-  CompactAndRefine
-} from "llamaindex";
+import { NodeWithScore, ResponseSynthesizer, TextNode } from "llamaindex";
 
 const responseSynthesizer = new ResponseSynthesizer();
 
@@ -42,7 +37,7 @@ const nodesWithScore: NodeWithScore[] = [
 
 const response = await responseSynthesizer.synthesize(
   "What age am I?",
-  nodesWithScore
+  nodesWithScore,
 );
 console.log(response.response);
 ```

apps/docs/docs/modules/low_level/retriever.md

@@ -4,10 +4,10 @@ sidebar_position: 5
 
 # Retriever
 
-A retriever in LlamaIndex is what is used to fetch `Node`s from an index using a query string. For example, a `ListIndexRetriever` will fetch all nodes no matter the query. Meanwhile, a `VectorIndexRetriever` will only fetch the top-k most similar nodes.
+A retriever in LlamaIndex is what is used to fetch `Node`s from an index using a query string. Aa `VectorIndexRetriever` will fetch the top-k most similar nodes. Meanwhile, a `SummaryIndexRetriever` will fetch all nodes no matter the query.
 
 ```typescript
-const retriever = vector_index.asRetriever()
+const retriever = vector_index.asRetriever();
 retriever.similarityTopK = 3;
 
 // Fetch nodes!
@@ -16,6 +16,6 @@ const nodesWithScore = await retriever.retrieve("query string");
 
 ## API Reference
 
-- [ListIndexRetriever](../../api/classes/ListIndexRetriever.md)
-- [ListIndexLLMRetriever](../../api/classes/ListIndexLLMRetriever.md)
+- [SummaryIndexRetriever](../../api/classes/SummaryIndexRetriever.md)
+- [SummaryIndexLLMRetriever](../../api/classes/SummaryIndexLLMRetriever.md)
 - [VectorIndexRetriever](../../api/classes/VectorIndexRetriever.md)

apps/docs/docs/modules/low_level/storage.md

@@ -11,10 +11,14 @@ Right now, only saving and loading from disk is supported, with future integrati
 ```typescript
 import { Document, VectorStoreIndex, storageContextFromDefaults } from "./src";
 
-const storageContext = await storageContextFromDefaults({ persistDir: "./storage" });
+const storageContext = await storageContextFromDefaults({
+  persistDir: "./storage",
+});
 
 const document = new Document({ text: "Test Text" });
-const index = await VectorStoreIndex.fromDocuments([document], { storageContext });
+const index = await VectorStoreIndex.fromDocuments([document], {
+  storageContext,
+});
 ```
 
 ## API Reference

apps/docs/docs/starter.md

@@ -25,7 +25,7 @@ async function main() {
   // Load essay from abramov.txt in Node
   const essay = await fs.readFile(
     "node_modules/llamaindex/examples/abramov.txt",
-    "utf-8"
+    "utf-8",
   );
 
   // Create Document object with essay
@@ -37,7 +37,7 @@ async function main() {
   // Query the index
   const queryEngine = index.asQueryEngine();
   const response = await queryEngine.query(
-    "What did the author do in college?"
+    "What did the author do in college?",
   );
 
   // Output response

apps/docs/docusaurus.config.js

@@ -139,6 +139,8 @@ const config = {
         entryPoints: ["../../packages/core/src/index.ts"],
         tsconfig: "../../packages/core/tsconfig.json",
         readme: "none",
+        sourceLinkTemplate:
+          "https://github.com/run-llama/LlamaIndexTS/blob/{gitRevision}/{path}#L{line}",
         sidebar: {
           position: 6,
         },

apps/docs/src/components/HomepageFeatures/index.tsx

@@ -1,5 +1,5 @@
-import React from "react";
 import clsx from "clsx";
+import React from "react";
 import styles from "./styles.module.css";
 
 type FeatureItem = {

apps/docs/src/css/custom.css

@@ -18,7 +18,7 @@
 }
 
 /* For readability concerns, you should choose a lighter palette in dark mode. */
-[data-theme='dark'] {
+[data-theme="dark"] {
   --ifm-color-primary: #25c2a0;
   --ifm-color-primary-dark: #21af90;
   --ifm-color-primary-darker: #1fa588;

apps/simple/CHANGELOG.md

@@ -1,5 +1,66 @@
 # simple
 
+## 0.0.25
+
+### Patch Changes
+
+- Updated dependencies [4a5591b]
+- Updated dependencies [4a5591b]
+- Updated dependencies [4a5591b]
+  - llamaindex@0.0.27
+
+## 0.0.24
+
+### Patch Changes
+
+- Updated dependencies [5bb55bc]
+  - llamaindex@0.0.26
+
+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [e21eca2]
+- Updated dependencies [40a8f07]
+- Updated dependencies [40a8f07]
+  - llamaindex@0.0.25
+
+## 0.0.22
+
+### Patch Changes
+
+- Updated dependencies [e4af7b3]
+- Updated dependencies [259fe63]
+  - llamaindex@0.0.24
+
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies
+- Updated dependencies [9d6b2ed]
+  - llamaindex@0.0.23
+
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [454f3f8]
+- Updated dependencies [454f3f8]
+- Updated dependencies [454f3f8]
+- Updated dependencies [454f3f8]
+- Updated dependencies [99df58f]
+  - llamaindex@0.0.22
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [f7a57ca]
+- Updated dependencies [0a09de2]
+- Updated dependencies [f7a57ca]
+  - llamaindex@0.0.21
+
 ## 0.0.18
 
 ### Patch Changes

apps/simple/README.md

@@ -1,9 +1,10 @@
 # Simple Examples
 
 Due to packaging, you will need to run these commands to get started.
+
 ```bash
-pnpm --filter llamaindex build
 pnpm install
+pnpm --filter llamaindex build
 ```
 
 Then run the examples with `ts-node`, for example `npx ts-node vectorIndex.ts`

apps/simple/csv.ts

@@ -4,7 +4,6 @@ import {
   PapaCSVReader,
   ResponseSynthesizer,
   serviceContextFromDefaults,
-  SimplePrompt,
   VectorStoreIndex,
 } from "llamaindex";
 
@@ -23,9 +22,7 @@ async function main() {
     serviceContext,
   });
 
-  const csvPrompt: SimplePrompt = (input) => {
-    const { context = "", query = "" } = input;
-
+  const csvPrompt = ({ context = "", query = "" }) => {
     return `The following CSV file is loaded from ${path}
 \`\`\`csv
 ${context}

apps/simple/keywordIndex.ts

@@ -0,0 +1,32 @@
+import {
+  Document,
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "llamaindex";
+import essay from "./essay";
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+  const index = await KeywordTableIndex.fromDocuments([document]);
+
+  const allModes: KeywordTableRetrieverMode[] = [
+    KeywordTableRetrieverMode.DEFAULT,
+    KeywordTableRetrieverMode.SIMPLE,
+    KeywordTableRetrieverMode.RAKE,
+  ];
+  allModes.forEach(async (mode) => {
+    const queryEngine = index.asQueryEngine({
+      retriever: index.asRetriever({
+        mode,
+      }),
+    });
+    const response = await queryEngine.query(
+      "What did the author do growing up?",
+    );
+    console.log(response.toString());
+  });
+}
+
+main().catch((e: Error) => {
+  console.error(e, e.stack);
+});

apps/simple/lowlevel.ts

@@ -1,9 +1,9 @@
 import {
   Document,
-  TextNode,
   NodeWithScore,
   ResponseSynthesizer,
   SimpleNodeParser,
+  TextNode,
 } from "llamaindex";
 
 (async () => {
@@ -29,7 +29,7 @@ import {
 
   const response = await responseSynthesizer.synthesize(
     "What age am I?",
-    nodesWithScore
+    nodesWithScore,
   );
   console.log(response.response);
 })();

apps/simple/markdown.ts

@@ -0,0 +1,20 @@
+import { MarkdownReader, VectorStoreIndex } from "llamaindex";
+
+async function main() {
+  // Load Markdown file
+  const reader = new MarkdownReader();
+  const documents = await reader.loadData("node_modules/llamaindex/README.md");
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  const index = await VectorStoreIndex.fromDocuments(documents);
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+
+  const response = await queryEngine.query("What does the example code do?");
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);

apps/simple/mongo.ts

@@ -0,0 +1,68 @@
+import { MongoClient } from "mongodb";
+import { VectorStoreIndex } from "../../packages/core/src/indices";
+import { Document } from "../../packages/core/src/Node";
+import { SimpleMongoReader } from "../../packages/core/src/readers/SimpleMongoReader";
+
+import { stdin as input, stdout as output } from "node:process";
+import readline from "node:readline/promises";
+
+async function main() {
+  //Dummy test code
+  const query: object = { _id: "waldo" };
+  const options: object = {};
+  const projections: object = { embedding: 0 };
+  const limit: number = Infinity;
+  const uri: string = process.env.MONGODB_URI ?? "fake_uri";
+  const client: MongoClient = new MongoClient(uri);
+
+  //Where the real code starts
+  const MR = new SimpleMongoReader(client);
+  const documents: Document[] = await MR.loadData(
+    "data",
+    "posts",
+    1,
+    {},
+    options,
+    projections,
+  );
+
+  //
+  //If you need to look at low-level details of
+  // a queryEngine (for example, needing to check each individual node)
+  //
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  // var storageContext = await storageContextFromDefaults({});
+  // var serviceContext = serviceContextFromDefaults({});
+  // const docStore = storageContext.docStore;
+
+  // for (const doc of documents) {
+  //   docStore.setDocumentHash(doc.id_, doc.hash);
+  // }
+  // const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
+  // console.log(nodes);
+
+  //
+  //Making Vector Store from documents
+  //
+
+  const index = await VectorStoreIndex.fromDocuments(documents);
+  // Create query engine
+  const queryEngine = index.asQueryEngine();
+
+  const rl = readline.createInterface({ input, output });
+  while (true) {
+    const query = await rl.question("Query: ");
+
+    if (!query) {
+      break;
+    }
+
+    const response = await queryEngine.query(query);
+
+    // Output response
+    console.log(response.toString());
+  }
+}
+
+main();

apps/simple/notion.ts

@@ -0,0 +1,89 @@
+import { Client } from "@notionhq/client";
+import { program } from "commander";
+import { NotionReader, VectorStoreIndex } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+
+program
+  .argument("[page]", "Notion page id (must be provided)")
+  .action(async (page, _options, command) => {
+    // Initializing a client
+
+    if (!process.env.NOTION_TOKEN) {
+      console.log(
+        "No NOTION_TOKEN found in environment variables. You will need to register an integration https://www.notion.com/my-integrations and put it in your NOTION_TOKEN environment variable.",
+      );
+      return;
+    }
+
+    const notion = new Client({
+      auth: process.env.NOTION_TOKEN,
+    });
+
+    if (!page) {
+      const response = await notion.search({
+        filter: {
+          value: "page",
+          property: "object",
+        },
+        sort: {
+          direction: "descending",
+          timestamp: "last_edited_time",
+        },
+      });
+
+      const { results } = response;
+
+      if (results.length === 0) {
+        console.log(
+          "No pages found. You will need to share it with your integration. (tap the three dots on the top right, find Add connections, and add your integration)",
+        );
+        return;
+      } else {
+        const pages = results
+          .map((result) => {
+            if (!("url" in result)) {
+              return null;
+            }
+
+            return {
+              id: result.id,
+              url: result.url,
+            };
+          })
+          .filter((page) => page !== null);
+        console.log("Found pages:");
+        console.table(pages);
+        console.log(`To run, run ts-node ${command.name()} [page id]`);
+        return;
+      }
+    }
+
+    const reader = new NotionReader({ client: notion });
+    const documents = await reader.loadData(page);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Query: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      // Output response
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

apps/simple/openai.ts

@@ -1,14 +1,7 @@
 import { OpenAI } from "llamaindex";
 
 (async () => {
-  const llm = new OpenAI({
-    model: "gpt-3.5-turbo",
-    temperature: 0.1,
-    additionalChatOptions: { frequency_penalty: 0.1 },
-    additionalSessionOptions: {
-      defaultHeaders: { "X-Test-Header-Please-Ignore": "true" },
-    },
-  });
+  const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 });
 
   // complete api
   const response1 = await llm.complete("How are you?");

apps/simple/package.json

@@ -1,12 +1,14 @@
 {
-  "version": "0.0.18",
+  "version": "0.0.25",
   "private": true,
   "name": "simple",
   "dependencies": {
+    "@notionhq/client": "^2.2.12",
+    "commander": "^11.0.0",
     "llamaindex": "workspace:*"
   },
   "devDependencies": {
-    "@types/node": "^18.17.6"
+    "@types/node": "^18.17.12"
   },
   "scripts": {
     "lint": "eslint ."

apps/simple/summaryIndex.ts

@@ -1,9 +1,9 @@
 import {
   Document,
-  ListIndex,
-  ListRetrieverMode,
-  serviceContextFromDefaults,
   SimpleNodeParser,
+  SummaryIndex,
+  SummaryRetrieverMode,
+  serviceContextFromDefaults,
 } from "llamaindex";
 import essay from "./essay";
 
@@ -14,9 +14,11 @@ async function main() {
     }),
   });
   const document = new Document({ text: essay, id_: "essay" });
-  const index = await ListIndex.fromDocuments([document], { serviceContext });
+  const index = await SummaryIndex.fromDocuments([document], {
+    serviceContext,
+  });
   const queryEngine = index.asQueryEngine({
-    retriever: index.asRetriever({ mode: ListRetrieverMode.LLM }),
+    retriever: index.asRetriever({ mode: SummaryRetrieverMode.LLM }),
   });
   const response = await queryEngine.query(
     "What did the author do growing up?",

apps/simple/vectorIndexCustomize.ts

@@ -12,7 +12,7 @@ async function main() {
   const document = new Document({ text: essay, id_: "essay" });
 
   const serviceContext = serviceContextFromDefaults({
-    llm: new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 }),
+    llm: new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 }),
   });
 
   const index = await VectorStoreIndex.fromDocuments([document], {

examples/csv.ts

@@ -4,7 +4,6 @@ import {
   PapaCSVReader,
   ResponseSynthesizer,
   serviceContextFromDefaults,
-  SimplePrompt,
   VectorStoreIndex,
 } from "llamaindex";
 
@@ -23,9 +22,7 @@ async function main() {
     serviceContext,
   });
 
-  const csvPrompt: SimplePrompt = (input) => {
-    const { context = "", query = "" } = input;
-
+  const csvPrompt = ({ context = "", query = "" }) => {
     return `The following CSV file is loaded from ${path}
 \`\`\`csv
 ${context}

examples/keywordIndex.ts

@@ -0,0 +1,32 @@
+import {
+  Document,
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "llamaindex";
+import essay from "./essay";
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+  const index = await KeywordTableIndex.fromDocuments([document]);
+
+  const allModes: KeywordTableRetrieverMode[] = [
+    KeywordTableRetrieverMode.DEFAULT,
+    KeywordTableRetrieverMode.SIMPLE,
+    KeywordTableRetrieverMode.RAKE,
+  ];
+  allModes.forEach(async (mode) => {
+    const queryEngine = index.asQueryEngine({
+      retriever: index.asRetriever({
+        mode,
+      }),
+    });
+    const response = await queryEngine.query(
+      "What did the author do growing up?",
+    );
+    console.log(response.toString());
+  });
+}
+
+main().catch((e: Error) => {
+  console.error(e, e.stack);
+});

examples/lowlevel.ts

@@ -1,9 +1,9 @@
 import {
   Document,
-  TextNode,
   NodeWithScore,
   ResponseSynthesizer,
   SimpleNodeParser,
+  TextNode,
 } from "llamaindex";
 
 (async () => {
@@ -29,7 +29,7 @@ import {
 
   const response = await responseSynthesizer.synthesize(
     "What age am I?",
-    nodesWithScore
+    nodesWithScore,
   );
   console.log(response.response);
 })();

examples/markdown.ts

@@ -0,0 +1,20 @@
+import { MarkdownReader, VectorStoreIndex } from "llamaindex";
+
+async function main() {
+  // Load Markdown file
+  const reader = new MarkdownReader();
+  const documents = await reader.loadData("node_modules/llamaindex/README.md");
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  const index = await VectorStoreIndex.fromDocuments(documents);
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+
+  const response = await queryEngine.query("What does the example code do?");
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);

examples/notion.ts

@@ -0,0 +1,89 @@
+import { Client } from "@notionhq/client";
+import { program } from "commander";
+import { NotionReader, VectorStoreIndex } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+
+program
+  .argument("[page]", "Notion page id (must be provided)")
+  .action(async (page, _options, command) => {
+    // Initializing a client
+
+    if (!process.env.NOTION_TOKEN) {
+      console.log(
+        "No NOTION_TOKEN found in environment variables. You will need to register an integration https://www.notion.com/my-integrations and put it in your NOTION_TOKEN environment variable.",
+      );
+      return;
+    }
+
+    const notion = new Client({
+      auth: process.env.NOTION_TOKEN,
+    });
+
+    if (!page) {
+      const response = await notion.search({
+        filter: {
+          value: "page",
+          property: "object",
+        },
+        sort: {
+          direction: "descending",
+          timestamp: "last_edited_time",
+        },
+      });
+
+      const { results } = response;
+
+      if (results.length === 0) {
+        console.log(
+          "No pages found. You will need to share it with your integration. (tap the three dots on the top right, find Add connections, and add your integration)",
+        );
+        return;
+      } else {
+        const pages = results
+          .map((result) => {
+            if (!("url" in result)) {
+              return null;
+            }
+
+            return {
+              id: result.id,
+              url: result.url,
+            };
+          })
+          .filter((page) => page !== null);
+        console.log("Found pages:");
+        console.table(pages);
+        console.log(`To run, run ts-node ${command.name()} [page id]`);
+        return;
+      }
+    }
+
+    const reader = new NotionReader({ client: notion });
+    const documents = await reader.loadData(page);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Query: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      // Output response
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

examples/openai.ts

@@ -2,12 +2,14 @@ import { OpenAI } from "llamaindex";
 
 (async () => {
   const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 });
-  
+
   // complete api
   const response1 = await llm.complete("How are you?");
   console.log(response1.message.content);
 
   // chat api
-  const response2 = await llm.chat([{ content: "Tell me a joke!", role: "user" }]);
+  const response2 = await llm.chat([
+    { content: "Tell me a joke!", role: "user" },
+  ]);
   console.log(response2.message.content);
 })();

examples/summaryIndex.ts

@@ -1,9 +1,9 @@
 import {
   Document,
-  ListIndex,
-  ListRetrieverMode,
-  serviceContextFromDefaults,
   SimpleNodeParser,
+  SummaryIndex,
+  SummaryRetrieverMode,
+  serviceContextFromDefaults,
 } from "llamaindex";
 import essay from "./essay";
 
@@ -14,9 +14,11 @@ async function main() {
     }),
   });
   const document = new Document({ text: essay, id_: "essay" });
-  const index = await ListIndex.fromDocuments([document], { serviceContext });
+  const index = await SummaryIndex.fromDocuments([document], {
+    serviceContext,
+  });
   const queryEngine = index.asQueryEngine({
-    retriever: index.asRetriever({ mode: ListRetrieverMode.LLM }),
+    retriever: index.asRetriever({ mode: SummaryRetrieverMode.LLM }),
   });
   const response = await queryEngine.query(
     "What did the author do growing up?",

examples/vectorIndexCustomize.ts

@@ -12,7 +12,7 @@ async function main() {
   const document = new Document({ text: essay, id_: "essay" });
 
   const serviceContext = serviceContextFromDefaults({
-    llm: new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 }),
+    llm: new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 }),
   });
 
   const index = await VectorStoreIndex.fromDocuments([document], {

package.json

@@ -11,16 +11,16 @@
     "publish-snapshot": "turbo run build lint test && changeset version --snapshot && changeset publish"
   },
   "devDependencies": {
-    "@turbo/gen": "^1.10.12",
-    "@types/jest": "^29.5.3",
+    "@turbo/gen": "^1.10.13",
+    "@types/jest": "^29.5.4",
     "eslint": "^7.32.0",
     "eslint-config-custom": "workspace:*",
     "husky": "^8.0.3",
-    "jest": "^29.6.2",
-    "prettier": "^3.0.2",
+    "jest": "^29.6.4",
+    "prettier": "^3.0.3",
     "prettier-plugin-organize-imports": "^3.2.3",
     "ts-jest": "^29.1.1",
-    "turbo": "^1.10.12"
+    "turbo": "^1.10.13"
   },
   "packageManager": "pnpm@7.15.0",
   "dependencies": {

packages/core/CHANGELOG.md

@@ -1,5 +1,59 @@
 # llamaindex
 
+## 0.0.27
+
+### Patch Changes
+
+- 4a5591b: Chat History summarization (thanks @marcusschiesser)
+- 4a5591b: Notion database support (thanks @TomPenguin)
+- 4a5591b: KeywordIndex (thanks @swk777)
+
+## 0.0.26
+
+### Patch Changes
+
+- 5bb55bc: Add notion loader (thank you @TomPenguin!)
+
+## 0.0.25
+
+### Patch Changes
+
+- e21eca2: OpenAI 4.3.1 and Anthropic 0.6.2
+- 40a8f07: Update READMEs (thanks @andfk)
+- 40a8f07: Bug: missing exports from storage (thanks @aashutoshrathi)
+
+## 0.0.24
+
+### Patch Changes
+
+- e4af7b3: Renamed ListIndex to SummaryIndex to better indicate its use.
+- 259fe63: Strong types for prompts.
+
+## 0.0.23
+
+### Patch Changes
+
+- Added MetadataMode to ResponseSynthesizer (thanks @TomPenguin)
+- 9d6b2ed: Added Markdown Reader (huge shoutout to @swk777)
+
+## 0.0.22
+
+### Patch Changes
+
+- 454f3f8: CJK sentence splitting (thanks @TomPenguin)
+- 454f3f8: Export options for Windows formatted text files
+- 454f3f8: Disable long sentence splitting by default
+- 454f3f8: Make sentence splitter not split on decimals.
+- 99df58f: Anthropic 0.6.1 and OpenAI 4.2.0. Changed Anthropic timeout back to 60s
+
+## 0.0.21
+
+### Patch Changes
+
+- f7a57ca: Fixed metadata deserialization (thanks @marcagve)
+- 0a09de2: Update to OpenAI 4.1.0
+- f7a57ca: ChatGPT optimized prompts (thanks @LoganMarkewich)
+
 ## 0.0.20
 
 ### Patch Changes

packages/core/README.md

@@ -20,7 +20,7 @@ In a new folder:
 export OPENAI_API_KEY="sk-......" # Replace with your key from https://platform.openai.com/account/api-keys
 pnpm init
 pnpm install typescript
-pnpm exec tsc –-init # if needed
+pnpm exec tsc --init # if needed
 pnpm install llamaindex
 pnpm install @types/node
 ```
@@ -36,7 +36,7 @@ async function main() {
   // Load essay from abramov.txt in Node
   const essay = await fs.readFile(
     "node_modules/llamaindex/examples/abramov.txt",
-    "utf-8"
+    "utf-8",
   );
 
   // Create Document object with essay
@@ -48,7 +48,7 @@ async function main() {
   // Query the index
   const queryEngine = index.asQueryEngine();
   const response = await queryEngine.query(
-    "What did the author do in college?"
+    "What did the author do in college?",
   );
 
   // Output response
@@ -61,7 +61,7 @@ main();
 Then you can run it using
 
 ```bash
-pnpm dlx ts-node example.ts
+pnpx ts-node example.ts
 ```
 
 ## Playground

packages/core/package.json

@@ -1,23 +1,28 @@
 {
   "name": "llamaindex",
-  "version": "0.0.20",
+  "version": "0.0.27",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.6.0",
+    "@anthropic-ai/sdk": "^0.6.2",
+    "@notionhq/client": "^2.2.13",
     "lodash": "^4.17.21",
-    "openai": "^4.0.1",
+    "md-utils-ts": "^2.0.0",
+    "mongodb": "^6.0.0",
+    "notion-md-crawler": "^0.0.2",
+    "openai": "^4.7.0",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
-    "replicate": "^0.16.1",
+    "rake-modified": "^1.0.8",
+    "replicate": "^0.18.0",
     "tiktoken-node": "^0.0.6",
-    "uuid": "^9.0.0",
+    "uuid": "^9.0.1",
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
-    "@types/lodash": "^4.14.197",
-    "@types/node": "^18.17.6",
-    "@types/papaparse": "^5.3.7",
+    "@types/lodash": "^4.14.198",
+    "@types/node": "^18.17.15",
+    "@types/papaparse": "^5.3.8",
     "@types/pdf-parse": "^1.1.1",
-    "@types/uuid": "^9.0.2",
+    "@types/uuid": "^9.0.4",
     "node-stdlib-browser": "^1.2.0",
     "tsup": "^7.2.0"
   },

packages/core/src/ChatEngine.ts

@@ -1,17 +1,19 @@
-import { ChatMessage, OpenAI, ChatResponse, LLM } from "./llm/LLM";
+import { v4 as uuidv4 } from "uuid";
+import { Event } from "./callbacks/CallbackManager";
+import { ChatHistory, SimpleChatHistory } from "./ChatHistory";
+import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 import { TextNode } from "./Node";
 import {
-  SimplePrompt,
-  contextSystemPrompt,
+  CondenseQuestionPrompt,
+  ContextSystemPrompt,
   defaultCondenseQuestionPrompt,
+  defaultContextSystemPrompt,
   messagesToHistoryStr,
 } from "./Prompt";
 import { BaseQueryEngine } from "./QueryEngine";
 import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { v4 as uuidv4 } from "uuid";
-import { Event } from "./callbacks/CallbackManager";
 
 /**
  * A ChatEngine is used to handle back and forth chats between the application and the LLM.
@@ -70,13 +72,13 @@ export class CondenseQuestionChatEngine implements ChatEngine {
   queryEngine: BaseQueryEngine;
   chatHistory: ChatMessage[];
   serviceContext: ServiceContext;
-  condenseMessagePrompt: SimplePrompt;
+  condenseMessagePrompt: CondenseQuestionPrompt;
 
   constructor(init: {
     queryEngine: BaseQueryEngine;
     chatHistory: ChatMessage[];
     serviceContext?: ServiceContext;
-    condenseMessagePrompt?: SimplePrompt;
+    condenseMessagePrompt?: CondenseQuestionPrompt;
   }) {
     this.queryEngine = init.queryEngine;
     this.chatHistory = init?.chatHistory ?? [];
@@ -92,14 +94,14 @@ export class CondenseQuestionChatEngine implements ChatEngine {
     return this.serviceContext.llm.complete(
       defaultCondenseQuestionPrompt({
         question: question,
-        chat_history: chatHistoryStr,
-      })
+        chatHistory: chatHistoryStr,
+      }),
     );
   }
 
   async chat(
     message: string,
-    chatHistory?: ChatMessage[] | undefined
+    chatHistory?: ChatMessage[] | undefined,
   ): Promise<Response> {
     chatHistory = chatHistory ?? this.chatHistory;
 
@@ -129,16 +131,20 @@ export class ContextChatEngine implements ChatEngine {
   retriever: BaseRetriever;
   chatModel: OpenAI;
   chatHistory: ChatMessage[];
+  contextSystemPrompt: ContextSystemPrompt;
 
   constructor(init: {
     retriever: BaseRetriever;
     chatModel?: OpenAI;
     chatHistory?: ChatMessage[];
+    contextSystemPrompt?: ContextSystemPrompt;
   }) {
     this.retriever = init.retriever;
     this.chatModel =
       init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
     this.chatHistory = init?.chatHistory ?? [];
+    this.contextSystemPrompt =
+      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
   }
 
   async chat(message: string, chatHistory?: ChatMessage[] | undefined) {
@@ -151,11 +157,11 @@ export class ContextChatEngine implements ChatEngine {
     };
     const sourceNodesWithScore = await this.retriever.retrieve(
       message,
-      parentEvent
+      parentEvent,
     );
 
     const systemMessage: ChatMessage = {
-      content: contextSystemPrompt({
+      content: this.contextSystemPrompt({
         context: sourceNodesWithScore
           .map((r) => (r.node as TextNode).text)
           .join("\n\n"),
@@ -167,7 +173,7 @@ export class ContextChatEngine implements ChatEngine {
 
     const response = await this.chatModel.chat(
       [systemMessage, ...chatHistory],
-      parentEvent
+      parentEvent,
     );
     chatHistory.push(response.message);
 
@@ -175,7 +181,7 @@ export class ContextChatEngine implements ChatEngine {
 
     return new Response(
       response.message.content,
-      sourceNodesWithScore.map((r) => r.node)
+      sourceNodesWithScore.map((r) => r.node),
     );
   }
 
@@ -183,3 +189,28 @@ export class ContextChatEngine implements ChatEngine {
     this.chatHistory = [];
   }
 }
+
+/**
+ * HistoryChatEngine is a ChatEngine that uses a ChatHistory to keep track of the chat history. This is an example with the same behavior as SimpleChatEngine
+ * TODO: generally use the ChatHistory instead of ChatMessage[] - breaking change
+ */
+export class HistoryChatEngine implements ChatEngine {
+  chatHistory: ChatHistory;
+  llm: LLM;
+
+  constructor(init?: Partial<HistoryChatEngine>) {
+    this.chatHistory = init?.chatHistory ?? new SimpleChatHistory();
+    this.llm = init?.llm ?? new OpenAI();
+  }
+
+  async chat(message: string): Promise<Response> {
+    this.chatHistory.addMessage({ content: message, role: "user" });
+    const response = await this.llm.chat(this.chatHistory.messages);
+    this.chatHistory.addMessage(response.message);
+    return new Response(response.message.content);
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+}

packages/core/src/ChatHistory.ts

@@ -0,0 +1,73 @@
+import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
+import {
+  defaultSummaryPrompt,
+  messagesToHistoryStr,
+  SummaryPrompt,
+} from "./Prompt";
+
+/**
+ * A ChatHistory is used to keep the state of back and forth chat messages
+ */
+export interface ChatHistory {
+  messages: ChatMessage[];
+  /**
+   * Adds a message to the chat history.
+   * @param message
+   */
+  addMessage(message: ChatMessage): Promise<void>;
+
+  /**
+   * Resets the chat history so that it's empty.
+   */
+  reset(): void;
+}
+
+export class SimpleChatHistory implements ChatHistory {
+  messages: ChatMessage[];
+
+  constructor(init?: Partial<SimpleChatHistory>) {
+    this.messages = init?.messages ?? [];
+  }
+
+  async addMessage(message: ChatMessage) {
+    this.messages.push(message);
+  }
+
+  reset() {
+    this.messages = [];
+  }
+}
+
+export class SummaryChatHistory implements ChatHistory {
+  messages: ChatMessage[];
+  summaryPrompt: SummaryPrompt;
+  llm: LLM;
+
+  constructor(init?: Partial<SummaryChatHistory>) {
+    this.messages = init?.messages ?? [];
+    this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
+    this.llm = init?.llm ?? new OpenAI();
+  }
+
+  private async summarize() {
+    const chatHistoryStr = messagesToHistoryStr(this.messages);
+
+    const response = await this.llm.complete(
+      this.summaryPrompt({ context: chatHistoryStr }),
+    );
+
+    this.messages = [{ content: response.message.content, role: "system" }];
+  }
+
+  async addMessage(message: ChatMessage) {
+    // TODO: check if summarization is necessary
+    // TBD what are good conditions, e.g. depending on the context length of the LLM?
+    // for now we just have a dummy implementation at always summarizes the messages
+    await this.summarize();
+    this.messages.push(message);
+  }
+
+  reset() {
+    this.messages = [];
+  }
+}

packages/core/src/Embedding.ts

@@ -8,7 +8,7 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./llm/azure";
-import { getOpenAISession, OpenAISession } from "./llm/openai";
+import { OpenAISession, getOpenAISession } from "./llm/openai";
 import { VectorStoreQueryMode } from "./storage/vectorStore/types";
 
 /**
@@ -280,9 +280,6 @@ export class OpenAIEmbedding extends BaseEmbedding {
   }
 
   private async getOpenAIEmbedding(input: string) {
-    input = input.replace(/\n/g, " ");
-    //^ NOTE this performance helper is in the OpenAI python library but may not be in the JS library
-
     const { data } = await this.session.openai.embeddings.create({
       model: this.model,
       input,

packages/core/src/GlobalsHelper.ts

@@ -1,5 +1,5 @@
-import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
 import { v4 as uuidv4 } from "uuid";
+import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
 
 /**
  * Helper class singleton

packages/core/src/Node.ts

@@ -23,19 +23,23 @@ export enum MetadataMode {
   NONE = "NONE",
 }
 
-export interface RelatedNodeInfo {
+export type Metadata = Record<string, any>;
+
+export interface RelatedNodeInfo<T extends Metadata = Metadata> {
   nodeId: string;
   nodeType?: ObjectType;
-  metadata: Record<string, any>;
+  metadata: T;
   hash?: string;
 }
 
-export type RelatedNodeType = RelatedNodeInfo | RelatedNodeInfo[];
+export type RelatedNodeType<T extends Metadata = Metadata> =
+  | RelatedNodeInfo<T>
+  | RelatedNodeInfo<T>[];
 
 /**
  * Generic abstract class for retrievable nodes
  */
-export abstract class BaseNode {
+export abstract class BaseNode<T extends Metadata = Metadata> {
   /**
    * The unique ID of the Node/Document. The trailing underscore is here
    * to avoid collisions with the id keyword in Python.
@@ -46,13 +50,13 @@ export abstract class BaseNode {
   embedding?: number[];
 
   // Metadata fields
-  metadata: Record<string, any> = {};
+  metadata: T = {} as T;
   excludedEmbedMetadataKeys: string[] = [];
   excludedLlmMetadataKeys: string[] = [];
-  relationships: Partial<Record<NodeRelationship, RelatedNodeType>> = {};
+  relationships: Partial<Record<NodeRelationship, RelatedNodeType<T>>> = {};
   hash: string = "";
 
-  constructor(init?: Partial<BaseNode>) {
+  constructor(init?: Partial<BaseNode<T>>) {
     Object.assign(this, init);
   }
 
@@ -62,7 +66,7 @@ export abstract class BaseNode {
   abstract getMetadataStr(metadataMode: MetadataMode): string;
   abstract setContent(value: any): void;
 
-  get sourceNode(): RelatedNodeInfo | undefined {
+  get sourceNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.SOURCE];
 
     if (Array.isArray(relationship)) {
@@ -72,7 +76,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get prevNode(): RelatedNodeInfo | undefined {
+  get prevNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.PREVIOUS];
 
     if (Array.isArray(relationship)) {
@@ -84,7 +88,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get nextNode(): RelatedNodeInfo | undefined {
+  get nextNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.NEXT];
 
     if (Array.isArray(relationship)) {
@@ -94,7 +98,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get parentNode(): RelatedNodeInfo | undefined {
+  get parentNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.PARENT];
 
     if (Array.isArray(relationship)) {
@@ -104,7 +108,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get childNodes(): RelatedNodeInfo[] | undefined {
+  get childNodes(): RelatedNodeInfo<T>[] | undefined {
     const relationship = this.relationships[NodeRelationship.CHILD];
 
     if (!Array.isArray(relationship)) {
@@ -126,7 +130,7 @@ export abstract class BaseNode {
     return this.embedding;
   }
 
-  asRelatedNodeInfo(): RelatedNodeInfo {
+  asRelatedNodeInfo(): RelatedNodeInfo<T> {
     return {
       nodeId: this.id_,
       metadata: this.metadata,
@@ -146,7 +150,7 @@ export abstract class BaseNode {
 /**
  * TextNode is the default node type for text. Most common node type in LlamaIndex.TS
  */
-export class TextNode extends BaseNode {
+export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
   text: string = "";
   startCharIdx?: number;
   endCharIdx?: number;
@@ -154,7 +158,7 @@ export class TextNode extends BaseNode {
   // metadataTemplate: NOTE write your own formatter if needed
   metadataSeparator: string = "\n";
 
-  constructor(init?: Partial<TextNode>) {
+  constructor(init?: Partial<TextNode<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -233,10 +237,10 @@ export class TextNode extends BaseNode {
 //   }
 // }
 
-export class IndexNode extends TextNode {
+export class IndexNode<T extends Metadata = Metadata> extends TextNode<T> {
   indexId: string = "";
 
-  constructor(init?: Partial<IndexNode>) {
+  constructor(init?: Partial<IndexNode<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -253,8 +257,8 @@ export class IndexNode extends TextNode {
 /**
  * A document is just a special text node with a docId.
  */
-export class Document extends TextNode {
-  constructor(init?: Partial<Document>) {
+export class Document<T extends Metadata = Metadata> extends TextNode<T> {
+  constructor(init?: Partial<Document<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -292,7 +296,7 @@ export function jsonToNode(json: any) {
 /**
  * A node with a similarity score
  */
-export interface NodeWithScore {
-  node: BaseNode;
-  score: number;
+export interface NodeWithScore<T extends Metadata = Metadata> {
+  node: BaseNode<T>;
+  score?: number;
 }

packages/core/src/NodeParser.ts

@@ -10,7 +10,7 @@ import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants";
  */
 export function getTextSplitsFromDocument(
   document: Document,
-  textSplitter: SentenceSplitter
+  textSplitter: SentenceSplitter,
 ) {
   const text = document.getText();
   const splits = textSplitter.splitText(text);
@@ -30,7 +30,7 @@ export function getNodesFromDocument(
   document: Document,
   textSplitter: SentenceSplitter,
   includeMetadata: boolean = true,
-  includePrevNextRel: boolean = true
+  includePrevNextRel: boolean = true,
 ) {
   let nodes: TextNode[] = [];
 
@@ -100,10 +100,10 @@ export class SimpleNodeParser implements NodeParser {
   }) {
     this.textSplitter =
       init?.textSplitter ??
-      new SentenceSplitter(
-        init?.chunkSize ?? DEFAULT_CHUNK_SIZE,
-        init?.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP
-      );
+      new SentenceSplitter({
+        chunkSize: init?.chunkSize ?? DEFAULT_CHUNK_SIZE,
+        chunkOverlap: init?.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP,
+      });
     this.includeMetadata = init?.includeMetadata ?? true;
     this.includePrevNextRel = init?.includePrevNextRel ?? true;
   }

packages/core/src/OutputParser.ts

@@ -28,7 +28,7 @@ class OutputParserError extends Error {
 
   constructor(
     message: string,
-    options: { cause?: Error; output?: string } = {}
+    options: { cause?: Error; output?: string } = {},
   ) {
     // @ts-ignore
     super(message, options); // https://github.com/tc39/proposal-error-cause
@@ -53,30 +53,31 @@ class OutputParserError extends Error {
  * @param text A markdown block with JSON
  * @returns parsed JSON object
  */
-function parseJsonMarkdown(text: string) {
+export function parseJsonMarkdown(text: string) {
   text = text.trim();
 
-  const beginDelimiter = "```json";
-  const endDelimiter = "```";
+  const left_square = text.indexOf("[");
+  const left_brace = text.indexOf("{");
 
-  const beginIndex = text.indexOf(beginDelimiter);
-  const endIndex = text.indexOf(
-    endDelimiter,
-    beginIndex + beginDelimiter.length
-  );
-  if (beginIndex === -1 || endIndex === -1) {
-    throw new OutputParserError("Not a json markdown", { output: text });
+  var left: number;
+  var right: number;
+  if (left_square < left_brace && left_square != -1) {
+    left = left_square;
+    right = text.lastIndexOf("]");
+  } else {
+    left = left_brace;
+    right = text.lastIndexOf("}");
   }
-
-  const jsonText = text.substring(beginIndex + beginDelimiter.length, endIndex);
-
+  const jsonText = text.substring(left, right + 1);
   try {
+    //Single JSON object case
+    if (left_square === -1) {
+      return [JSON.parse(jsonText)];
+    }
+    //Multiple JSON object case.
     return JSON.parse(jsonText);
   } catch (e) {
-    throw new OutputParserError("Not a valid json", {
-      cause: e as Error,
-      output: text,
-    });
+    throw new OutputParserError("Not a json markdown", { output: text });
   }
 }
 

packages/core/src/Prompt.ts

@@ -7,30 +7,35 @@ import { ToolMetadata } from "./Tool";
  * NOTE this is a different interface compared to LlamaIndex Python
  * NOTE 2: we default to empty string to make it easy to calculate prompt sizes
  */
-export type SimplePrompt = (input: Record<string, string>) => string;
+export type SimplePrompt = (
+  input: Record<string, string | undefined>,
+) => string;
 
 /*
 DEFAULT_TEXT_QA_PROMPT_TMPL = (
-    "Context information is below. \n"
+    "Context information is below.\n"
+    "---------------------\n"
+    "{context_str}\n"
     "---------------------\n"
-    "{context_str}"
-    "\n---------------------\n"
     "Given the context information and not prior knowledge, "
-    "answer the question: {query_str}\n"
+    "answer the query.\n"
+    "Query: {query_str}\n"
+    "Answer: "
 )
 */
 
-export const defaultTextQaPrompt: SimplePrompt = (input) => {
-  const { context = "", query = "" } = input;
-
+export const defaultTextQaPrompt = ({ context = "", query = "" }) => {
   return `Context information is below.
 ---------------------
 ${context}
 ---------------------
-Given the context information and not prior knowledge, answer the question: ${query}
-`;
+Given the context information and not prior knowledge, answer the query.
+Query: ${query}
+Answer:`;
 };
 
+export type TextQaPrompt = typeof defaultTextQaPrompt;
+
 /*
 DEFAULT_SUMMARY_PROMPT_TMPL = (
     "Write a summary of the following. Try to use only the "
@@ -45,9 +50,7 @@ DEFAULT_SUMMARY_PROMPT_TMPL = (
 )
 */
 
-export const defaultSummaryPrompt: SimplePrompt = (input) => {
-  const { context = "" } = input;
-
+export const defaultSummaryPrompt = ({ context = "" }) => {
   return `Write a summary of the following. Try to use only the information provided. Try to include as many key details as possible.
 
 
@@ -58,9 +61,11 @@ SUMMARY:"""
 `;
 };
 
+export type SummaryPrompt = typeof defaultSummaryPrompt;
+
 /*
 DEFAULT_REFINE_PROMPT_TMPL = (
-    "The original question is as follows: {query_str}\n"
+    "The original query is as follows: {query_str}\n"
     "We have provided an existing answer: {existing_answer}\n"
     "We have the opportunity to refine the existing answer "
     "(only if needed) with some more context below.\n"
@@ -68,26 +73,55 @@ DEFAULT_REFINE_PROMPT_TMPL = (
     "{context_msg}\n"
     "------------\n"
     "Given the new context, refine the original answer to better "
-    "answer the question. "
-    "If the context isn't useful, return the original answer."
+    "answer the query. "
+    "If the context isn't useful, return the original answer.\n"
+    "Refined Answer: "
 )
 */
 
-export const defaultRefinePrompt: SimplePrompt = (input) => {
-  const { query = "", existingAnswer = "", context = "" } = input;
-
-  return `The original question is as follows: ${query}
+export const defaultRefinePrompt = ({
+  query = "",
+  existingAnswer = "",
+  context = "",
+}) => {
+  return `The original query is as follows: ${query}
 We have provided an existing answer: ${existingAnswer}
 We have the opportunity to refine the existing answer (only if needed) with some more context below.
 ------------
 ${context}
 ------------
-Given the new context, refine the original answer to better answer the question. If the context isn't useful, return the original answer.`;
+Given the new context, refine the original answer to better answer the query. If the context isn't useful, return the original answer.
+Refined Answer:`;
 };
 
-export const defaultChoiceSelectPrompt: SimplePrompt = (input) => {
-  const { context = "", query = "" } = input;
+export type RefinePrompt = typeof defaultRefinePrompt;
 
+/*
+DEFAULT_TREE_SUMMARIZE_TMPL = (
+  "Context information from multiple sources is below.\n"
+  "---------------------\n"
+  "{context_str}\n"
+  "---------------------\n"
+  "Given the information from multiple sources and not prior knowledge, "
+  "answer the query.\n"
+  "Query: {query_str}\n"
+  "Answer: "
+)
+*/
+
+export const defaultTreeSummarizePrompt = ({ context = "", query = "" }) => {
+  return `Context information from multiple sources is below.
+---------------------
+${context}
+---------------------
+Given the information from multiple sources and not prior knowledge, answer the query.
+Query: ${query}
+Answer:`;
+};
+
+export type TreeSummarizePrompt = typeof defaultTreeSummarizePrompt;
+
+export const defaultChoiceSelectPrompt = ({ context = "", query = "" }) => {
   return `A list of documents is shown below. Each document has a number next to it along 
 with a summary of the document. A question is also provided.
 Respond with the numbers of the documents
@@ -119,6 +153,8 @@ Question: ${query}
 Answer:`;
 };
 
+export type ChoiceSelectPrompt = typeof defaultChoiceSelectPrompt;
+
 /*
 PREFIX = """\
 Given a user question, and a list of tools, output a list of relevant sub-questions \
@@ -236,9 +272,7 @@ const exampleOutput: SubQuestion[] = [
   },
 ];
 
-export const defaultSubQuestionPrompt: SimplePrompt = (input) => {
-  const { toolsStr, queryStr } = input;
-
+export const defaultSubQuestionPrompt = ({ toolsStr = "", queryStr = "" }) => {
   return `Given a user question, and a list of tools, output a list of relevant sub-questions that when composed can help answer the full user question:
 
 # Example 1
@@ -268,6 +302,8 @@ ${queryStr}
 `;
 };
 
+export type SubQuestionPrompt = typeof defaultSubQuestionPrompt;
+
 // DEFAULT_TEMPLATE = """\
 // Given a conversation (between Human and Assistant) and a follow up message from Human, \
 // rewrite the message to be a standalone question that captures all relevant context \
@@ -282,9 +318,10 @@ ${queryStr}
 // <Standalone question>
 // """
 
-export const defaultCondenseQuestionPrompt: SimplePrompt = (input) => {
-  const { chatHistory, question } = input;
-
+export const defaultCondenseQuestionPrompt = ({
+  chatHistory = "",
+  question = "",
+}) => {
   return `Given a conversation (between Human and Assistant) and a follow up message from Human, rewrite the message to be a standalone question that captures all relevant context from the conversation.
 
 <Chat History>
@@ -297,6 +334,8 @@ ${question}
 `;
 };
 
+export type CondenseQuestionPrompt = typeof defaultCondenseQuestionPrompt;
+
 export function messagesToHistoryStr(messages: ChatMessage[]) {
   return messages.reduce((acc, message) => {
     acc += acc ? "\n" : "";
@@ -309,11 +348,42 @@ export function messagesToHistoryStr(messages: ChatMessage[]) {
   }, "");
 }
 
-export const contextSystemPrompt: SimplePrompt = (input) => {
-  const { context } = input;
-
+export const defaultContextSystemPrompt = ({ context = "" }) => {
   return `Context information is below.
 ---------------------
 ${context}
 ---------------------`;
 };
+
+export type ContextSystemPrompt = typeof defaultContextSystemPrompt;
+
+export const defaultKeywordExtractPrompt = ({
+  context = "",
+  maxKeywords = 10,
+}) => {
+  return `
+Some text is provided below. Given the text, extract up to ${maxKeywords} keywords from the text. Avoid stopwords.
+---------------------
+${context}
+---------------------
+Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'
+`;
+};
+
+export type KeywordExtractPrompt = typeof defaultKeywordExtractPrompt;
+
+export const defaultQueryKeywordExtractPrompt = ({
+  question = "",
+  maxKeywords = 10,
+}) => {
+  return `(
+  "A question is provided below. Given the question, extract up to ${maxKeywords} "
+  "keywords from the text. Focus on extracting the keywords that we can use "
+  "to best lookup answers to the question. Avoid stopwords."
+  "---------------------"
+  "${question}"
+  "---------------------"
+  "Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'"
+)`;
+};
+export type QueryKeywordExtractPrompt = typeof defaultQueryKeywordExtractPrompt;

packages/core/src/PromptHelper.ts

@@ -2,9 +2,9 @@ import { globalsHelper } from "./GlobalsHelper";
 import { SimplePrompt } from "./Prompt";
 import { SentenceSplitter } from "./TextSplitter";
 import {
+  DEFAULT_CHUNK_OVERLAP_RATIO,
   DEFAULT_CONTEXT_WINDOW,
   DEFAULT_NUM_OUTPUTS,
-  DEFAULT_CHUNK_OVERLAP_RATIO,
   DEFAULT_PADDING,
 } from "./constants";
 
@@ -43,7 +43,7 @@ export class PromptHelper {
     chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO,
     chunkSizeLimit?: number,
     tokenizer?: (text: string) => number[],
-    separator = " "
+    separator = " ",
   ) {
     this.contextWindow = contextWindow;
     this.numOutput = numOutput;
@@ -76,7 +76,7 @@ export class PromptHelper {
   private getAvailableChunkSize(
     prompt: SimplePrompt,
     numChunks = 1,
-    padding = 5
+    padding = 5,
   ) {
     const availableContextSize = this.getAvailableContextSize(prompt);
 
@@ -99,14 +99,14 @@ export class PromptHelper {
   getTextSplitterGivenPrompt(
     prompt: SimplePrompt,
     numChunks = 1,
-    padding = DEFAULT_PADDING
+    padding = DEFAULT_PADDING,
   ) {
     const chunkSize = this.getAvailableChunkSize(prompt, numChunks, padding);
     if (chunkSize === 0) {
       throw new Error("Got 0 as available chunk size");
     }
     const chunkOverlap = this.chunkOverlapRatio * chunkSize;
-    const textSplitter = new SentenceSplitter(chunkSize, chunkOverlap);
+    const textSplitter = new SentenceSplitter({ chunkSize, chunkOverlap });
     return textSplitter;
   }
 
@@ -120,7 +120,7 @@ export class PromptHelper {
   repack(
     prompt: SimplePrompt,
     textChunks: string[],
-    padding = DEFAULT_PADDING
+    padding = DEFAULT_PADDING,
   ) {
     const textSplitter = this.getTextSplitterGivenPrompt(prompt, 1, padding);
     const combinedStr = textChunks.join("\n\n");

packages/core/src/QueryEngine.ts

@@ -1,3 +1,4 @@
+import { v4 as uuidv4 } from "uuid";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   BaseQuestionGenerator,
@@ -7,10 +8,9 @@ import {
 import { Response } from "./Response";
 import { CompactAndRefine, ResponseSynthesizer } from "./ResponseSynthesizer";
 import { BaseRetriever } from "./Retriever";
-import { v4 as uuidv4 } from "uuid";
-import { Event } from "./callbacks/CallbackManager";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { QueryEngineTool, ToolMetadata } from "./Tool";
+import { Event } from "./callbacks/CallbackManager";
 
 /**
  * A query engine is a question answerer that can use one or more steps.
@@ -33,7 +33,7 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
 
   constructor(
     retriever: BaseRetriever,
-    responseSynthesizer?: ResponseSynthesizer
+    responseSynthesizer?: ResponseSynthesizer,
   ) {
     this.retriever = retriever;
     const serviceContext: ServiceContext | undefined =
@@ -122,7 +122,7 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
     };
 
     const subQNodes = await Promise.all(
-      subQuestions.map((subQ) => this.querySubQ(subQ, subQueryParentEvent))
+      subQuestions.map((subQ) => this.querySubQ(subQ, subQueryParentEvent)),
     );
 
     const nodes = subQNodes
@@ -133,7 +133,7 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
 
   private async querySubQ(
     subQ: SubQuestion,
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<NodeWithScore | null> {
     try {
       const question = subQ.subQuestion;

packages/core/src/QuestionGenerator.ts

@@ -4,7 +4,7 @@ import {
   SubQuestionOutputParser,
 } from "./OutputParser";
 import {
-  SimplePrompt,
+  SubQuestionPrompt,
   buildToolsText,
   defaultSubQuestionPrompt,
 } from "./Prompt";
@@ -28,7 +28,7 @@ export interface BaseQuestionGenerator {
  */
 export class LLMQuestionGenerator implements BaseQuestionGenerator {
   llm: LLM;
-  prompt: SimplePrompt;
+  prompt: SubQuestionPrompt;
   outputParser: BaseOutputParser<StructuredOutput<SubQuestion[]>>;
 
   constructor(init?: Partial<LLMQuestionGenerator>) {
@@ -45,7 +45,7 @@ export class LLMQuestionGenerator implements BaseQuestionGenerator {
         this.prompt({
           toolsStr,
           queryStr,
-        })
+        }),
       )
     ).message.content;
 

packages/core/src/ResponseSynthesizer.ts

@@ -1,8 +1,12 @@
 import { MetadataMode, NodeWithScore } from "./Node";
 import {
+  RefinePrompt,
   SimplePrompt,
+  TextQaPrompt,
+  TreeSummarizePrompt,
   defaultRefinePrompt,
   defaultTextQaPrompt,
+  defaultTreeSummarizePrompt,
 } from "./Prompt";
 import { getBiggestPrompt } from "./PromptHelper";
 import { Response } from "./Response";
@@ -35,7 +39,7 @@ interface BaseResponseBuilder {
     query: string,
     textChunks: string[],
     parentEvent?: Event,
-    prevResponse?: string
+    prevResponse?: string,
   ): Promise<string>;
 }
 
@@ -54,7 +58,7 @@ export class SimpleResponseBuilder implements BaseResponseBuilder {
   async getResponse(
     query: string,
     textChunks: string[],
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<string> {
     const input = {
       query,
@@ -72,13 +76,13 @@ export class SimpleResponseBuilder implements BaseResponseBuilder {
  */
 export class Refine implements BaseResponseBuilder {
   serviceContext: ServiceContext;
-  textQATemplate: SimplePrompt;
-  refineTemplate: SimplePrompt;
+  textQATemplate: TextQaPrompt;
+  refineTemplate: RefinePrompt;
 
   constructor(
     serviceContext: ServiceContext,
-    textQATemplate?: SimplePrompt,
-    refineTemplate?: SimplePrompt
+    textQATemplate?: TextQaPrompt,
+    refineTemplate?: RefinePrompt,
   ) {
     this.serviceContext = serviceContext;
     this.textQATemplate = textQATemplate ?? defaultTextQaPrompt;
@@ -89,7 +93,7 @@ export class Refine implements BaseResponseBuilder {
     query: string,
     textChunks: string[],
     parentEvent?: Event,
-    prevResponse?: string
+    prevResponse?: string,
   ): Promise<string> {
     let response: string | undefined = undefined;
 
@@ -101,7 +105,7 @@ export class Refine implements BaseResponseBuilder {
           prevResponse,
           query,
           chunk,
-          parentEvent
+          parentEvent,
         );
       }
       prevResponse = response;
@@ -113,7 +117,7 @@ export class Refine implements BaseResponseBuilder {
   private async giveResponseSingle(
     queryStr: string,
     textChunk: string,
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<string> {
     const textQATemplate: SimplePrompt = (input) =>
       this.textQATemplate({ ...input, query: queryStr });
@@ -130,7 +134,7 @@ export class Refine implements BaseResponseBuilder {
             textQATemplate({
               context: chunk,
             }),
-            parentEvent
+            parentEvent,
           )
         ).message.content;
       } else {
@@ -138,7 +142,7 @@ export class Refine implements BaseResponseBuilder {
           response,
           queryStr,
           chunk,
-          parentEvent
+          parentEvent,
         );
       }
     }
@@ -150,7 +154,7 @@ export class Refine implements BaseResponseBuilder {
     response: string,
     queryStr: string,
     textChunk: string,
-    parentEvent?: Event
+    parentEvent?: Event,
   ) {
     const refineTemplate: SimplePrompt = (input) =>
       this.refineTemplate({ ...input, query: queryStr });
@@ -166,7 +170,7 @@ export class Refine implements BaseResponseBuilder {
             context: chunk,
             existingAnswer: response,
           }),
-          parentEvent
+          parentEvent,
         )
       ).message.content;
     }
@@ -182,7 +186,7 @@ export class CompactAndRefine extends Refine {
     query: string,
     textChunks: string[],
     parentEvent?: Event,
-    prevResponse?: string
+    prevResponse?: string,
   ): Promise<string> {
     const textQATemplate: SimplePrompt = (input) =>
       this.textQATemplate({ ...input, query: query });
@@ -192,13 +196,13 @@ export class CompactAndRefine extends Refine {
     const maxPrompt = getBiggestPrompt([textQATemplate, refineTemplate]);
     const newTexts = this.serviceContext.promptHelper.repack(
       maxPrompt,
-      textChunks
+      textChunks,
     );
     const response = super.getResponse(
       query,
       newTexts,
       parentEvent,
-      prevResponse
+      prevResponse,
     );
     return response;
   }
@@ -208,52 +212,54 @@ export class CompactAndRefine extends Refine {
  */
 export class TreeSummarize implements BaseResponseBuilder {
   serviceContext: ServiceContext;
+  summaryTemplate: TreeSummarizePrompt;
 
-  constructor(serviceContext: ServiceContext) {
+  constructor(
+    serviceContext: ServiceContext,
+    summaryTemplate?: TreeSummarizePrompt,
+  ) {
     this.serviceContext = serviceContext;
+    this.summaryTemplate = summaryTemplate ?? defaultTreeSummarizePrompt;
   }
 
   async getResponse(
     query: string,
     textChunks: string[],
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<string> {
-    const summaryTemplate: SimplePrompt = (input) =>
-      defaultTextQaPrompt({ ...input, query: query });
-
     if (!textChunks || textChunks.length === 0) {
       throw new Error("Must have at least one text chunk");
     }
 
     const packedTextChunks = this.serviceContext.promptHelper.repack(
-      summaryTemplate,
-      textChunks
+      this.summaryTemplate,
+      textChunks,
     );
 
     if (packedTextChunks.length === 1) {
       return (
         await this.serviceContext.llm.complete(
-          summaryTemplate({
+          this.summaryTemplate({
             context: packedTextChunks[0],
           }),
-          parentEvent
+          parentEvent,
         )
       ).message.content;
     } else {
       const summaries = await Promise.all(
         packedTextChunks.map((chunk) =>
           this.serviceContext.llm.complete(
-            summaryTemplate({
+            this.summaryTemplate({
               context: chunk,
             }),
-            parentEvent
-          )
-        )
+            parentEvent,
+          ),
+        ),
       );
 
       return this.getResponse(
         query,
-        summaries.map((s) => s.message.content)
+        summaries.map((s) => s.message.content),
       );
     }
   }
@@ -261,7 +267,7 @@ export class TreeSummarize implements BaseResponseBuilder {
 
 export function getResponseBuilder(
   serviceContext: ServiceContext,
-  responseMode?: ResponseMode
+  responseMode?: ResponseMode,
 ): BaseResponseBuilder {
   switch (responseMode) {
     case ResponseMode.SIMPLE:
@@ -281,31 +287,39 @@ export function getResponseBuilder(
 export class ResponseSynthesizer {
   responseBuilder: BaseResponseBuilder;
   serviceContext: ServiceContext;
+  metadataMode: MetadataMode;
 
   constructor({
     responseBuilder,
     serviceContext,
+    metadataMode = MetadataMode.NONE,
   }: {
     responseBuilder?: BaseResponseBuilder;
     serviceContext?: ServiceContext;
+    metadataMode?: MetadataMode;
   } = {}) {
     this.serviceContext = serviceContext ?? serviceContextFromDefaults();
     this.responseBuilder =
       responseBuilder ?? getResponseBuilder(this.serviceContext);
+    this.metadataMode = metadataMode;
   }
 
-  async synthesize(query: string, nodes: NodeWithScore[], parentEvent?: Event) {
-    let textChunks: string[] = nodes.map((node) =>
-      node.node.getContent(MetadataMode.NONE)
+  async synthesize(
+    query: string,
+    nodesWithScore: NodeWithScore[],
+    parentEvent?: Event,
+  ) {
+    let textChunks: string[] = nodesWithScore.map(({ node }) =>
+      node.getContent(this.metadataMode),
     );
     const response = await this.responseBuilder.getResponse(
       query,
       textChunks,
-      parentEvent
+      parentEvent,
     );
     return new Response(
       response,
-      nodes.map((node) => node.node)
+      nodesWithScore.map(({ node }) => node),
     );
   }
 }

packages/core/src/ServiceContext.ts

@@ -1,8 +1,8 @@
 import { BaseEmbedding, OpenAIEmbedding } from "./Embedding";
-import { LLM, OpenAI } from "./llm/LLM";
 import { NodeParser, SimpleNodeParser } from "./NodeParser";
 import { PromptHelper } from "./PromptHelper";
 import { CallbackManager } from "./callbacks/CallbackManager";
+import { LLM, OpenAI } from "./llm/LLM";
 
 /**
  * The ServiceContext is a collection of components that are used in different parts of the application.
@@ -47,7 +47,7 @@ export function serviceContextFromDefaults(options?: ServiceContextOptions) {
 
 export function serviceContextFromServiceContext(
   serviceContext: ServiceContext,
-  options: ServiceContextOptions
+  options: ServiceContextOptions,
 ) {
   const newServiceContext = { ...serviceContext };
   if (options.llm) {

packages/core/src/TextSplitter.ts

@@ -1,7 +1,7 @@
 // GitHub translated
 
 import { globalsHelper } from "./GlobalsHelper";
-import { DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_OVERLAP } from "./constants";
+import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants";
 
 class TextSplit {
   textChunk: string;
@@ -9,7 +9,7 @@ class TextSplit {
 
   constructor(
     textChunk: string,
-    numCharOverlap: number | undefined = undefined
+    numCharOverlap: number | undefined = undefined,
   ) {
     this.textChunk = textChunk;
     this.numCharOverlap = numCharOverlap;
@@ -18,6 +18,38 @@ class TextSplit {
 
 type SplitRep = { text: string; numTokens: number };
 
+/**
+ * Tokenizes sentences. Suitable for English and most European languages.
+ * @param text
+ * @returns
+ */
+export const englishSentenceTokenizer = (text: string) => {
+  // The first part is a lazy match for any character.
+  return text.match(/.+?[.?!]+[\])'"`’”]*(?:\s|$)|.+/g);
+};
+
+/**
+ * Tokenizes sentences. Suitable for Chinese, Japanese, and Korean.
+ * @param text
+ * @returns
+ */
+export const cjkSentenceTokenizer = (text: string) => {
+  // Accepts english style sentence endings with space and
+  // CJK style sentence endings with no space.
+  return text.match(
+    /.+?[.?!]+[\])'"`’”]*(?:\s|$)|.+?[。？！]+[\])'"`’”]*(?:\s|$)?|.+/g,
+  );
+};
+
+export const unixLineSeparator = "\n";
+export const windowsLineSeparator = "\r\n";
+export const unixParagraphSeparator = unixLineSeparator + unixLineSeparator;
+export const windowsParagraphSeparator =
+  windowsLineSeparator + windowsLineSeparator;
+
+// In theory there's also Mac style \r only, but it's pre-OSX and I don't think
+// many documents will use it.
+
 /**
  * SentenceSplitter is our default text splitter that supports splitting into sentences, paragraphs, or fixed length chunks with overlap.
  *
@@ -29,46 +61,44 @@ export class SentenceSplitter {
   private tokenizer: any;
   private tokenizerDecoder: any;
   private paragraphSeparator: string;
-  private chunkingTokenizerFn: any;
-  // private _callback_manager: any;
+  private chunkingTokenizerFn: (text: string) => RegExpMatchArray | null;
+  private splitLongSentences: boolean;
+
+  constructor(options?: {
+    chunkSize?: number;
+    chunkOverlap?: number;
+    tokenizer?: any;
+    tokenizerDecoder?: any;
+    paragraphSeparator?: string;
+    chunkingTokenizerFn?: (text: string) => RegExpMatchArray | null;
+    splitLongSentences?: boolean;
+  }) {
+    const {
+      chunkSize = DEFAULT_CHUNK_SIZE,
+      chunkOverlap = DEFAULT_CHUNK_OVERLAP,
+      tokenizer = null,
+      tokenizerDecoder = null,
+      paragraphSeparator = unixParagraphSeparator,
+      chunkingTokenizerFn = undefined,
+      splitLongSentences = false,
+    } = options ?? {};
 
-  constructor(
-    chunkSize: number = DEFAULT_CHUNK_SIZE,
-    chunkOverlap: number = DEFAULT_CHUNK_OVERLAP,
-    tokenizer: any = null,
-    tokenizerDecoder: any = null,
-    paragraphSeparator: string = "\n\n\n",
-    chunkingTokenizerFn: any = undefined
-    // callback_manager: any = undefined
-  ) {
     if (chunkOverlap > chunkSize) {
       throw new Error(
-        `Got a larger chunk overlap (${chunkOverlap}) than chunk size (${chunkSize}), should be smaller.`
+        `Got a larger chunk overlap (${chunkOverlap}) than chunk size (${chunkSize}), should be smaller.`,
       );
     }
     this.chunkSize = chunkSize;
     this.chunkOverlap = chunkOverlap;
     // this._callback_manager = callback_manager || new CallbackManager([]);
 
-    if (chunkingTokenizerFn == undefined) {
-      // define a callable mapping a string to a list of strings
-      const defaultChunkingTokenizerFn = (text: string) => {
-        var result = text.match(/[^.?!]+[.!?]+[\])'"`’”]*|.+/g);
-        return result;
-      };
-
-      chunkingTokenizerFn = defaultChunkingTokenizerFn;
-    }
-
-    if (tokenizer == undefined || tokenizerDecoder == undefined) {
-      tokenizer = globalsHelper.tokenizer();
-      tokenizerDecoder = globalsHelper.tokenizerDecoder();
-    }
-    this.tokenizer = tokenizer;
-    this.tokenizerDecoder = tokenizerDecoder;
+    this.tokenizer = tokenizer ?? globalsHelper.tokenizer();
+    this.tokenizerDecoder =
+      tokenizerDecoder ?? globalsHelper.tokenizerDecoder();
 
     this.paragraphSeparator = paragraphSeparator;
-    this.chunkingTokenizerFn = chunkingTokenizerFn;
+    this.chunkingTokenizerFn = chunkingTokenizerFn ?? englishSentenceTokenizer;
+    this.splitLongSentences = splitLongSentences;
   }
 
   private getEffectiveChunkSize(extraInfoStr?: string): number {
@@ -79,7 +109,7 @@ export class SentenceSplitter {
       effectiveChunkSize = this.chunkSize - numExtraTokens;
       if (effectiveChunkSize <= 0) {
         throw new Error(
-          "Effective chunk size is non positive after considering extra_info"
+          "Effective chunk size is non positive after considering extra_info",
         );
       }
     } else {
@@ -119,7 +149,12 @@ export class SentenceSplitter {
     // Next we split the text using the chunk tokenizer fn/
     let splits = [];
     for (const parText of paragraphSplits) {
-      let sentenceSplits = this.chunkingTokenizerFn(parText);
+      const sentenceSplits = this.chunkingTokenizerFn(parText);
+
+      if (!sentenceSplits) {
+        continue;
+      }
+
       for (const sentence_split of sentenceSplits) {
         splits.push(sentence_split.trim());
       }
@@ -127,13 +162,28 @@ export class SentenceSplitter {
     return splits;
   }
 
+  /**
+   * Splits sentences into chunks if necessary.
+   *
+   * This isn't great behavior because it can split down the middle of a
+   * word or in non-English split down the middle of a Unicode codepoint
+   * so the splitting is turned off by default. If you need it, please
+   * set the splitLongSentences option to true.
+   * @param sentenceSplits
+   * @param effectiveChunkSize
+   * @returns
+   */
   private processSentenceSplits(
     sentenceSplits: string[],
-    effectiveChunkSize: number
+    effectiveChunkSize: number,
   ): SplitRep[] {
-    // Process sentence splits
-    // Primarily check if any sentences exceed the chunk size. If they don't,
-    // force split by tokenizer
+    if (!this.splitLongSentences) {
+      return sentenceSplits.map((split) => ({
+        text: split,
+        numTokens: this.tokenizer(split).length,
+      }));
+    }
+
     let newSplits: SplitRep[] = [];
     for (const split of sentenceSplits) {
       let splitTokens = this.tokenizer(split);
@@ -143,7 +193,7 @@ export class SentenceSplitter {
       } else {
         for (let i = 0; i < splitLen; i += effectiveChunkSize) {
           const cur_split = this.tokenizerDecoder(
-            splitTokens.slice(i, i + effectiveChunkSize)
+            splitTokens.slice(i, i + effectiveChunkSize),
           );
           newSplits.push({ text: cur_split, numTokens: effectiveChunkSize });
         }
@@ -154,7 +204,7 @@ export class SentenceSplitter {
 
   combineTextSplits(
     newSentenceSplits: SplitRep[],
-    effectiveChunkSize: number
+    effectiveChunkSize: number,
   ): TextSplit[] {
     // go through sentence splits, combine to chunks that are within the chunk size
 
@@ -178,8 +228,8 @@ export class SentenceSplitter {
             curChunkSentences
               .map((sentence) => sentence.text)
               .join(" ")
-              .trim()
-          )
+              .trim(),
+          ),
         );
 
         const lastChunkSentences = curChunkSentences;
@@ -210,8 +260,8 @@ export class SentenceSplitter {
         curChunkSentences
           .map((sentence) => sentence.text)
           .join(" ")
-          .trim()
-      )
+          .trim(),
+      ),
     );
     return docs;
   }
@@ -232,13 +282,13 @@ export class SentenceSplitter {
     // force split by tokenizer
     let newSentenceSplits = this.processSentenceSplits(
       sentenceSplits,
-      effectiveChunkSize
+      effectiveChunkSize,
     );
 
     // combine sentence splits into chunks of text that can then be returned
     let combinedTextSplits = this.combineTextSplits(
       newSentenceSplits,
-      effectiveChunkSize
+      effectiveChunkSize,
     );
 
     return combinedTextSplits;

packages/core/src/index.ts

@@ -1,28 +1,30 @@
 export * from "./ChatEngine";
-export * from "./constants";
 export * from "./Embedding";
 export * from "./GlobalsHelper";
-export * from "./llm/LLM";
 export * from "./Node";
 export * from "./NodeParser";
 export * from "./OutputParser";
 export * from "./Prompt";
-export * from "./QuestionGenerator";
 export * from "./QueryEngine";
+export * from "./QuestionGenerator";
 export * from "./Response";
 export * from "./ResponseSynthesizer";
 export * from "./Retriever";
 export * from "./ServiceContext";
 export * from "./TextSplitter";
 export * from "./Tool";
+export * from "./constants";
+export * from "./llm/LLM";
 
 export * from "./indices";
 
 export * from "./callbacks/CallbackManager";
 
-export * from "./readers/base";
-export * from "./readers/PDFReader";
 export * from "./readers/CSVReader";
+export * from "./readers/MarkdownReader";
+export * from "./readers/NotionReader";
+export * from "./readers/PDFReader";
 export * from "./readers/SimpleDirectoryReader";
+export * from "./readers/base";
 
 export * from "./storage";

packages/core/src/indices/BaseIndex.ts

@@ -4,9 +4,9 @@ import { BaseQueryEngine } from "../QueryEngine";
 import { ResponseSynthesizer } from "../ResponseSynthesizer";
 import { BaseRetriever } from "../Retriever";
 import { ServiceContext } from "../ServiceContext";
+import { StorageContext } from "../storage/StorageContext";
 import { BaseDocumentStore } from "../storage/docStore/types";
 import { BaseIndexStore } from "../storage/indexStore/types";
-import { StorageContext } from "../storage/StorageContext";
 import { VectorStore } from "../storage/vectorStore/types";
 
 /**
@@ -39,6 +39,7 @@ export abstract class IndexStruct {
 export enum IndexStructType {
   SIMPLE_DICT = "simple_dict",
   LIST = "list",
+  KEYWORD_TABLE = "keyword_table",
 }
 
 export class IndexDict extends IndexStruct {
@@ -106,6 +107,36 @@ export class IndexList extends IndexStruct {
   }
 }
 
+// A table of keywords mapping keywords to text chunks.
+export class KeywordTable extends IndexStruct {
+  table: Map<string, Set<string>> = new Map();
+  type: IndexStructType = IndexStructType.KEYWORD_TABLE;
+  addNode(keywords: string[], nodeId: string): void {
+    keywords.forEach((keyword) => {
+      if (!this.table.has(keyword)) {
+        this.table.set(keyword, new Set());
+      }
+      this.table.get(keyword)!.add(nodeId);
+    });
+  }
+
+  deleteNode(keywords: string[], nodeId: string) {
+    keywords.forEach((keyword) => {
+      if (this.table.has(keyword)) {
+        this.table.get(keyword)!.delete(nodeId);
+      }
+    });
+  }
+
+  toJson(): Record<string, unknown> {
+    return {
+      ...super.toJson(),
+      table: this.table,
+      type: this.type,
+    };
+  }
+}
+
 export interface BaseIndexInit<T> {
   serviceContext: ServiceContext;
   storageContext: StorageContext;

packages/core/src/indices/index.ts

@@ -1,3 +1,4 @@
 export * from "./BaseIndex";
-export * from "./list";
+export * from "./keyword";
+export * from "./summary";
 export * from "./vectorStore";

packages/core/src/indices/keyword/KeywordTableIndex.ts

@@ -0,0 +1,269 @@
+import { BaseNode, Document, MetadataMode } from "../../Node";
+import { defaultKeywordExtractPrompt } from "../../Prompt";
+import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
+import { ResponseSynthesizer } from "../../ResponseSynthesizer";
+import { BaseRetriever } from "../../Retriever";
+import {
+  ServiceContext,
+  serviceContextFromDefaults,
+} from "../../ServiceContext";
+import { StorageContext, storageContextFromDefaults } from "../../storage";
+import { BaseDocumentStore } from "../../storage/docStore/types";
+import {
+  BaseIndex,
+  BaseIndexInit,
+  IndexStructType,
+  KeywordTable,
+} from "../BaseIndex";
+import {
+  KeywordTableLLMRetriever,
+  KeywordTableRAKERetriever,
+  KeywordTableSimpleRetriever,
+} from "./KeywordTableIndexRetriever";
+import { extractKeywordsGivenResponse } from "./utils";
+
+export interface KeywordIndexOptions {
+  nodes?: BaseNode[];
+  indexStruct?: KeywordTable;
+  indexId?: string;
+  serviceContext?: ServiceContext;
+  storageContext?: StorageContext;
+}
+export enum KeywordTableRetrieverMode {
+  DEFAULT = "DEFAULT",
+  SIMPLE = "SIMPLE",
+  RAKE = "RAKE",
+}
+
+const KeywordTableRetrieverMap = {
+  [KeywordTableRetrieverMode.DEFAULT]: KeywordTableLLMRetriever,
+  [KeywordTableRetrieverMode.SIMPLE]: KeywordTableSimpleRetriever,
+  [KeywordTableRetrieverMode.RAKE]: KeywordTableRAKERetriever,
+};
+
+/**
+ * The KeywordTableIndex, an index that extracts keywords from each Node and builds a mapping from each keyword to the corresponding Nodes of that keyword.
+ */
+export class KeywordTableIndex extends BaseIndex<KeywordTable> {
+  constructor(init: BaseIndexInit<KeywordTable>) {
+    super(init);
+  }
+
+  static async init(options: KeywordIndexOptions): Promise<KeywordTableIndex> {
+    const storageContext =
+      options.storageContext ?? (await storageContextFromDefaults({}));
+    const serviceContext =
+      options.serviceContext ?? serviceContextFromDefaults({});
+    const { docStore, indexStore } = storageContext;
+
+    // Setup IndexStruct from storage
+    let indexStructs = (await indexStore.getIndexStructs()) as KeywordTable[];
+    let indexStruct: KeywordTable | null;
+
+    if (options.indexStruct && indexStructs.length > 0) {
+      throw new Error(
+        "Cannot initialize index with both indexStruct and indexStore",
+      );
+    }
+
+    if (options.indexStruct) {
+      indexStruct = options.indexStruct;
+    } else if (indexStructs.length == 1) {
+      indexStruct = indexStructs[0];
+    } else if (indexStructs.length > 1 && options.indexId) {
+      indexStruct = (await indexStore.getIndexStruct(
+        options.indexId,
+      )) as KeywordTable;
+    } else {
+      indexStruct = null;
+    }
+
+    // check indexStruct type
+    if (indexStruct && indexStruct.type !== IndexStructType.KEYWORD_TABLE) {
+      throw new Error(
+        "Attempting to initialize KeywordTableIndex with non-keyword table indexStruct",
+      );
+    }
+
+    if (indexStruct) {
+      if (options.nodes) {
+        throw new Error(
+          "Cannot initialize KeywordTableIndex with both nodes and indexStruct",
+        );
+      }
+    } else {
+      if (!options.nodes) {
+        throw new Error(
+          "Cannot initialize KeywordTableIndex without nodes or indexStruct",
+        );
+      }
+      indexStruct = await KeywordTableIndex.buildIndexFromNodes(
+        options.nodes,
+        storageContext.docStore,
+        serviceContext,
+      );
+
+      await indexStore.addIndexStruct(indexStruct);
+    }
+
+    return new KeywordTableIndex({
+      storageContext,
+      serviceContext,
+      docStore,
+      indexStore,
+      indexStruct,
+    });
+  }
+
+  asRetriever(options?: any): BaseRetriever {
+    const { mode = KeywordTableRetrieverMode.DEFAULT, ...otherOptions } =
+      options ?? {};
+    const KeywordTableRetriever =
+      KeywordTableRetrieverMap[mode as KeywordTableRetrieverMode];
+    if (KeywordTableRetriever) {
+      return new KeywordTableRetriever({ index: this, ...otherOptions });
+    }
+    throw new Error(`Unknown retriever mode: ${mode}`);
+  }
+
+  asQueryEngine(options?: {
+    retriever?: BaseRetriever;
+    responseSynthesizer?: ResponseSynthesizer;
+  }): BaseQueryEngine {
+    const { retriever, responseSynthesizer } = options ?? {};
+    return new RetrieverQueryEngine(
+      retriever ?? this.asRetriever(),
+      responseSynthesizer,
+    );
+  }
+
+  static async extractKeywords(
+    text: string,
+    serviceContext: ServiceContext,
+  ): Promise<Set<string>> {
+    const response = await serviceContext.llm.complete(
+      defaultKeywordExtractPrompt({
+        context: text,
+      }),
+    );
+    return extractKeywordsGivenResponse(response.message.content, "KEYWORDS:");
+  }
+
+  /**
+   * High level API: split documents, get keywords, and build index.
+   * @param documents
+   * @param storageContext
+   * @param serviceContext
+   * @returns
+   */
+  static async fromDocuments(
+    documents: Document[],
+    args: {
+      storageContext?: StorageContext;
+      serviceContext?: ServiceContext;
+    } = {},
+  ): Promise<KeywordTableIndex> {
+    let { storageContext, serviceContext } = args;
+    storageContext = storageContext ?? (await storageContextFromDefaults({}));
+    serviceContext = serviceContext ?? serviceContextFromDefaults({});
+    const docStore = storageContext.docStore;
+
+    docStore.addDocuments(documents, true);
+    for (const doc of documents) {
+      docStore.setDocumentHash(doc.id_, doc.hash);
+    }
+
+    const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
+    const index = await KeywordTableIndex.init({
+      nodes,
+      storageContext,
+      serviceContext,
+    });
+    return index;
+  }
+
+  /**
+   * Get keywords for nodes and place them into the index.
+   * @param nodes
+   * @param serviceContext
+   * @param vectorStore
+   * @returns
+   */
+  static async buildIndexFromNodes(
+    nodes: BaseNode[],
+    docStore: BaseDocumentStore,
+    serviceContext: ServiceContext,
+  ): Promise<KeywordTable> {
+    const indexStruct = new KeywordTable();
+    await docStore.addDocuments(nodes, true);
+    for (const node of nodes) {
+      const keywords = await KeywordTableIndex.extractKeywords(
+        node.getContent(MetadataMode.LLM),
+        serviceContext,
+      );
+      indexStruct.addNode([...keywords], node.id_);
+    }
+    return indexStruct;
+  }
+
+  async insertNodes(nodes: BaseNode[]) {
+    for (let node of nodes) {
+      const keywords = await KeywordTableIndex.extractKeywords(
+        node.getContent(MetadataMode.LLM),
+        this.serviceContext,
+      );
+      this.indexStruct.addNode([...keywords], node.id_);
+    }
+  }
+
+  deleteNode(nodeId: string): void {
+    const keywordsToDelete: Set<string> = new Set();
+    for (const [keyword, existingNodeIds] of Object.entries(
+      this.indexStruct.table,
+    )) {
+      const index = existingNodeIds.indexOf(nodeId);
+      if (index !== -1) {
+        existingNodeIds.splice(index, 1);
+
+        // Delete keywords that have zero nodes
+        if (existingNodeIds.length === 0) {
+          keywordsToDelete.add(keyword);
+        }
+      }
+    }
+    this.indexStruct.deleteNode([...keywordsToDelete], nodeId);
+  }
+
+  async deleteNodes(nodeIds: string[], deleteFromDocStore: boolean) {
+    nodeIds.forEach((nodeId) => {
+      this.deleteNode(nodeId);
+    });
+
+    if (deleteFromDocStore) {
+      for (const nodeId of nodeIds) {
+        await this.docStore.deleteDocument(nodeId, false);
+      }
+    }
+
+    await this.storageContext.indexStore.addIndexStruct(this.indexStruct);
+  }
+
+  async deleteRefDoc(
+    refDocId: string,
+    deleteFromDocStore?: boolean,
+  ): Promise<void> {
+    const refDocInfo = await this.docStore.getRefDocInfo(refDocId);
+
+    if (!refDocInfo) {
+      return;
+    }
+
+    await this.deleteNodes(refDocInfo.nodeIds, false);
+
+    if (deleteFromDocStore) {
+      await this.docStore.deleteRefDoc(refDocId, false);
+    }
+
+    return;
+  }
+}

packages/core/src/indices/keyword/KeywordTableIndexRetriever.ts

@@ -0,0 +1,119 @@
+import { NodeWithScore } from "../../Node";
+import {
+  defaultKeywordExtractPrompt,
+  defaultQueryKeywordExtractPrompt,
+  KeywordExtractPrompt,
+  QueryKeywordExtractPrompt,
+} from "../../Prompt";
+import { BaseRetriever } from "../../Retriever";
+import { ServiceContext } from "../../ServiceContext";
+import { BaseDocumentStore } from "../../storage/docStore/types";
+import { KeywordTable } from "../BaseIndex";
+import { KeywordTableIndex } from "./KeywordTableIndex";
+import {
+  extractKeywordsGivenResponse,
+  rakeExtractKeywords,
+  simpleExtractKeywords,
+} from "./utils";
+
+// Base Keyword Table Retriever
+abstract class BaseKeywordTableRetriever implements BaseRetriever {
+  protected index: KeywordTableIndex;
+  protected indexStruct: KeywordTable;
+  protected docstore: BaseDocumentStore;
+  protected serviceContext: ServiceContext;
+
+  protected maxKeywordsPerQuery: number; // Maximum number of keywords to extract from query.
+  protected numChunksPerQuery: number; // Maximum number of text chunks to query.
+  protected keywordExtractTemplate: KeywordExtractPrompt; // A Keyword Extraction Prompt
+  protected queryKeywordExtractTemplate: QueryKeywordExtractPrompt; // A Query Keyword Extraction Prompt
+
+  constructor({
+    index,
+    keywordExtractTemplate,
+    queryKeywordExtractTemplate,
+    maxKeywordsPerQuery = 10,
+    numChunksPerQuery = 10,
+  }: {
+    index: KeywordTableIndex;
+    keywordExtractTemplate?: KeywordExtractPrompt;
+    queryKeywordExtractTemplate?: QueryKeywordExtractPrompt;
+    maxKeywordsPerQuery: number;
+    numChunksPerQuery: number;
+  }) {
+    this.index = index;
+    this.indexStruct = index.indexStruct;
+    this.docstore = index.docStore;
+    this.serviceContext = index.serviceContext;
+
+    this.maxKeywordsPerQuery = maxKeywordsPerQuery;
+    this.numChunksPerQuery = numChunksPerQuery;
+    this.keywordExtractTemplate =
+      keywordExtractTemplate || defaultKeywordExtractPrompt;
+    this.queryKeywordExtractTemplate =
+      queryKeywordExtractTemplate || defaultQueryKeywordExtractPrompt;
+  }
+
+  abstract getKeywords(query: string): Promise<string[]>;
+
+  async retrieve(query: string): Promise<NodeWithScore[]> {
+    const keywords = await this.getKeywords(query);
+    const chunkIndicesCount: { [key: string]: number } = {};
+    const filteredKeywords = keywords.filter((keyword) =>
+      this.indexStruct.table.has(keyword),
+    );
+
+    for (let keyword of filteredKeywords) {
+      for (let nodeId of this.indexStruct.table.get(keyword) || []) {
+        chunkIndicesCount[nodeId] = (chunkIndicesCount[nodeId] ?? 0) + 1;
+      }
+    }
+
+    const sortedChunkIndices = Object.keys(chunkIndicesCount)
+      .sort((a, b) => chunkIndicesCount[b] - chunkIndicesCount[a])
+      .slice(0, this.numChunksPerQuery);
+
+    const sortedNodes = await this.docstore.getNodes(sortedChunkIndices);
+
+    return sortedNodes.map((node) => ({ node }));
+  }
+
+  getServiceContext(): ServiceContext {
+    return this.index.serviceContext;
+  }
+}
+
+// Extracts keywords using LLMs.
+export class KeywordTableLLMRetriever extends BaseKeywordTableRetriever {
+  async getKeywords(query: string): Promise<string[]> {
+    const response = await this.serviceContext.llm.complete(
+      this.queryKeywordExtractTemplate({
+        question: query,
+        maxKeywords: this.maxKeywordsPerQuery,
+      }),
+    );
+    const keywords = extractKeywordsGivenResponse(
+      response.message.content,
+      "KEYWORDS:",
+    );
+    return [...keywords];
+  }
+}
+
+// Extracts keywords using simple regex-based keyword extractor.
+export class KeywordTableSimpleRetriever extends BaseKeywordTableRetriever {
+  getKeywords(query: string): Promise<string[]> {
+    return Promise.resolve([
+      ...simpleExtractKeywords(query, this.maxKeywordsPerQuery),
+    ]);
+  }
+}
+
+// Extracts keywords using RAKE keyword extractor
+export class KeywordTableRAKERetriever extends BaseKeywordTableRetriever {
+  getKeywords(query: string): Promise<string[]> {
+    return Promise.resolve([
+      ...rakeExtractKeywords(query, this.maxKeywordsPerQuery),
+    ]);
+  }
+}

packages/core/src/indices/keyword/index.ts

@@ -0,0 +1,9 @@
+export {
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "./KeywordTableIndex";
+export {
+  KeywordTableLLMRetriever,
+  KeywordTableRAKERetriever,
+  KeywordTableSimpleRetriever,
+} from "./KeywordTableIndexRetriever";

packages/core/src/indices/keyword/utils.ts

@@ -0,0 +1,81 @@
+// @ts-ignore
+import rake from "rake-modified";
+
+// Get subtokens from a list of tokens., filtering for stopwords.
+export function expandTokensWithSubtokens(tokens: Set<string>): Set<string> {
+  const results: Set<string> = new Set();
+  const regex: RegExp = /\w+/g;
+
+  for (let token of tokens) {
+    results.add(token);
+    const subTokens: RegExpMatchArray | null = token.match(regex);
+    if (subTokens && subTokens.length > 1) {
+      for (let w of subTokens) {
+        results.add(w);
+      }
+    }
+  }
+  return results;
+}
+
+export function extractKeywordsGivenResponse(
+  response: string,
+  startToken: string = "",
+  lowercase: boolean = true,
+): Set<string> {
+  const results: string[] = [];
+  response = response.trim();
+
+  if (response.startsWith(startToken)) {
+    response = response.substring(startToken.length);
+  }
+
+  const keywords: string[] = response.split(",");
+  for (let k of keywords) {
+    let rk: string = k;
+    if (lowercase) {
+      rk = rk.toLowerCase();
+    }
+    results.push(rk.trim());
+  }
+
+  return expandTokensWithSubtokens(new Set(results));
+}
+
+export function simpleExtractKeywords(
+  textChunk: string,
+  maxKeywords?: number,
+): Set<string> {
+  const regex: RegExp = /\w+/g;
+  let tokens: string[] = [...textChunk.matchAll(regex)].map((token) =>
+    token[0].toLowerCase().trim(),
+  );
+
+  // Creating a frequency map
+  const valueCounts: { [key: string]: number } = {};
+  for (let token of tokens) {
+    valueCounts[token] = (valueCounts[token] || 0) + 1;
+  }
+
+  // Sorting tokens by frequency
+  const sortedTokens: string[] = Object.keys(valueCounts).sort(
+    (a, b) => valueCounts[b] - valueCounts[a],
+  );
+
+  const keywords: string[] = maxKeywords
+    ? sortedTokens.slice(0, maxKeywords)
+    : sortedTokens;
+
+  return new Set(keywords);
+}
+
+export function rakeExtractKeywords(
+  textChunk: string,
+  maxKeywords?: number,
+): Set<string> {
+  const keywords = Object.keys(rake(textChunk));
+  const limitedKeywords = maxKeywords
+    ? keywords.slice(0, maxKeywords)
+    : keywords;
+  return new Set(limitedKeywords);
+}

packages/core/src/indices/list/index.ts

@@ -1,5 +0,0 @@
-export { ListIndex, ListRetrieverMode } from "./ListIndex";
-export {
-  ListIndexRetriever,
-  ListIndexLLMRetriever,
-} from "./ListIndexRetriever";

packages/core/src/indices/summary/SummaryIndex.ts

@@ -10,11 +10,11 @@ import {
   ServiceContext,
   serviceContextFromDefaults,
 } from "../../ServiceContext";
-import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   StorageContext,
   storageContextFromDefaults,
 } from "../../storage/StorageContext";
+import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   BaseIndex,
   BaseIndexInit,
@@ -22,17 +22,17 @@ import {
   IndexStructType,
 } from "../BaseIndex";
 import {
-  ListIndexLLMRetriever,
-  ListIndexRetriever,
-} from "./ListIndexRetriever";
+  SummaryIndexLLMRetriever,
+  SummaryIndexRetriever,
+} from "./SummaryIndexRetriever";
 
-export enum ListRetrieverMode {
+export enum SummaryRetrieverMode {
   DEFAULT = "default",
   // EMBEDDING = "embedding",
   LLM = "llm",
 }
 
-export interface ListIndexOptions {
+export interface SummaryIndexOptions {
   nodes?: BaseNode[];
   indexStruct?: IndexList;
   indexId?: string;
@@ -41,14 +41,14 @@ export interface ListIndexOptions {
 }
 
 /**
- * A ListIndex keeps nodes in a sequential list structure
+ * A SummaryIndex keeps nodes in a sequential order for use with summarization.
  */
-export class ListIndex extends BaseIndex<IndexList> {
+export class SummaryIndex extends BaseIndex<IndexList> {
   constructor(init: BaseIndexInit<IndexList>) {
     super(init);
   }
 
-  static async init(options: ListIndexOptions): Promise<ListIndex> {
+  static async init(options: SummaryIndexOptions): Promise<SummaryIndex> {
     const storageContext =
       options.storageContext ?? (await storageContextFromDefaults({}));
     const serviceContext =
@@ -80,23 +80,23 @@ export class ListIndex extends BaseIndex<IndexList> {
     // check indexStruct type
     if (indexStruct && indexStruct.type !== IndexStructType.LIST) {
       throw new Error(
-        "Attempting to initialize ListIndex with non-list indexStruct",
+        "Attempting to initialize SummaryIndex with non-list indexStruct",
       );
     }
 
     if (indexStruct) {
       if (options.nodes) {
         throw new Error(
-          "Cannot initialize VectorStoreIndex with both nodes and indexStruct",
+          "Cannot initialize SummaryIndex with both nodes and indexStruct",
         );
       }
     } else {
       if (!options.nodes) {
         throw new Error(
-          "Cannot initialize VectorStoreIndex without nodes or indexStruct",
+          "Cannot initialize SummaryIndex without nodes or indexStruct",
         );
       }
-      indexStruct = await ListIndex.buildIndexFromNodes(
+      indexStruct = await SummaryIndex.buildIndexFromNodes(
         options.nodes,
         storageContext.docStore,
       );
@@ -104,7 +104,7 @@ export class ListIndex extends BaseIndex<IndexList> {
       await indexStore.addIndexStruct(indexStruct);
     }
 
-    return new ListIndex({
+    return new SummaryIndex({
       storageContext,
       serviceContext,
       docStore,
@@ -119,7 +119,7 @@ export class ListIndex extends BaseIndex<IndexList> {
       storageContext?: StorageContext;
       serviceContext?: ServiceContext;
     } = {},
-  ): Promise<ListIndex> {
+  ): Promise<SummaryIndex> {
     let { storageContext, serviceContext } = args;
     storageContext = storageContext ?? (await storageContextFromDefaults({}));
     serviceContext = serviceContext ?? serviceContextFromDefaults({});
@@ -131,7 +131,7 @@ export class ListIndex extends BaseIndex<IndexList> {
     }
 
     const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
-    const index = await ListIndex.init({
+    const index = await SummaryIndex.init({
       nodes,
       storageContext,
       serviceContext,
@@ -139,14 +139,14 @@ export class ListIndex extends BaseIndex<IndexList> {
     return index;
   }
 
-  asRetriever(options?: { mode: ListRetrieverMode }): BaseRetriever {
-    const { mode = ListRetrieverMode.DEFAULT } = options ?? {};
+  asRetriever(options?: { mode: SummaryRetrieverMode }): BaseRetriever {
+    const { mode = SummaryRetrieverMode.DEFAULT } = options ?? {};
 
     switch (mode) {
-      case ListRetrieverMode.DEFAULT:
-        return new ListIndexRetriever(this);
-      case ListRetrieverMode.LLM:
-        return new ListIndexLLMRetriever(this);
+      case SummaryRetrieverMode.DEFAULT:
+        return new SummaryIndexRetriever(this);
+      case SummaryRetrieverMode.LLM:
+        return new SummaryIndexLLMRetriever(this);
       default:
         throw new Error(`Unknown retriever mode: ${mode}`);
     }
@@ -253,4 +253,5 @@ export class ListIndex extends BaseIndex<IndexList> {
 }
 
 // Legacy
-export type GPTListIndex = ListIndex;
+export type ListIndex = SummaryIndex;
+export type ListRetrieverMode = SummaryRetrieverMode;

packages/core/src/indices/summary/SummaryIndexRetriever.ts

@@ -1,25 +1,25 @@
-import { BaseRetriever } from "../../Retriever";
+import _ from "lodash";
+import { globalsHelper } from "../../GlobalsHelper";
 import { NodeWithScore } from "../../Node";
-import { ListIndex } from "./ListIndex";
+import { ChoiceSelectPrompt, defaultChoiceSelectPrompt } from "../../Prompt";
+import { BaseRetriever } from "../../Retriever";
 import { ServiceContext } from "../../ServiceContext";
+import { Event } from "../../callbacks/CallbackManager";
+import { SummaryIndex } from "./SummaryIndex";
 import {
-  NodeFormatterFunction,
   ChoiceSelectParserFunction,
+  NodeFormatterFunction,
   defaultFormatNodeBatchFn,
   defaultParseChoiceSelectAnswerFn,
 } from "./utils";
-import { SimplePrompt, defaultChoiceSelectPrompt } from "../../Prompt";
-import _ from "lodash";
-import { globalsHelper } from "../../GlobalsHelper";
-import { Event } from "../../callbacks/CallbackManager";
 
 /**
- * Simple retriever for ListIndex that returns all nodes
+ * Simple retriever for SummaryIndex that returns all nodes
  */
-export class ListIndexRetriever implements BaseRetriever {
-  index: ListIndex;
+export class SummaryIndexRetriever implements BaseRetriever {
+  index: SummaryIndex;
 
-  constructor(index: ListIndex) {
+  constructor(index: SummaryIndex) {
     this.index = index;
   }
 
@@ -51,23 +51,23 @@ export class ListIndexRetriever implements BaseRetriever {
 }
 
 /**
- * LLM retriever for ListIndex.
+ * LLM retriever for SummaryIndex which lets you select the most relevant chunks.
  */
-export class ListIndexLLMRetriever implements BaseRetriever {
-  index: ListIndex;
-  choiceSelectPrompt: SimplePrompt;
+export class SummaryIndexLLMRetriever implements BaseRetriever {
+  index: SummaryIndex;
+  choiceSelectPrompt: ChoiceSelectPrompt;
   choiceBatchSize: number;
   formatNodeBatchFn: NodeFormatterFunction;
   parseChoiceSelectAnswerFn: ChoiceSelectParserFunction;
   serviceContext: ServiceContext;
 
   constructor(
-    index: ListIndex,
-    choiceSelectPrompt?: SimplePrompt,
+    index: SummaryIndex,
+    choiceSelectPrompt?: ChoiceSelectPrompt,
     choiceBatchSize: number = 10,
     formatNodeBatchFn?: NodeFormatterFunction,
     parseChoiceSelectAnswerFn?: ChoiceSelectParserFunction,
-    serviceContext?: ServiceContext
+    serviceContext?: ServiceContext,
   ) {
     this.index = index;
     this.choiceSelectPrompt = choiceSelectPrompt || defaultChoiceSelectPrompt;
@@ -95,7 +95,7 @@ export class ListIndexLLMRetriever implements BaseRetriever {
       // parseResult is a map from doc number to relevance score
       const parseResult = this.parseChoiceSelectAnswerFn(
         rawResponse,
-        nodesBatch.length
+        nodesBatch.length,
       );
       const choiceNodeIds = nodeIdsBatch.filter((nodeId, idx) => {
         return `${idx}` in parseResult;
@@ -128,3 +128,7 @@ export class ListIndexLLMRetriever implements BaseRetriever {
     return this.serviceContext;
   }
 }
+
+// Legacy
+export type ListIndexRetriever = SummaryIndexRetriever;
+export type ListIndexLLMRetriever = SummaryIndexLLMRetriever;

packages/core/src/indices/summary/index.ts

@@ -0,0 +1,10 @@
+export { SummaryIndex, SummaryRetrieverMode } from "./SummaryIndex";
+export type { ListIndex, ListRetrieverMode } from "./SummaryIndex";
+export {
+  SummaryIndexLLMRetriever,
+  SummaryIndexRetriever,
+} from "./SummaryIndexRetriever";
+export type {
+  ListIndexLLMRetriever,
+  ListIndexRetriever,
+} from "./SummaryIndexRetriever";

packages/core/src/indices/summary/utils.ts

@@ -1,9 +1,9 @@
-import { BaseNode, MetadataMode } from "../../Node";
 import _ from "lodash";
+import { BaseNode, MetadataMode } from "../../Node";
 
 export type NodeFormatterFunction = (summaryNodes: BaseNode[]) => string;
 export const defaultFormatNodeBatchFn: NodeFormatterFunction = (
-  summaryNodes: BaseNode[]
+  summaryNodes: BaseNode[],
 ): string => {
   return summaryNodes
     .map((node, idx) => {
@@ -20,13 +20,13 @@ export type ChoiceSelectParseResult = { [docNumber: number]: number };
 export type ChoiceSelectParserFunction = (
   answer: string,
   numChoices: number,
-  raiseErr?: boolean
+  raiseErr?: boolean,
 ) => ChoiceSelectParseResult;
 
 export const defaultParseChoiceSelectAnswerFn: ChoiceSelectParserFunction = (
   answer: string,
   numChoices: number,
-  raiseErr: boolean = false
+  raiseErr: boolean = false,
 ): ChoiceSelectParseResult => {
   // split the line into the answer number and relevance score portions
   const lineTokens: string[][] = answer
@@ -36,7 +36,7 @@ export const defaultParseChoiceSelectAnswerFn: ChoiceSelectParserFunction = (
       if (lineTokens.length !== 2) {
         if (raiseErr) {
           throw new Error(
-            `Invalid answer line: ${line}. Answer line must be of the form: answer_num: <int>, answer_relevance: <float>`
+            `Invalid answer line: ${line}. Answer line must be of the form: answer_num: <int>, answer_relevance: <float>`,
           );
         } else {
           return null;
@@ -55,7 +55,7 @@ export const defaultParseChoiceSelectAnswerFn: ChoiceSelectParserFunction = (
         if (docNum < 1 || docNum > numChoices) {
           if (raiseErr) {
             throw new Error(
-              `Invalid answer number: ${docNum}. Answer number must be between 1 and ${numChoices}`
+              `Invalid answer number: ${docNum}. Answer number must be between 1 and ${numChoices}`,
             );
           }
         } else {
@@ -68,6 +68,6 @@ export const defaultParseChoiceSelectAnswerFn: ChoiceSelectParserFunction = (
       }
       return parseResult;
     },
-    {}
+    {},
   );
 };

packages/core/src/indices/vectorStore/VectorIndexRetriever.ts

@@ -1,6 +1,6 @@
-import { VectorStoreIndex } from "./VectorStoreIndex";
 import { globalsHelper } from "../../GlobalsHelper";
 import { NodeWithScore } from "../../Node";
+import { BaseRetriever } from "../../Retriever";
 import { ServiceContext } from "../../ServiceContext";
 import { Event } from "../../callbacks/CallbackManager";
 import { DEFAULT_SIMILARITY_TOP_K } from "../../constants";
@@ -8,7 +8,7 @@ import {
   VectorStoreQuery,
   VectorStoreQueryMode,
 } from "../../storage/vectorStore/types";
-import { BaseRetriever } from "../../Retriever";
+import { VectorStoreIndex } from "./VectorStoreIndex";
 
 /**
  * VectorIndexRetriever retrieves nodes from a VectorIndex.

packages/core/src/indices/vectorStore/index.ts

@@ -1,2 +1,2 @@
-export { VectorStoreIndex } from "./VectorStoreIndex";
 export { VectorIndexRetriever } from "./VectorIndexRetriever";
+export { VectorStoreIndex } from "./VectorStoreIndex";

packages/core/src/llm/LLM.ts

@@ -2,9 +2,9 @@ import OpenAILLM, { ClientOptions as OpenAIClientOptions } from "openai";
 import { CallbackManager, Event } from "../callbacks/CallbackManager";
 import { handleOpenAIStream } from "../callbacks/utility/handleOpenAIStream";
 import {
-  AnthropicSession,
   ANTHROPIC_AI_PROMPT,
   ANTHROPIC_HUMAN_PROMPT,
+  AnthropicSession,
   getAnthropicSession,
 } from "./anthropic";
 import {
@@ -14,7 +14,7 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure";
-import { getOpenAISession, OpenAISession } from "./openai";
+import { OpenAISession, getOpenAISession } from "./openai";
 import { ReplicateSession } from "./replicate";
 
 export type MessageType =
@@ -471,7 +471,7 @@ export class Anthropic implements LLM {
 
     this.apiKey = init?.apiKey ?? undefined;
     this.maxRetries = init?.maxRetries ?? 10;
-    this.timeout = init?.timeout ?? undefined; // Default is 60 seconds
+    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
     this.session =
       init?.session ??
       getAnthropicSession({

packages/core/src/llm/anthropic.ts

@@ -1,6 +1,6 @@
 import Anthropic, {
-  ClientOptions,
   AI_PROMPT,
+  ClientOptions,
   HUMAN_PROMPT,
 } from "@anthropic-ai/sdk";
 import _ from "lodash";

packages/core/src/llm/azure.ts

@@ -38,7 +38,7 @@ const DEFAULT_API_VERSION = "2023-05-15";
 //^ NOTE: this will change over time, if you want to pin it, use a specific version
 
 export function getAzureConfigFromEnv(
-  init?: Partial<AzureOpenAIConfig> & { model?: string }
+  init?: Partial<AzureOpenAIConfig> & { model?: string },
 ): AzureOpenAIConfig {
   return {
     apiKey:
@@ -71,7 +71,7 @@ export function getAzureBaseUrl(config: AzureOpenAIConfig): string {
 
 export function getAzureModel(openAIModel: string) {
   for (const [key, value] of Object.entries(
-    ALL_AZURE_OPENAI_EMBEDDING_MODELS
+    ALL_AZURE_OPENAI_EMBEDDING_MODELS,
   )) {
     if (value.openAIModel === openAIModel) {
       return key;

packages/core/src/llm/openai.ts

@@ -1,5 +1,5 @@
-import OpenAI, { ClientOptions } from "openai";
 import _ from "lodash";
+import OpenAI, { ClientOptions } from "openai";
 
 export class AzureOpenAI extends OpenAI {
   protected override authHeaders() {
@@ -42,7 +42,7 @@ let defaultOpenAISession: { session: OpenAISession; options: ClientOptions }[] =
  * @returns
  */
 export function getOpenAISession(
-  options: ClientOptions & { azure?: boolean } = {}
+  options: ClientOptions & { azure?: boolean } = {},
 ) {
   let session = defaultOpenAISession.find((session) => {
     return _.isEqual(session.options, options);

packages/core/src/llm/replicate.ts

@@ -11,7 +11,7 @@ export class ReplicateSession {
       this.replicateKey = process.env.REPLICATE_API_TOKEN;
     } else {
       throw new Error(
-        "Set Replicate token in REPLICATE_API_TOKEN env variable"
+        "Set Replicate token in REPLICATE_API_TOKEN env variable",
       );
     }
 

packages/core/src/readers/CSVReader.ts

@@ -1,7 +1,7 @@
-import { DEFAULT_FS, GenericFileSystem } from "../storage/FileSystem";
 import Papa, { ParseConfig } from "papaparse";
-import { BaseReader } from "./base";
 import { Document } from "../Node";
+import { DEFAULT_FS, GenericFileSystem } from "../storage/FileSystem";
+import { BaseReader } from "./base";
 
 /**
  * papaparse-based csv parser
@@ -24,7 +24,7 @@ export class PapaCSVReader implements BaseReader {
     concatRows: boolean = true,
     colJoiner: string = ", ",
     rowJoiner: string = "\n",
-    papaConfig?: ParseConfig
+    papaConfig?: ParseConfig,
   ) {
     this.concatRows = concatRows;
     this.colJoiner = colJoiner;
@@ -40,7 +40,7 @@ export class PapaCSVReader implements BaseReader {
    */
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
   ): Promise<Document[]> {
     const fileContent: string = await fs.readFile(file, "utf-8");
     const result = Papa.parse(fileContent, this.papaConfig);

packages/core/src/readers/MarkdownReader.ts

@@ -0,0 +1,110 @@
+import { Document } from "../Node";
+import { DEFAULT_FS, GenericFileSystem } from "../storage";
+import { BaseReader } from "./base";
+
+type MarkdownTuple = [string | null, string];
+
+/**
+ * Extract text from markdown files.
+ * Returns dictionary with keys as headers and values as the text between headers.
+ */
+export class MarkdownReader implements BaseReader {
+  private _removeHyperlinks: boolean;
+  private _removeImages: boolean;
+
+  /**
+   * @param {boolean} [removeHyperlinks=true] - Indicates whether hyperlinks should be removed.
+   * @param {boolean} [removeImages=true] - Indicates whether images should be removed.
+   */
+  constructor(removeHyperlinks: boolean = true, removeImages: boolean = true) {
+    this._removeHyperlinks = removeHyperlinks;
+    this._removeImages = removeImages;
+  }
+
+  /**
+   * Convert a markdown file to a dictionary.
+   * The keys are the headers and the values are the text under each header.
+   * @param {string} markdownText - The markdown text to convert.
+   * @returns {Array<MarkdownTuple>} - An array of tuples, where each tuple contains a header (or null) and its corresponding text.
+   */
+  markdownToTups(markdownText: string): MarkdownTuple[] {
+    const markdownTups: MarkdownTuple[] = [];
+    const lines = markdownText.split("\n");
+
+    let currentHeader: string | null = null;
+    let currentText = "";
+
+    for (const line of lines) {
+      const headerMatch = line.match(/^#+\s/);
+      if (headerMatch) {
+        if (currentHeader) {
+          if (!currentText) {
+            currentHeader += line + "\n";
+            continue;
+          }
+          markdownTups.push([currentHeader, currentText]);
+        }
+
+        currentHeader = line;
+        currentText = "";
+      } else {
+        currentText += line + "\n";
+      }
+    }
+    markdownTups.push([currentHeader, currentText]);
+
+    if (currentHeader) {
+      // pass linting, assert keys are defined
+      markdownTups.map((tuple) => [
+        tuple[0]?.replace(/#/g, "").trim() || null,
+        tuple[1].replace(/<.*?>/g, ""),
+      ]);
+    } else {
+      markdownTups.map((tuple) => [tuple[0], tuple[1].replace(/<.*?>/g, "")]);
+    }
+
+    return markdownTups;
+  }
+
+  removeImages(content: string): string {
+    const pattern = /!{1}\[\[(.*)\]\]/g;
+    return content.replace(pattern, "");
+  }
+
+  removeHyperlinks(content: string): string {
+    const pattern = /\[(.*?)\]\((.*?)\)/g;
+    return content.replace(pattern, "$1");
+  }
+
+  parseTups(content: string): MarkdownTuple[] {
+    let modifiedContent = content;
+    if (this._removeHyperlinks) {
+      modifiedContent = this.removeHyperlinks(modifiedContent);
+    }
+    if (this._removeImages) {
+      modifiedContent = this.removeImages(modifiedContent);
+    }
+    return this.markdownToTups(modifiedContent);
+  }
+
+  async loadData(
+    file: string,
+    fs: GenericFileSystem = DEFAULT_FS,
+  ): Promise<Document[]> {
+    const content = await fs.readFile(file, { encoding: "utf-8" });
+    const tups = this.parseTups(content);
+    const results: Document[] = [];
+    for (const [header, value] of tups) {
+      if (header) {
+        results.push(
+          new Document({
+            text: `\n\n${header}\n${value}`,
+          }),
+        );
+      } else {
+        results.push(new Document({ text: value }));
+      }
+    }
+    return results;
+  }
+}

packages/core/src/readers/NotionReader.ts

@@ -0,0 +1,67 @@
+import { Client } from "@notionhq/client";
+import { crawler, Crawler, Pages, pageToString } from "notion-md-crawler";
+import { Document } from "../Node";
+import { BaseReader } from "./base";
+
+type OptionalSerializers = Parameters<Crawler>[number]["serializers"];
+
+/**
+ * Options for initializing the NotionReader class
+ * @typedef {Object} NotionReaderOptions
+ * @property {Client} client - The Notion Client object for API interactions
+ * @property {OptionalSerializers} [serializers] - Option to customize serialization. See [the url](https://github.com/TomPenguin/notion-md-crawler/tree/main) for details.
+ */
+type NotionReaderOptions = {
+  client: Client;
+  serializers?: OptionalSerializers;
+};
+
+/**
+ * Notion pages are retrieved recursively and converted to Document objects.
+ * Notion Database can also be loaded, and [the serialization method can be customized](https://github.com/TomPenguin/notion-md-crawler/tree/main).
+ *
+ * [Note] To use this reader, must be created the Notion integration must be created in advance
+ * Please refer to [this document](https://www.notion.so/help/create-integrations-with-the-notion-api) for details.
+ */
+export class NotionReader implements BaseReader {
+  private crawl: ReturnType<Crawler>;
+
+  /**
+   * Constructor for the NotionReader class
+   * @param {NotionReaderOptions} options - Configuration options for the reader
+   */
+  constructor({ client, serializers }: NotionReaderOptions) {
+    this.crawl = crawler({ client, serializers });
+  }
+
+  /**
+   * Converts Pages to an array of Document objects
+   * @param {Pages} pages - The Notion pages to convert (Return value of `loadPages`)
+   * @returns {Document[]} An array of Document objects
+   */
+  toDocuments(pages: Pages): Document[] {
+    return Object.values(pages).map((page) => {
+      const text = pageToString(page);
+      return new Document({ text, metadata: page.metadata });
+    });
+  }
+
+  /**
+   * Loads recursively the Notion page with the specified root page ID.
+   * @param {string} rootPageId - The root Notion page ID
+   * @returns {Promise<Pages>} A Promise that resolves to a Pages object(Convertible with the `toDocuments` method)
+   */
+  async loadPages(rootPageId: string): Promise<Pages> {
+    return this.crawl(rootPageId);
+  }
+
+  /**
+   * Loads recursively Notion pages and converts them to an array of Document objects
+   * @param {string} rootPageId - The root Notion page ID
+   * @returns {Promise<Document[]>} A Promise that resolves to an array of Document objects
+   */
+  async loadData(rootPageId: string): Promise<Document[]> {
+    const pages = await this.loadPages(rootPageId);
+    return this.toDocuments(pages);
+  }
+}

packages/core/src/readers/PDFReader.ts

@@ -1,8 +1,8 @@
+import pdfParse from "pdf-parse";
 import { Document } from "../Node";
-import { BaseReader } from "./base";
 import { GenericFileSystem } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
-import pdfParse from "pdf-parse";
+import { BaseReader } from "./base";
 
 /**
  * Read the text of a PDF
@@ -10,7 +10,7 @@ import pdfParse from "pdf-parse";
 export class PDFReader implements BaseReader {
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
   ): Promise<Document[]> {
     const dataBuffer = (await fs.readFile(file)) as any;
     const data = await pdfParse(dataBuffer);

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -1,10 +1,11 @@
 import _ from "lodash";
 import { Document } from "../Node";
-import { BaseReader } from "./base";
 import { CompleteFileSystem, walk } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
-import { PDFReader } from "./PDFReader";
 import { PapaCSVReader } from "./CSVReader";
+import { MarkdownReader } from "./MarkdownReader";
+import { PDFReader } from "./PDFReader";
+import { BaseReader } from "./base";
 
 /**
  * Read a .txt file
@@ -12,7 +13,7 @@ import { PapaCSVReader } from "./CSVReader";
 export class TextFileReader implements BaseReader {
   async loadData(
     file: string,
-    fs: CompleteFileSystem = DEFAULT_FS as CompleteFileSystem
+    fs: CompleteFileSystem = DEFAULT_FS as CompleteFileSystem,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readFile(file, "utf-8");
     return [new Document({ text: dataBuffer, id_: file })];
@@ -23,6 +24,7 @@ const FILE_EXT_TO_READER: Record<string, BaseReader> = {
   txt: new TextFileReader(),
   pdf: new PDFReader(),
   csv: new PapaCSVReader(),
+  md: new MarkdownReader(),
 };
 
 export type SimpleDirectoryReaderLoadDataProps = {

packages/core/src/readers/SimpleMongoReader.ts

@@ -0,0 +1,51 @@
+import { MongoClient } from "mongodb";
+import { Document } from "../Node";
+import { BaseReader } from "./base";
+
+/**
+ * Read in from MongoDB
+ */
+export class SimpleMongoReader implements BaseReader {
+  private client: MongoClient;
+
+  constructor(client: MongoClient) {
+    this.client = client;
+  }
+
+  /**
+   * Loads data from MongoDB collection
+   * @param {string} db_name - The name of the database to load.
+   * @param {string} collection_name - The name of the collection to load.
+   * @param {Number} [max_docs = 0] - Maximum number of documents to return. 0 means no limit.
+   * @param {Record<string, any>} [query_dict={}] - Specific query, as specified by MongoDB NodeJS documentation.
+   * @param {Record<string, any>} [query_options={}] - Specific query options, as specified by MongoDB NodeJS documentation.
+   * @param {Record<string, any>} [projection = {}] - Projection options, as specified by MongoDB NodeJS documentation.
+   * @returns {Promise<Document[]>}
+   */
+  async loadData(
+    db_name: string,
+    collection_name: string,
+    max_docs = 0,
+    //For later: Think about whether we want to pass generic objects in...
+    query_dict: Record<string, any> = {},
+    query_options: Record<string, any> = {},
+    projection: Record<string, any> = {},
+  ): Promise<Document[]> {
+    //Get items from collection using built-in functions
+    const cursor: Partial<Document>[] = await this.client
+      .db(db_name)
+      .collection(collection_name)
+      .find(query_dict, query_options)
+      .limit(max_docs)
+      .project(projection)
+      .toArray();
+
+    //Aggregate results and return
+    const documents: Document[] = [];
+    cursor.forEach((element: Partial<Document>) => {
+      //For later: Metadata filtering
+      documents.push(new Document({ text: JSON.stringify(element) }));
+    });
+    return documents;
+  }
+}

packages/core/src/storage/FileSystem.ts

@@ -73,7 +73,7 @@ export const DEFAULT_FS: GenericFileSystem | CompleteFileSystem =
  */
 export async function exists(
   fs: GenericFileSystem,
-  path: string
+  path: string,
 ): Promise<boolean> {
   try {
     await fs.access(path);
@@ -90,11 +90,11 @@ export async function exists(
  */
 export async function* walk(
   fs: WalkableFileSystem,
-  dirPath: string
+  dirPath: string,
 ): AsyncIterable<string> {
   if (fs instanceof InMemoryFileSystem) {
     throw new Error(
-      "The InMemoryFileSystem does not support directory traversal."
+      "The InMemoryFileSystem does not support directory traversal.",
     );
   }
 

packages/core/src/storage/StorageContext.ts

@@ -1,15 +1,11 @@
-import { BaseDocumentStore } from "./docStore/types";
-import { BaseIndexStore } from "./indexStore/types";
-import { VectorStore } from "./vectorStore/types";
-import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore";
-import { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
-import { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
 import { GenericFileSystem } from "./FileSystem";
-import {
-  DEFAULT_PERSIST_DIR,
-  DEFAULT_FS,
-  DEFAULT_NAMESPACE,
-} from "./constants";
+import { DEFAULT_FS, DEFAULT_NAMESPACE } from "./constants";
+import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore";
+import { BaseDocumentStore } from "./docStore/types";
+import { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
+import { BaseIndexStore } from "./indexStore/types";
+import { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
+import { VectorStore } from "./vectorStore/types";
 
 export interface StorageContext {
   docStore: BaseDocumentStore;
@@ -43,7 +39,7 @@ export async function storageContextFromDefaults({
       (await SimpleDocumentStore.fromPersistDir(
         persistDir,
         DEFAULT_NAMESPACE,
-        fs
+        fs,
       ));
     indexStore =
       indexStore || (await SimpleIndexStore.fromPersistDir(persistDir, fs));

packages/core/src/storage/docStore/SimpleDocumentStore.ts

@@ -1,15 +1,15 @@
-import * as path from "path";
 import _ from "lodash";
-import { KVDocumentStore } from "./KVDocumentStore";
-import { SimpleKVStore } from "../kvStore/SimpleKVStore";
-import { BaseInMemoryKVStore } from "../kvStore/types";
+import * as path from "path";
 import { GenericFileSystem } from "../FileSystem";
 import {
-  DEFAULT_PERSIST_DIR,
-  DEFAULT_NAMESPACE,
   DEFAULT_DOC_STORE_PERSIST_FILENAME,
   DEFAULT_FS,
+  DEFAULT_NAMESPACE,
+  DEFAULT_PERSIST_DIR,
 } from "../constants";
+import { SimpleKVStore } from "../kvStore/SimpleKVStore";
+import { BaseInMemoryKVStore } from "../kvStore/types";
+import { KVDocumentStore } from "./KVDocumentStore";
 
 type SaveDict = Record<string, any>;
 
@@ -26,23 +26,23 @@ export class SimpleDocumentStore extends KVDocumentStore {
   static async fromPersistDir(
     persistDir: string = DEFAULT_PERSIST_DIR,
     namespace?: string,
-    fsModule?: GenericFileSystem
+    fsModule?: GenericFileSystem,
   ): Promise<SimpleDocumentStore> {
     const persistPath = path.join(
       persistDir,
-      DEFAULT_DOC_STORE_PERSIST_FILENAME
+      DEFAULT_DOC_STORE_PERSIST_FILENAME,
     );
     return await SimpleDocumentStore.fromPersistPath(
       persistPath,
       namespace,
-      fsModule
+      fsModule,
     );
   }
 
   static async fromPersistPath(
     persistPath: string,
     namespace?: string,
-    fs?: GenericFileSystem
+    fs?: GenericFileSystem,
   ): Promise<SimpleDocumentStore> {
     fs = fs || DEFAULT_FS;
     const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
@@ -52,9 +52,9 @@ export class SimpleDocumentStore extends KVDocumentStore {
   async persist(
     persistPath: string = path.join(
       DEFAULT_PERSIST_DIR,
-      DEFAULT_DOC_STORE_PERSIST_FILENAME
+      DEFAULT_DOC_STORE_PERSIST_FILENAME,
     ),
-    fs?: GenericFileSystem
+    fs?: GenericFileSystem,
   ): Promise<void> {
     fs = fs || DEFAULT_FS;
     if (

packages/core/src/storage/docStore/utils.ts

@@ -1,4 +1,4 @@
-import { BaseNode, Document, TextNode, ObjectType } from "../../Node";
+import { BaseNode, Document, ObjectType, TextNode } from "../../Node";
 
 const TYPE_KEY = "__type__";
 const DATA_KEY = "__data__";
@@ -21,12 +21,14 @@ export function jsonToDoc(docDict: Record<string, any>): BaseNode {
       id_: dataDict.id_,
       embedding: dataDict.embedding,
       hash: dataDict.hash,
+      metadata: dataDict.metadata,
     });
   } else if (docType === ObjectType.TEXT) {
     doc = new TextNode({
       text: dataDict.text,
       id_: dataDict.id_,
       hash: dataDict.hash,
+      metadata: dataDict.metadata,
     });
   } else {
     throw new Error(`Unknown doc type: ${docType}`);

packages/core/src/storage/index.ts

@@ -1,7 +1,11 @@
-export * from "./constants";
 export * from "./FileSystem";
 export * from "./StorageContext";
-export * from "./vectorStore/types";
+export * from "./constants";
 export { SimpleDocumentStore } from "./docStore/SimpleDocumentStore";
+export * from "./docStore/types";
 export { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
+export * from "./indexStore/types";
+export { SimpleKVStore } from "./kvStore/SimpleKVStore";
+export * from "./kvStore/types";
 export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
+export * from "./vectorStore/types";

packages/core/src/storage/indexStore/KVIndexStore.ts

@@ -1,7 +1,7 @@
-import { BaseKVStore } from "../kvStore/types";
-import { IndexStruct, jsonToIndexStruct } from "../../indices/BaseIndex";
 import _ from "lodash";
+import { IndexStruct, jsonToIndexStruct } from "../../indices/BaseIndex";
 import { DEFAULT_NAMESPACE } from "../constants";
+import { BaseKVStore } from "../kvStore/types";
 import { BaseIndexStore } from "./types";
 
 export class KVIndexStore extends BaseIndexStore {

packages/core/src/storage/indexStore/SimpleIndexStore.ts

@@ -1,14 +1,13 @@
 import * as path from "path";
-import * as _ from "lodash";
-import { BaseInMemoryKVStore } from "../kvStore/types";
-import { SimpleKVStore, DataType } from "../kvStore/SimpleKVStore";
-import { KVIndexStore } from "./KVIndexStore";
-import {
-  DEFAULT_PERSIST_DIR,
-  DEFAULT_INDEX_STORE_PERSIST_FILENAME,
-  DEFAULT_FS,
-} from "../constants";
 import { GenericFileSystem } from "../FileSystem";
+import {
+  DEFAULT_FS,
+  DEFAULT_INDEX_STORE_PERSIST_FILENAME,
+  DEFAULT_PERSIST_DIR,
+} from "../constants";
+import { DataType, SimpleKVStore } from "../kvStore/SimpleKVStore";
+import { BaseInMemoryKVStore } from "../kvStore/types";
+import { KVIndexStore } from "./KVIndexStore";
 
 export class SimpleIndexStore extends KVIndexStore {
   private kvStore: BaseInMemoryKVStore;
@@ -21,18 +20,18 @@ export class SimpleIndexStore extends KVIndexStore {
 
   static async fromPersistDir(
     persistDir: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
   ): Promise<SimpleIndexStore> {
     const persistPath = path.join(
       persistDir,
-      DEFAULT_INDEX_STORE_PERSIST_FILENAME
+      DEFAULT_INDEX_STORE_PERSIST_FILENAME,
     );
     return this.fromPersistPath(persistPath, fs);
   }
 
   static async fromPersistPath(
     persistPath: string,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
   ): Promise<SimpleIndexStore> {
     let simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
     return new SimpleIndexStore(simpleKVStore);
@@ -40,7 +39,7 @@ export class SimpleIndexStore extends KVIndexStore {
 
   async persist(
     persistPath: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
   ): Promise<void> {
     await this.kvStore.persist(persistPath, fs);
   }

packages/core/src/storage/indexStore/types.ts

@@ -1,8 +1,8 @@
 import { IndexStruct } from "../../indices/BaseIndex";
 import { GenericFileSystem } from "../FileSystem";
 import {
-  DEFAULT_PERSIST_DIR,
   DEFAULT_INDEX_STORE_PERSIST_FILENAME,
+  DEFAULT_PERSIST_DIR,
 } from "../constants";
 
 const defaultPersistPath = `${DEFAULT_PERSIST_DIR}/${DEFAULT_INDEX_STORE_PERSIST_FILENAME}`;
@@ -18,7 +18,7 @@ export abstract class BaseIndexStore {
 
   async persist(
     persistPath: string = defaultPersistPath,
-    fs?: GenericFileSystem
+    fs?: GenericFileSystem,
   ): Promise<void> {
     // Persist the index store to disk.
   }

packages/core/src/storage/kvStore/SimpleKVStore.ts

@@ -1,7 +1,7 @@
+import * as _ from "lodash";
 import * as path from "path";
 import { GenericFileSystem, exists } from "../FileSystem";
 import { DEFAULT_COLLECTION, DEFAULT_FS } from "../constants";
-import * as _ from "lodash";
 import { BaseKVStore } from "./types";
 
 export type DataType = Record<string, Record<string, any>>;
@@ -19,7 +19,7 @@ export class SimpleKVStore extends BaseKVStore {
   async put(
     key: string,
     val: any,
-    collection: string = DEFAULT_COLLECTION
+    collection: string = DEFAULT_COLLECTION,
   ): Promise<void> {
     if (!(collection in this.data)) {
       this.data[collection] = {};
@@ -33,7 +33,7 @@ export class SimpleKVStore extends BaseKVStore {
 
   async get(
     key: string,
-    collection: string = DEFAULT_COLLECTION
+    collection: string = DEFAULT_COLLECTION,
   ): Promise<any> {
     let collectionData = this.data[collection];
     if (_.isNil(collectionData)) {
@@ -51,7 +51,7 @@ export class SimpleKVStore extends BaseKVStore {
 
   async delete(
     key: string,
-    collection: string = DEFAULT_COLLECTION
+    collection: string = DEFAULT_COLLECTION,
   ): Promise<boolean> {
     if (key in this.data[collection]) {
       delete this.data[collection][key];
@@ -72,7 +72,7 @@ export class SimpleKVStore extends BaseKVStore {
 
   static async fromPersistPath(
     persistPath: string,
-    fs?: GenericFileSystem
+    fs?: GenericFileSystem,
   ): Promise<SimpleKVStore> {
     fs = fs || DEFAULT_FS;
     let dirPath = path.dirname(persistPath);
@@ -86,7 +86,7 @@ export class SimpleKVStore extends BaseKVStore {
       data = JSON.parse(fileData.toString());
     } catch (e) {
       console.error(
-        `No valid data found at path: ${persistPath} starting new store.`
+        `No valid data found at path: ${persistPath} starting new store.`,
       );
     }
 

packages/core/src/storage/kvStore/types.ts

@@ -7,7 +7,7 @@ export abstract class BaseKVStore {
   abstract put(
     key: string,
     val: Record<string, any>,
-    collection?: string
+    collection?: string,
   ): Promise<void>;
   abstract get(key: string, collection?: string): Promise<StoredValue>;
   abstract getAll(collection?: string): Promise<Record<string, StoredValue>>;