updated exports for storage/index.ts

Merge pull request #93 from andfk/patch-1
Update README.md for apps example
2026-07-16 07:14:29 -04:00 · 2023-08-30 12:06:58 -07:00 · 2023-08-30 06:20:40 -07:00 · 2023-08-30 18:13:00 +08:00 · 2023-08-29 22:35:41 -07:00 · 2023-08-29 20:30:02 -07:00
88 changed files with 1040 additions and 741 deletions
@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+OpenAI 4.3.1 and Anthropic 0.6.2
@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Update READMEs (thanks @andfk)
@@ -1,5 +0,0 @@
---
-"llamaindex": patch
---
-
-Added Markdown Reader (huge shoutout to @swk777)
@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Bug: missing exports from storage (thanks @aashutoshrathi)
@@ -1,5 +1,5 @@
 name: Bugfix
-title: 'Sweep: '
+title: "Sweep: "
 description: Write something like "We notice ... behavior when ... happens instead of ...""
 labels: sweep
 body:
@@ -8,4 +8,4 @@ body:
    attributes:
      label: Details
      description: More details about the bug
-      placeholder: The bug might be in ... file
+      placeholder: The bug might be in ... file
@@ -1,5 +1,5 @@
 name: Feature Request
-title: 'Sweep: '
+title: "Sweep: "
 description: Write something like "Write an api endpoint that does "..." in the "..." file"
 labels: sweep
 body:
@@ -8,4 +8,4 @@ body:
    attributes:
      label: Details
      description: More details for Sweep
-      placeholder: The new endpoint should use the ... class from ... file because it contains ... logic
+      placeholder: The new endpoint should use the ... class from ... file because it contains ... logic
@@ -1,5 +1,5 @@
 name: Refactor
-title: 'Sweep: '
+title: "Sweep: "
 description: Write something like "Modify the ... api endpoint to use ... version and ... framework"
 labels: sweep
 body:
@@ -8,4 +8,4 @@ body:
    attributes:
      label: Details
      description: More details for Sweep
-      placeholder: We are migrating this function to ... version because ...
+      placeholder: We are migrating this function to ... version because ...
@@ -22,4 +22,4 @@ jobs:
        run: pnpm install

      - name: Run lint
-        run: pnpm run lint
+        run: pnpm run lint
@@ -7,18 +7,18 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v2

-    - name: Setup Node.js
-      uses: actions/setup-node@v2
-      with:
-        node-version: '18'
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: "18"

-    - name: Install dependencies
-      run: |
-        npm i -g pnpm
-        pnpm install
+      - name: Install dependencies
+        run: |
+          npm i -g pnpm
+          pnpm install

-    - name: Run tests
-      run: pnpm run test
+      - name: Run tests
+        run: pnpm run test
@@ -20,7 +20,7 @@ In a new folder:
 export OPENAI_API_KEY="sk-......" # Replace with your key from https://platform.openai.com/account/api-keys
 pnpm init
 pnpm install typescript
-pnpm exec tsc –-init # if needed
+pnpm exec tsc --init # if needed
 pnpm install llamaindex
 pnpm install @types/node
 ```
@@ -36,7 +36,7 @@ async function main() {
  // Load essay from abramov.txt in Node
  const essay = await fs.readFile(
    "node_modules/llamaindex/examples/abramov.txt",
-    "utf-8"
+    "utf-8",
  );

  // Create Document object with essay
@@ -48,7 +48,7 @@ async function main() {
  // Query the index
  const queryEngine = index.asQueryEngine();
  const response = await queryEngine.query(
-    "What did the author do in college?"
+    "What did the author do in college?",
  );

  // Output response
@@ -61,7 +61,7 @@ main();
 Then you can run it using

 ```bash
-pnpm dlx ts-node example.ts
+pnpx ts-node example.ts
 ```

 ## Playground
@@ -1,3 +1,3 @@
 module.exports = {
-  presets: [require.resolve('@docusaurus/core/lib/babel/preset')],
+  presets: [require.resolve("@docusaurus/core/lib/babel/preset")],
 };
@@ -8,40 +8,42 @@ LlamaIndex.TS helps you build LLM-powered applications (e.g. Q&A, chatbot) over

 In this high-level concepts guide, you will learn:

-* how an LLM can answer questions using your own data.
-* key concepts and modules in LlamaIndex.TS for composing your own query pipeline.
+- how an LLM can answer questions using your own data.
+- key concepts and modules in LlamaIndex.TS for composing your own query pipeline.

 ## Answering Questions Across Your Data

 LlamaIndex uses a two stage method when using an LLM with your data:

-1) **indexing stage**: preparing a knowledge base, and
-2) **querying stage**: retrieving relevant context from the knowledge to assist the LLM in responding to a question
+1. **indexing stage**: preparing a knowledge base, and
+2. **querying stage**: retrieving relevant context from the knowledge to assist the LLM in responding to a question

 ![](./_static/concepts/rag.jpg)

 This process is also known as Retrieval Augmented Generation (RAG).

-LlamaIndex.TS provides the essential toolkit for making both steps super easy. 
+LlamaIndex.TS provides the essential toolkit for making both steps super easy.

 Let's explore each stage in detail.

 ### Indexing Stage
+
 LlamaIndex.TS help you prepare the knowledge base with a suite of data connectors and indexes.

-![](./_static/concepts/indexing.jpg) 
+![](./_static/concepts/indexing.jpg)

 [**Data Loaders**](./modules/high_level/data_loader.md):
 A data connector (i.e. `Reader`) ingest data from different data sources and data formats into a simple `Document` representation (text and simple metadata).

 [**Documents / Nodes**](./modules/high_level/documents_and_nodes.md): A `Document` is a generic container around any data source - for instance, a PDF, an API output, or retrieved data from a database. A `Node` is the atomic unit of data in LlamaIndex and represents a "chunk" of a source `Document`. It's a rich representation that includes metadata and relationships (to other nodes) to enable accurate and expressive retrieval operations.

-[**Data Indexes**](./modules/high_level/data_index.md): 
+[**Data Indexes**](./modules/high_level/data_index.md):
 Once you've ingested your data, LlamaIndex helps you index data into a format that's easy to retrieve.

 Under the hood, LlamaIndex parses the raw documents into intermediate representations, calculates vector embeddings, and stores your data in-memory or to disk.

 ### Querying Stage
+
 In the querying stage, the query pipeline retrieves the most relevant context given a user query,
 and pass that to the LLM (along with the query) to synthesize a response.

@@ -57,12 +59,13 @@ These building blocks can be customized to reflect ranking preferences, as well
 ![](./_static/concepts/querying.jpg)

 #### Building Blocks
-[**Retrievers**](./modules/low_level/retriever.md): 
+
+[**Retrievers**](./modules/low_level/retriever.md):
 A retriever defines how to efficiently retrieve relevant context from a knowledge base (i.e. index) when given a query.
 The specific retrieval logic differs for difference indices, the most popular being dense retrieval against a vector index.

 [**Response Synthesizers**](./modules/low_level/response_synthesizer.md):
-A response synthesizer generates a response from an LLM, using a user query and a given set of retrieved text chunks.  
+A response synthesizer generates a response from an LLM, using a user query and a given set of retrieved text chunks.

 #### Pipelines

@@ -70,7 +73,6 @@ A response synthesizer generates a response from an LLM, using a user query and
 A query engine is an end-to-end pipeline that allow you to ask question over your data.
 It takes in a natural language query, and returns a response, along with reference context retrieved and passed to the LLM.

-
-[**Chat Engines**](./modules/high_level/chat_engine.md): 
+[**Chat Engines**](./modules/high_level/chat_engine.md):
 A chat engine is an end-to-end pipeline for having a conversation with your data
 (multiple back-and-forth instead of a single question & answer).
@@ -10,14 +10,14 @@ We include several end-to-end examples using LlamaIndex.TS in the repository

 Read a file and chat about it with the LLM.

-## [List Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/listIndex.ts)
-
-Create a list index and query it. This example also use the `LLMRetriever`, which will use the LLM to select the best nodes to use when generating answer.
-
 ## [Vector Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/vectorIndex.ts)

 Create a vector index and query it. The vector index will use embeddings to fetch the top k most relevant nodes. By default, the top k is 2.

+## [Summary Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/summarIndex.ts)
+
+Create a list index and query it. This example also use the `LLMRetriever`, which will use the LLM to select the best nodes to use when generating answer.
+
 ## [Save / Load an Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/storageContext.ts)

 Create and load a vector index. Persistance to disk in LlamaIndex.TS happens automatically once a storage context object is created.
@@ -28,7 +28,7 @@ Create a vector index and query it, while also configuring the the `LLM`, the `S

 ## [OpenAI LLM](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/openai.ts)

-Create an OpenAI LLM and directly use it for chat. 
+Create an OpenAI LLM and directly use it for chat.

 ## [Llama2 DeuceLLM](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/llamadeuce.ts)

@@ -40,4 +40,4 @@ Uses the `SubQuestionQueryEngine`, which breaks complex queries into multiple qu

 ## [Low Level Modules](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/lowlevel.ts)

-This example uses several low-level components, which removes the need for an actual query engine. These components can be used anywhere, in any application, or customized and sub-classed to meet your own needs.
+This example uses several low-level components, which removes the need for an actual query engine. These components can be used anywhere, in any application, or customized and sub-classed to meet your own needs.
@@ -1,3 +1,3 @@
 label: "Modules"
 collapsed: false
-position: 5
+position: 5
@@ -1 +1 @@
-label: High-Level Modules
+label: High-Level Modules
@@ -6,23 +6,18 @@ sidebar_position: 2

 An index is the basic container and organization for your data. LlamaIndex.TS supports two indexes:

- `ListIndex` - will send every `Node` in the index to the LLM in order to generate a response
 - `VectorStoreIndex` - will send the top-k `Node`s to the LLM when generating a response. The default top-k is 2.
+- `SummaryIndex` - will send every `Node` in the index to the LLM in order to generate a response

 ```typescript
-import {
-  Document,
-  VectorStoreIndex,
-} from "llamaindex";
+import { Document, VectorStoreIndex } from "llamaindex";

 const document = new Document({ text: "test" });

-const index = await VectorStoreIndex.fromDocuments(
-  [document]
-);
+const index = await VectorStoreIndex.fromDocuments([document]);
 ```

 ## API Reference

- [ListIndex](../../api/classes/ListIndex.md)
- [VectorStoreIndex](../../api/classes/VectorStoreIndex.md)
+- [SummaryIndex](../../api/classes/SummaryIndex.md)
+- [VectorStoreIndex](../../api/classes/VectorStoreIndex.md)
@@ -9,7 +9,7 @@ sidebar_position: 0
 ```typescript
 import { Document } from "llamaindex";

-document = new Document({ text: "text", metadata: { "key": "val" }});
+document = new Document({ text: "text", metadata: { key: "val" } });
 ```

 ## API Reference
@@ -1 +1 @@
-label: Low-Level Modules
+label: Low-Level Modules
@@ -4,7 +4,7 @@ sidebar_position: 1

 # Embedding

-The embedding model in LlamaIndex is responsible for creating numerical representations of text. By default, LlamaIndex will use the `text-embedding-ada-002` model from OpenAI. 
+The embedding model in LlamaIndex is responsible for creating numerical representations of text. By default, LlamaIndex will use the `text-embedding-ada-002` model from OpenAI.

 This can be explicitly set in the `ServiceContext` object.

@@ -4,7 +4,7 @@ sidebar_position: 0

 # LLM

-The LLM is responsible for reading text and generating natural language responses to queries. By default, LlamaIndex.TS uses `gpt-3.5-turbo`. 
+The LLM is responsible for reading text and generating natural language responses to queries. By default, LlamaIndex.TS uses `gpt-3.5-turbo`.

 The LLM can be explicitly set in the `ServiceContext` object.

@@ -19,4 +19,4 @@ const serviceContext = serviceContextFromDefaults({ llm: openaiLLM });
 ## API Reference

 - [OpenAI](../../api/classes/OpenAI.md)
- [ServiceContext](../../api/interfaces/ServiceContext.md)
+- [ServiceContext](../../api/interfaces/ServiceContext.md)
@@ -7,10 +7,7 @@ sidebar_position: 3
 The `NodeParser` in LlamaIndex is responbile for splitting `Document` objects into more manageable `Node` objects. When you call `.fromDocuments()`, the `NodeParser` from the `ServiceContext` is used to do this automatically for you. Alternatively, you can use it to split documents ahead of time.

 ```typescript
-import {
-  Document,
-  SimpleNodeParser,
-} from "llamaindex";
+import { Document, SimpleNodeParser } from "llamaindex";

 const nodeParser = new SimpleNodeParser();
 const nodes = nodeParser.getNodesFromDocuments([
@@ -25,7 +22,7 @@ The underlying text splitter will split text by sentences. It can also be used a
 ```typescript
 import { SentenceSplitter } from "llamaindex";

-const splitter = new SentenceSplitter({ chunkSize: 1, });
+const splitter = new SentenceSplitter({ chunkSize: 1 });

 const textSplits = splitter.splitText("Hello World");
 ```
@@ -6,26 +6,21 @@ sidebar_position: 6

 The ResponseSynthesizer is responsible for sending the query, nodes, and prompt templates to the LLM to generate a response. There are a few key modes for generating a response:

- `Refine`: "create and refine" an answer by sequentially going through each retrieved text chunk. 
-    This makes a separate LLM call per Node. Good for more detailed answers.
- `CompactAndRefine` (default): "compact" the prompt during each LLM call by stuffing as 
-    many text chunks that can fit within the maximum prompt size. If there are 
-    too many chunks to stuff in one prompt, "create and refine" an answer by going through
-    multiple compact prompts. The same as `refine`, but should result in less LLM calls.
- `TreeSummarize`: Given a set of text chunks and the query, recursively construct a tree 
-    and return the root node as the response. Good for summarization purposes.
+- `Refine`: "create and refine" an answer by sequentially going through each retrieved text chunk.
+  This makes a separate LLM call per Node. Good for more detailed answers.
+- `CompactAndRefine` (default): "compact" the prompt during each LLM call by stuffing as
+  many text chunks that can fit within the maximum prompt size. If there are
+  too many chunks to stuff in one prompt, "create and refine" an answer by going through
+  multiple compact prompts. The same as `refine`, but should result in less LLM calls.
+- `TreeSummarize`: Given a set of text chunks and the query, recursively construct a tree
+  and return the root node as the response. Good for summarization purposes.
 - `SimpleResponseBuilder`: Given a set of text chunks and the query, apply the query to each text
-    chunk while accumulating the responses into an array. Returns a concatenated string of all
-    responses. Good for when you need to run the same query separately against each text
-    chunk.
+  chunk while accumulating the responses into an array. Returns a concatenated string of all
+  responses. Good for when you need to run the same query separately against each text
+  chunk.

 ```typescript
-import {
-  TextNode,
-  NodeWithScore,
-  ResponseSynthesizer,
-  CompactAndRefine
-} from "llamaindex";
+import { NodeWithScore, ResponseSynthesizer, TextNode } from "llamaindex";

 const responseSynthesizer = new ResponseSynthesizer();

@@ -42,7 +37,7 @@ const nodesWithScore: NodeWithScore[] = [

 const response = await responseSynthesizer.synthesize(
  "What age am I?",
-  nodesWithScore
+  nodesWithScore,
 );
 console.log(response.response);
 ```
@@ -4,10 +4,10 @@ sidebar_position: 5

 # Retriever

-A retriever in LlamaIndex is what is used to fetch `Node`s from an index using a query string. For example, a `ListIndexRetriever` will fetch all nodes no matter the query. Meanwhile, a `VectorIndexRetriever` will only fetch the top-k most similar nodes.
+A retriever in LlamaIndex is what is used to fetch `Node`s from an index using a query string. Aa `VectorIndexRetriever` will fetch the top-k most similar nodes. Meanwhile, a `SummaryIndexRetriever` will fetch all nodes no matter the query.

 ```typescript
-const retriever = vector_index.asRetriever()
+const retriever = vector_index.asRetriever();
 retriever.similarityTopK = 3;

 // Fetch nodes!
@@ -16,6 +16,6 @@ const nodesWithScore = await retriever.retrieve("query string");

 ## API Reference

- [ListIndexRetriever](../../api/classes/ListIndexRetriever.md)
- [ListIndexLLMRetriever](../../api/classes/ListIndexLLMRetriever.md)
+- [SummaryIndexRetriever](../../api/classes/SummaryIndexRetriever.md)
+- [SummaryIndexLLMRetriever](../../api/classes/SummaryIndexLLMRetriever.md)
 - [VectorIndexRetriever](../../api/classes/VectorIndexRetriever.md)
@@ -11,10 +11,14 @@ Right now, only saving and loading from disk is supported, with future integrati
 ```typescript
 import { Document, VectorStoreIndex, storageContextFromDefaults } from "./src";

-const storageContext = await storageContextFromDefaults({ persistDir: "./storage" });
+const storageContext = await storageContextFromDefaults({
+  persistDir: "./storage",
+});

 const document = new Document({ text: "Test Text" });
-const index = await VectorStoreIndex.fromDocuments([document], { storageContext });
+const index = await VectorStoreIndex.fromDocuments([document], {
+  storageContext,
+});
 ```

 ## API Reference
@@ -25,7 +25,7 @@ async function main() {
  // Load essay from abramov.txt in Node
  const essay = await fs.readFile(
    "node_modules/llamaindex/examples/abramov.txt",
-    "utf-8"
+    "utf-8",
  );

  // Create Document object with essay
@@ -37,7 +37,7 @@ async function main() {
  // Query the index
  const queryEngine = index.asQueryEngine();
  const response = await queryEngine.query(
-    "What did the author do in college?"
+    "What did the author do in college?",
  );

  // Output response
@@ -1,5 +1,5 @@
-import React from "react";
 import clsx from "clsx";
+import React from "react";
 import styles from "./styles.module.css";

 type FeatureItem = {
@@ -18,7 +18,7 @@
 }

 /* For readability concerns, you should choose a lighter palette in dark mode. */
-[data-theme='dark'] {
+[data-theme="dark"] {
  --ifm-color-primary: #25c2a0;
  --ifm-color-primary-dark: #21af90;
  --ifm-color-primary-darker: #1fa588;
@@ -1,5 +1,21 @@
 # simple

+## 0.0.22
+
+### Patch Changes
+
+- Updated dependencies [e4af7b3]
+- Updated dependencies [259fe63]
+  - llamaindex@0.0.24
+
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies
+- Updated dependencies [9d6b2ed]
+  - llamaindex@0.0.23
+
 ## 0.0.20

 ### Patch Changes
@@ -1,9 +1,10 @@
 # Simple Examples

 Due to packaging, you will need to run these commands to get started.
+
 ```bash
-pnpm --filter llamaindex build
 pnpm install
+pnpm --filter llamaindex build
 ```

 Then run the examples with `ts-node`, for example `npx ts-node vectorIndex.ts`
@@ -4,7 +4,6 @@ import {
  PapaCSVReader,
  ResponseSynthesizer,
  serviceContextFromDefaults,
-  SimplePrompt,
  VectorStoreIndex,
 } from "llamaindex";

@@ -23,9 +22,7 @@ async function main() {
    serviceContext,
  });

-  const csvPrompt: SimplePrompt = (input) => {
-    const { context = "", query = "" } = input;
-
+  const csvPrompt = ({ context = "", query = "" }) => {
    return `The following CSV file is loaded from ${path}
 \`\`\`csv
 ${context}
@@ -1,9 +1,9 @@
 import {
  Document,
-  TextNode,
  NodeWithScore,
  ResponseSynthesizer,
  SimpleNodeParser,
+  TextNode,
 } from "llamaindex";

 (async () => {
@@ -29,7 +29,7 @@ import {

  const response = await responseSynthesizer.synthesize(
    "What age am I?",
-    nodesWithScore
+    nodesWithScore,
  );
  console.log(response.response);
 })();
@@ -1,14 +1,7 @@
 import { OpenAI } from "llamaindex";

 (async () => {
-  const llm = new OpenAI({
-    model: "gpt-3.5-turbo",
-    temperature: 0.1,
-    additionalChatOptions: { frequency_penalty: 0.1 },
-    additionalSessionOptions: {
-      defaultHeaders: { "X-Test-Header-Please-Ignore": "true" },
-    },
-  });
+  const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 });

  // complete api
  const response1 = await llm.complete("How are you?");
@@ -1,5 +1,5 @@
 {
-  "version": "0.0.20",
+  "version": "0.0.22",
  "private": true,
  "name": "simple",
  "dependencies": {
@@ -1,9 +1,9 @@
 import {
  Document,
-  ListIndex,
-  ListRetrieverMode,
-  serviceContextFromDefaults,
  SimpleNodeParser,
+  SummaryIndex,
+  SummaryRetrieverMode,
+  serviceContextFromDefaults,
 } from "llamaindex";
 import essay from "./essay";

@@ -14,9 +14,11 @@ async function main() {
    }),
  });
  const document = new Document({ text: essay, id_: "essay" });
-  const index = await ListIndex.fromDocuments([document], { serviceContext });
+  const index = await SummaryIndex.fromDocuments([document], {
+    serviceContext,
+  });
  const queryEngine = index.asQueryEngine({
-    retriever: index.asRetriever({ mode: ListRetrieverMode.LLM }),
+    retriever: index.asRetriever({ mode: SummaryRetrieverMode.LLM }),
  });
  const response = await queryEngine.query(
    "What did the author do growing up?",
@@ -12,7 +12,7 @@ async function main() {
  const document = new Document({ text: essay, id_: "essay" });

  const serviceContext = serviceContextFromDefaults({
-    llm: new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 }),
+    llm: new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 }),
  });

  const index = await VectorStoreIndex.fromDocuments([document], {
@@ -4,7 +4,6 @@ import {
  PapaCSVReader,
  ResponseSynthesizer,
  serviceContextFromDefaults,
-  SimplePrompt,
  VectorStoreIndex,
 } from "llamaindex";

@@ -23,9 +22,7 @@ async function main() {
    serviceContext,
  });

-  const csvPrompt: SimplePrompt = (input) => {
-    const { context = "", query = "" } = input;
-
+  const csvPrompt = ({ context = "", query = "" }) => {
    return `The following CSV file is loaded from ${path}
 \`\`\`csv
 ${context}
@@ -1,9 +1,9 @@
 import {
  Document,
-  TextNode,
  NodeWithScore,
  ResponseSynthesizer,
  SimpleNodeParser,
+  TextNode,
 } from "llamaindex";

 (async () => {
@@ -29,7 +29,7 @@ import {

  const response = await responseSynthesizer.synthesize(
    "What age am I?",
-    nodesWithScore
+    nodesWithScore,
  );
  console.log(response.response);
 })();
@@ -2,12 +2,14 @@ import { OpenAI } from "llamaindex";

 (async () => {
  const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 });
-  
+
  // complete api
  const response1 = await llm.complete("How are you?");
  console.log(response1.message.content);

  // chat api
-  const response2 = await llm.chat([{ content: "Tell me a joke!", role: "user" }]);
+  const response2 = await llm.chat([
+    { content: "Tell me a joke!", role: "user" },
+  ]);
  console.log(response2.message.content);
 })();
@@ -1,9 +1,9 @@
 import {
  Document,
-  ListIndex,
-  ListRetrieverMode,
-  serviceContextFromDefaults,
  SimpleNodeParser,
+  SummaryIndex,
+  SummaryRetrieverMode,
+  serviceContextFromDefaults,
 } from "llamaindex";
 import essay from "./essay";

@@ -14,9 +14,11 @@ async function main() {
    }),
  });
  const document = new Document({ text: essay, id_: "essay" });
-  const index = await ListIndex.fromDocuments([document], { serviceContext });
+  const index = await SummaryIndex.fromDocuments([document], {
+    serviceContext,
+  });
  const queryEngine = index.asQueryEngine({
-    retriever: index.asRetriever({ mode: ListRetrieverMode.LLM }),
+    retriever: index.asRetriever({ mode: SummaryRetrieverMode.LLM }),
  });
  const response = await queryEngine.query(
    "What did the author do growing up?",
@@ -12,7 +12,7 @@ async function main() {
  const document = new Document({ text: essay, id_: "essay" });

  const serviceContext = serviceContextFromDefaults({
-    llm: new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 }),
+    llm: new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 }),
  });

  const index = await VectorStoreIndex.fromDocuments([document], {
@@ -16,8 +16,8 @@
    "eslint": "^7.32.0",
    "eslint-config-custom": "workspace:*",
    "husky": "^8.0.3",
-    "jest": "^29.6.3",
-    "prettier": "^3.0.2",
+    "jest": "^29.6.4",
+    "prettier": "^3.0.3",
    "prettier-plugin-organize-imports": "^3.2.3",
    "ts-jest": "^29.1.1",
    "turbo": "^1.10.13"
@@ -1,5 +1,19 @@
 # llamaindex

+## 0.0.24
+
+### Patch Changes
+
+- e4af7b3: Renamed ListIndex to SummaryIndex to better indicate its use.
+- 259fe63: Strong types for prompts.
+
+## 0.0.23
+
+### Patch Changes
+
+- Added MetadataMode to ResponseSynthesizer (thanks @TomPenguin)
+- 9d6b2ed: Added Markdown Reader (huge shoutout to @swk777)
+
 ## 0.0.22

 ### Patch Changes
@@ -20,7 +20,7 @@ In a new folder:
 export OPENAI_API_KEY="sk-......" # Replace with your key from https://platform.openai.com/account/api-keys
 pnpm init
 pnpm install typescript
-pnpm exec tsc –-init # if needed
+pnpm exec tsc --init # if needed
 pnpm install llamaindex
 pnpm install @types/node
 ```
@@ -36,7 +36,7 @@ async function main() {
  // Load essay from abramov.txt in Node
  const essay = await fs.readFile(
    "node_modules/llamaindex/examples/abramov.txt",
-    "utf-8"
+    "utf-8",
  );

  // Create Document object with essay
@@ -48,7 +48,7 @@ async function main() {
  // Query the index
  const queryEngine = index.asQueryEngine();
  const response = await queryEngine.query(
-    "What did the author do in college?"
+    "What did the author do in college?",
  );

  // Output response
@@ -61,7 +61,7 @@ main();
 Then you can run it using

 ```bash
-pnpm dlx ts-node example.ts
+pnpx ts-node example.ts
 ```

 ## Playground
@@ -1,10 +1,10 @@
 {
  "name": "llamaindex",
-  "version": "0.0.22",
+  "version": "0.0.24",
  "dependencies": {
-    "@anthropic-ai/sdk": "^0.6.1",
+    "@anthropic-ai/sdk": "^0.6.2",
    "lodash": "^4.17.21",
-    "openai": "^4.2.0",
+    "openai": "^4.3.1",
    "papaparse": "^5.4.1",
    "pdf-parse": "^1.1.1",
    "replicate": "^0.16.1",
@@ -14,7 +14,7 @@
  },
  "devDependencies": {
    "@types/lodash": "^4.14.197",
-    "@types/node": "^18.17.9",
+    "@types/node": "^18.17.12",
    "@types/papaparse": "^5.3.8",
    "@types/pdf-parse": "^1.1.1",
    "@types/uuid": "^9.0.2",
@@ -1,17 +1,18 @@
-import { ChatMessage, OpenAI, ChatResponse, LLM } from "./llm/LLM";
+import { v4 as uuidv4 } from "uuid";
 import { TextNode } from "./Node";
 import {
-  SimplePrompt,
-  contextSystemPrompt,
+  CondenseQuestionPrompt,
+  ContextSystemPrompt,
  defaultCondenseQuestionPrompt,
+  defaultContextSystemPrompt,
  messagesToHistoryStr,
 } from "./Prompt";
 import { BaseQueryEngine } from "./QueryEngine";
 import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { v4 as uuidv4 } from "uuid";
 import { Event } from "./callbacks/CallbackManager";
+import { ChatMessage, LLM, OpenAI } from "./llm/LLM";

 /**
 * A ChatEngine is used to handle back and forth chats between the application and the LLM.
@@ -70,13 +71,13 @@ export class CondenseQuestionChatEngine implements ChatEngine {
  queryEngine: BaseQueryEngine;
  chatHistory: ChatMessage[];
  serviceContext: ServiceContext;
-  condenseMessagePrompt: SimplePrompt;
+  condenseMessagePrompt: CondenseQuestionPrompt;

  constructor(init: {
    queryEngine: BaseQueryEngine;
    chatHistory: ChatMessage[];
    serviceContext?: ServiceContext;
-    condenseMessagePrompt?: SimplePrompt;
+    condenseMessagePrompt?: CondenseQuestionPrompt;
  }) {
    this.queryEngine = init.queryEngine;
    this.chatHistory = init?.chatHistory ?? [];
@@ -92,14 +93,14 @@ export class CondenseQuestionChatEngine implements ChatEngine {
    return this.serviceContext.llm.complete(
      defaultCondenseQuestionPrompt({
        question: question,
-        chat_history: chatHistoryStr,
-      })
+        chatHistory: chatHistoryStr,
+      }),
    );
  }

  async chat(
    message: string,
-    chatHistory?: ChatMessage[] | undefined
+    chatHistory?: ChatMessage[] | undefined,
  ): Promise<Response> {
    chatHistory = chatHistory ?? this.chatHistory;

@@ -129,16 +130,20 @@ export class ContextChatEngine implements ChatEngine {
  retriever: BaseRetriever;
  chatModel: OpenAI;
  chatHistory: ChatMessage[];
+  contextSystemPrompt: ContextSystemPrompt;

  constructor(init: {
    retriever: BaseRetriever;
    chatModel?: OpenAI;
    chatHistory?: ChatMessage[];
+    contextSystemPrompt?: ContextSystemPrompt;
  }) {
    this.retriever = init.retriever;
    this.chatModel =
      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
    this.chatHistory = init?.chatHistory ?? [];
+    this.contextSystemPrompt =
+      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
  }

  async chat(message: string, chatHistory?: ChatMessage[] | undefined) {
@@ -151,11 +156,11 @@ export class ContextChatEngine implements ChatEngine {
    };
    const sourceNodesWithScore = await this.retriever.retrieve(
      message,
-      parentEvent
+      parentEvent,
    );

    const systemMessage: ChatMessage = {
-      content: contextSystemPrompt({
+      content: this.contextSystemPrompt({
        context: sourceNodesWithScore
          .map((r) => (r.node as TextNode).text)
          .join("\n\n"),
@@ -167,7 +172,7 @@ export class ContextChatEngine implements ChatEngine {

    const response = await this.chatModel.chat(
      [systemMessage, ...chatHistory],
-      parentEvent
+      parentEvent,
    );
    chatHistory.push(response.message);

@@ -175,7 +180,7 @@ export class ContextChatEngine implements ChatEngine {

    return new Response(
      response.message.content,
-      sourceNodesWithScore.map((r) => r.node)
+      sourceNodesWithScore.map((r) => r.node),
    );
  }

@@ -8,7 +8,7 @@ import {
  getAzureModel,
  shouldUseAzure,
 } from "./llm/azure";
-import { getOpenAISession, OpenAISession } from "./llm/openai";
+import { OpenAISession, getOpenAISession } from "./llm/openai";
 import { VectorStoreQueryMode } from "./storage/vectorStore/types";

 /**
@@ -280,9 +280,6 @@ export class OpenAIEmbedding extends BaseEmbedding {
  }

  private async getOpenAIEmbedding(input: string) {
-    input = input.replace(/\n/g, " ");
-    //^ NOTE this performance helper is in the OpenAI python library but may not be in the JS library
-
    const { data } = await this.session.openai.embeddings.create({
      model: this.model,
      input,
@@ -1,5 +1,5 @@
-import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
 import { v4 as uuidv4 } from "uuid";
+import { Event, EventTag, EventType } from "./callbacks/CallbackManager";

 /**
 * Helper class singleton
@@ -28,7 +28,7 @@ class OutputParserError extends Error {

  constructor(
    message: string,
-    options: { cause?: Error; output?: string } = {}
+    options: { cause?: Error; output?: string } = {},
  ) {
    // @ts-ignore
    super(message, options); // https://github.com/tc39/proposal-error-cause
@@ -62,7 +62,7 @@ function parseJsonMarkdown(text: string) {
  const beginIndex = text.indexOf(beginDelimiter);
  const endIndex = text.indexOf(
    endDelimiter,
-    beginIndex + beginDelimiter.length
+    beginIndex + beginDelimiter.length,
  );
  if (beginIndex === -1 || endIndex === -1) {
    throw new OutputParserError("Not a json markdown", { output: text });
@@ -7,7 +7,9 @@ import { ToolMetadata } from "./Tool";
 * NOTE this is a different interface compared to LlamaIndex Python
 * NOTE 2: we default to empty string to make it easy to calculate prompt sizes
 */
-export type SimplePrompt = (input: Record<string, string>) => string;
+export type SimplePrompt = (
+  input: Record<string, string | undefined>,
+) => string;

 /*
 DEFAULT_TEXT_QA_PROMPT_TMPL = (
@@ -22,9 +24,7 @@ DEFAULT_TEXT_QA_PROMPT_TMPL = (
 )
 */

-export const defaultTextQaPrompt: SimplePrompt = (input) => {
-  const { context = "", query = "" } = input;
-
+export const defaultTextQaPrompt = ({ context = "", query = "" }) => {
  return `Context information is below.
 ---------------------
 ${context}
@@ -34,6 +34,8 @@ Query: ${query}
 Answer:`;
 };

+export type TextQaPrompt = typeof defaultTextQaPrompt;
+
 /*
 DEFAULT_SUMMARY_PROMPT_TMPL = (
    "Write a summary of the following. Try to use only the "
@@ -48,9 +50,7 @@ DEFAULT_SUMMARY_PROMPT_TMPL = (
 )
 */

-export const defaultSummaryPrompt: SimplePrompt = (input) => {
-  const { context = "" } = input;
-
+export const defaultSummaryPrompt = ({ context = "" }) => {
  return `Write a summary of the following. Try to use only the information provided. Try to include as many key details as possible.


@@ -61,6 +61,8 @@ SUMMARY:"""
 `;
 };

+export type SummaryPrompt = typeof defaultSummaryPrompt;
+
 /*
 DEFAULT_REFINE_PROMPT_TMPL = (
    "The original query is as follows: {query_str}\n"
@@ -77,9 +79,11 @@ DEFAULT_REFINE_PROMPT_TMPL = (
 )
 */

-export const defaultRefinePrompt: SimplePrompt = (input) => {
-  const { query = "", existingAnswer = "", context = "" } = input;
-
+export const defaultRefinePrompt = ({
+  query = "",
+  existingAnswer = "",
+  context = "",
+}) => {
  return `The original query is as follows: ${query}
 We have provided an existing answer: ${existingAnswer}
 We have the opportunity to refine the existing answer (only if needed) with some more context below.
@@ -90,6 +94,8 @@ Given the new context, refine the original answer to better answer the query. If
 Refined Answer:`;
 };

+export type RefinePrompt = typeof defaultRefinePrompt;
+
 /*
 DEFAULT_TREE_SUMMARIZE_TMPL = (
  "Context information from multiple sources is below.\n"
@@ -103,9 +109,7 @@ DEFAULT_TREE_SUMMARIZE_TMPL = (
 )
 */

-export const defaultTreeSummarizePrompt: SimplePrompt = (input) => {
-  const { context = "", query = "" } = input;
-
+export const defaultTreeSummarizePrompt = ({ context = "", query = "" }) => {
  return `Context information from multiple sources is below.
 ---------------------
 ${context}
@@ -115,9 +119,9 @@ Query: ${query}
 Answer:`;
 };

-export const defaultChoiceSelectPrompt: SimplePrompt = (input) => {
-  const { context = "", query = "" } = input;
+export type TreeSummarizePrompt = typeof defaultTreeSummarizePrompt;

+export const defaultChoiceSelectPrompt = ({ context = "", query = "" }) => {
  return `A list of documents is shown below. Each document has a number next to it along 
 with a summary of the document. A question is also provided.
 Respond with the numbers of the documents
@@ -149,6 +153,8 @@ Question: ${query}
 Answer:`;
 };

+export type ChoiceSelectPrompt = typeof defaultChoiceSelectPrompt;
+
 /*
 PREFIX = """\
 Given a user question, and a list of tools, output a list of relevant sub-questions \
@@ -266,9 +272,7 @@ const exampleOutput: SubQuestion[] = [
  },
 ];

-export const defaultSubQuestionPrompt: SimplePrompt = (input) => {
-  const { toolsStr, queryStr } = input;
-
+export const defaultSubQuestionPrompt = ({ toolsStr = "", queryStr = "" }) => {
  return `Given a user question, and a list of tools, output a list of relevant sub-questions that when composed can help answer the full user question:

 # Example 1
@@ -298,6 +302,8 @@ ${queryStr}
 `;
 };

+export type SubQuestionPrompt = typeof defaultSubQuestionPrompt;
+
 // DEFAULT_TEMPLATE = """\
 // Given a conversation (between Human and Assistant) and a follow up message from Human, \
 // rewrite the message to be a standalone question that captures all relevant context \
@@ -312,9 +318,10 @@ ${queryStr}
 // <Standalone question>
 // """

-export const defaultCondenseQuestionPrompt: SimplePrompt = (input) => {
-  const { chatHistory, question } = input;
-
+export const defaultCondenseQuestionPrompt = ({
+  chatHistory = "",
+  question = "",
+}) => {
  return `Given a conversation (between Human and Assistant) and a follow up message from Human, rewrite the message to be a standalone question that captures all relevant context from the conversation.

 <Chat History>
@@ -327,6 +334,8 @@ ${question}
 `;
 };

+export type CondenseQuestionPrompt = typeof defaultCondenseQuestionPrompt;
+
 export function messagesToHistoryStr(messages: ChatMessage[]) {
  return messages.reduce((acc, message) => {
    acc += acc ? "\n" : "";
@@ -339,11 +348,11 @@ export function messagesToHistoryStr(messages: ChatMessage[]) {
  }, "");
 }

-export const contextSystemPrompt: SimplePrompt = (input) => {
-  const { context } = input;
-
+export const defaultContextSystemPrompt = ({ context = "" }) => {
  return `Context information is below.
 ---------------------
 ${context}
 ---------------------`;
 };
+
+export type ContextSystemPrompt = typeof defaultContextSystemPrompt;
@@ -1,3 +1,4 @@
+import { v4 as uuidv4 } from "uuid";
 import { NodeWithScore, TextNode } from "./Node";
 import {
  BaseQuestionGenerator,
@@ -7,10 +8,9 @@ import {
 import { Response } from "./Response";
 import { CompactAndRefine, ResponseSynthesizer } from "./ResponseSynthesizer";
 import { BaseRetriever } from "./Retriever";
-import { v4 as uuidv4 } from "uuid";
-import { Event } from "./callbacks/CallbackManager";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { QueryEngineTool, ToolMetadata } from "./Tool";
+import { Event } from "./callbacks/CallbackManager";

 /**
 * A query engine is a question answerer that can use one or more steps.
@@ -33,7 +33,7 @@ export class RetrieverQueryEngine implements BaseQueryEngine {

  constructor(
    retriever: BaseRetriever,
-    responseSynthesizer?: ResponseSynthesizer
+    responseSynthesizer?: ResponseSynthesizer,
  ) {
    this.retriever = retriever;
    const serviceContext: ServiceContext | undefined =
@@ -122,7 +122,7 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
    };

    const subQNodes = await Promise.all(
-      subQuestions.map((subQ) => this.querySubQ(subQ, subQueryParentEvent))
+      subQuestions.map((subQ) => this.querySubQ(subQ, subQueryParentEvent)),
    );

    const nodes = subQNodes
@@ -133,7 +133,7 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {

  private async querySubQ(
    subQ: SubQuestion,
-    parentEvent?: Event
+    parentEvent?: Event,
  ): Promise<NodeWithScore | null> {
    try {
      const question = subQ.subQuestion;
@@ -4,7 +4,7 @@ import {
  SubQuestionOutputParser,
 } from "./OutputParser";
 import {
-  SimplePrompt,
+  SubQuestionPrompt,
  buildToolsText,
  defaultSubQuestionPrompt,
 } from "./Prompt";
@@ -28,7 +28,7 @@ export interface BaseQuestionGenerator {
 */
 export class LLMQuestionGenerator implements BaseQuestionGenerator {
  llm: LLM;
-  prompt: SimplePrompt;
+  prompt: SubQuestionPrompt;
  outputParser: BaseOutputParser<StructuredOutput<SubQuestion[]>>;

  constructor(init?: Partial<LLMQuestionGenerator>) {
@@ -45,7 +45,7 @@ export class LLMQuestionGenerator implements BaseQuestionGenerator {
        this.prompt({
          toolsStr,
          queryStr,
-        })
+        }),
      )
    ).message.content;

@@ -1,6 +1,9 @@
 import { MetadataMode, NodeWithScore } from "./Node";
 import {
+  RefinePrompt,
  SimplePrompt,
+  TextQaPrompt,
+  TreeSummarizePrompt,
  defaultRefinePrompt,
  defaultTextQaPrompt,
  defaultTreeSummarizePrompt,
@@ -73,13 +76,13 @@ export class SimpleResponseBuilder implements BaseResponseBuilder {
 */
 export class Refine implements BaseResponseBuilder {
  serviceContext: ServiceContext;
-  textQATemplate: SimplePrompt;
-  refineTemplate: SimplePrompt;
+  textQATemplate: TextQaPrompt;
+  refineTemplate: RefinePrompt;

  constructor(
    serviceContext: ServiceContext,
-    textQATemplate?: SimplePrompt,
-    refineTemplate?: SimplePrompt,
+    textQATemplate?: TextQaPrompt,
+    refineTemplate?: RefinePrompt,
  ) {
    this.serviceContext = serviceContext;
    this.textQATemplate = textQATemplate ?? defaultTextQaPrompt;
@@ -209,9 +212,14 @@ export class CompactAndRefine extends Refine {
 */
 export class TreeSummarize implements BaseResponseBuilder {
  serviceContext: ServiceContext;
+  summaryTemplate: TreeSummarizePrompt;

-  constructor(serviceContext: ServiceContext) {
+  constructor(
+    serviceContext: ServiceContext,
+    summaryTemplate?: TreeSummarizePrompt,
+  ) {
    this.serviceContext = serviceContext;
+    this.summaryTemplate = summaryTemplate ?? defaultTreeSummarizePrompt;
  }

  async getResponse(
@@ -219,21 +227,19 @@ export class TreeSummarize implements BaseResponseBuilder {
    textChunks: string[],
    parentEvent?: Event,
  ): Promise<string> {
-    const summaryTemplate: SimplePrompt = defaultTreeSummarizePrompt;
-
    if (!textChunks || textChunks.length === 0) {
      throw new Error("Must have at least one text chunk");
    }

    const packedTextChunks = this.serviceContext.promptHelper.repack(
-      summaryTemplate,
+      this.summaryTemplate,
      textChunks,
    );

    if (packedTextChunks.length === 1) {
      return (
        await this.serviceContext.llm.complete(
-          summaryTemplate({
+          this.summaryTemplate({
            context: packedTextChunks[0],
          }),
          parentEvent,
@@ -243,7 +249,7 @@ export class TreeSummarize implements BaseResponseBuilder {
      const summaries = await Promise.all(
        packedTextChunks.map((chunk) =>
          this.serviceContext.llm.complete(
-            summaryTemplate({
+            this.summaryTemplate({
              context: chunk,
            }),
            parentEvent,
@@ -298,9 +304,13 @@ export class ResponseSynthesizer {
    this.metadataMode = metadataMode;
  }

-  async synthesize(query: string, nodes: NodeWithScore[], parentEvent?: Event) {
-    let textChunks: string[] = nodes.map((node) =>
-      node.node.getContent(this.metadataMode)
+  async synthesize(
+    query: string,
+    nodesWithScore: NodeWithScore[],
+    parentEvent?: Event,
+  ) {
+    let textChunks: string[] = nodesWithScore.map(({ node }) =>
+      node.getContent(this.metadataMode),
    );
    const response = await this.responseBuilder.getResponse(
      query,
@@ -309,7 +319,7 @@ export class ResponseSynthesizer {
    );
    return new Response(
      response,
-      nodes.map((node) => node.node),
+      nodesWithScore.map(({ node }) => node),
    );
  }
 }
@@ -1,8 +1,8 @@
 import { BaseEmbedding, OpenAIEmbedding } from "./Embedding";
-import { LLM, OpenAI } from "./llm/LLM";
 import { NodeParser, SimpleNodeParser } from "./NodeParser";
 import { PromptHelper } from "./PromptHelper";
 import { CallbackManager } from "./callbacks/CallbackManager";
+import { LLM, OpenAI } from "./llm/LLM";

 /**
 * The ServiceContext is a collection of components that are used in different parts of the application.
@@ -47,7 +47,7 @@ export function serviceContextFromDefaults(options?: ServiceContextOptions) {

 export function serviceContextFromServiceContext(
  serviceContext: ServiceContext,
-  options: ServiceContextOptions
+  options: ServiceContextOptions,
 ) {
  const newServiceContext = { ...serviceContext };
  if (options.llm) {
@@ -1,29 +1,29 @@
 export * from "./ChatEngine";
-export * from "./constants";
 export * from "./Embedding";
 export * from "./GlobalsHelper";
-export * from "./llm/LLM";
 export * from "./Node";
 export * from "./NodeParser";
 export * from "./OutputParser";
 export * from "./Prompt";
-export * from "./QuestionGenerator";
 export * from "./QueryEngine";
+export * from "./QuestionGenerator";
 export * from "./Response";
 export * from "./ResponseSynthesizer";
 export * from "./Retriever";
 export * from "./ServiceContext";
 export * from "./TextSplitter";
 export * from "./Tool";
+export * from "./constants";
+export * from "./llm/LLM";

 export * from "./indices";

 export * from "./callbacks/CallbackManager";

-export * from "./readers/base";
-export * from "./readers/PDFReader";
 export * from "./readers/CSVReader";
 export * from "./readers/MarkdownReader";
+export * from "./readers/PDFReader";
 export * from "./readers/SimpleDirectoryReader";
+export * from "./readers/base";

 export * from "./storage";
@@ -4,9 +4,9 @@ import { BaseQueryEngine } from "../QueryEngine";
 import { ResponseSynthesizer } from "../ResponseSynthesizer";
 import { BaseRetriever } from "../Retriever";
 import { ServiceContext } from "../ServiceContext";
+import { StorageContext } from "../storage/StorageContext";
 import { BaseDocumentStore } from "../storage/docStore/types";
 import { BaseIndexStore } from "../storage/indexStore/types";
-import { StorageContext } from "../storage/StorageContext";
 import { VectorStore } from "../storage/vectorStore/types";

 /**
@@ -1,3 +1,3 @@
 export * from "./BaseIndex";
-export * from "./list";
+export * from "./summary";
 export * from "./vectorStore";
@@ -1,5 +0,0 @@
-export { ListIndex, ListRetrieverMode } from "./ListIndex";
-export {
-  ListIndexRetriever,
-  ListIndexLLMRetriever,
-} from "./ListIndexRetriever";
@@ -10,11 +10,11 @@ import {
  ServiceContext,
  serviceContextFromDefaults,
 } from "../../ServiceContext";
-import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
  StorageContext,
  storageContextFromDefaults,
 } from "../../storage/StorageContext";
+import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
  BaseIndex,
  BaseIndexInit,
@@ -22,17 +22,17 @@ import {
  IndexStructType,
 } from "../BaseIndex";
 import {
-  ListIndexLLMRetriever,
-  ListIndexRetriever,
-} from "./ListIndexRetriever";
+  SummaryIndexLLMRetriever,
+  SummaryIndexRetriever,
+} from "./SummaryIndexRetriever";

-export enum ListRetrieverMode {
+export enum SummaryRetrieverMode {
  DEFAULT = "default",
  // EMBEDDING = "embedding",
  LLM = "llm",
 }

-export interface ListIndexOptions {
+export interface SummaryIndexOptions {
  nodes?: BaseNode[];
  indexStruct?: IndexList;
  indexId?: string;
@@ -41,14 +41,14 @@ export interface ListIndexOptions {
 }

 /**
- * A ListIndex keeps nodes in a sequential list structure
+ * A SummaryIndex keeps nodes in a sequential order for use with summarization.
 */
-export class ListIndex extends BaseIndex<IndexList> {
+export class SummaryIndex extends BaseIndex<IndexList> {
  constructor(init: BaseIndexInit<IndexList>) {
    super(init);
  }

-  static async init(options: ListIndexOptions): Promise<ListIndex> {
+  static async init(options: SummaryIndexOptions): Promise<SummaryIndex> {
    const storageContext =
      options.storageContext ?? (await storageContextFromDefaults({}));
    const serviceContext =
@@ -80,23 +80,23 @@ export class ListIndex extends BaseIndex<IndexList> {
    // check indexStruct type
    if (indexStruct && indexStruct.type !== IndexStructType.LIST) {
      throw new Error(
-        "Attempting to initialize ListIndex with non-list indexStruct",
+        "Attempting to initialize SummaryIndex with non-list indexStruct",
      );
    }

    if (indexStruct) {
      if (options.nodes) {
        throw new Error(
-          "Cannot initialize VectorStoreIndex with both nodes and indexStruct",
+          "Cannot initialize SummaryIndex with both nodes and indexStruct",
        );
      }
    } else {
      if (!options.nodes) {
        throw new Error(
-          "Cannot initialize VectorStoreIndex without nodes or indexStruct",
+          "Cannot initialize SummaryIndex without nodes or indexStruct",
        );
      }
-      indexStruct = await ListIndex.buildIndexFromNodes(
+      indexStruct = await SummaryIndex.buildIndexFromNodes(
        options.nodes,
        storageContext.docStore,
      );
@@ -104,7 +104,7 @@ export class ListIndex extends BaseIndex<IndexList> {
      await indexStore.addIndexStruct(indexStruct);
    }

-    return new ListIndex({
+    return new SummaryIndex({
      storageContext,
      serviceContext,
      docStore,
@@ -119,7 +119,7 @@ export class ListIndex extends BaseIndex<IndexList> {
      storageContext?: StorageContext;
      serviceContext?: ServiceContext;
    } = {},
-  ): Promise<ListIndex> {
+  ): Promise<SummaryIndex> {
    let { storageContext, serviceContext } = args;
    storageContext = storageContext ?? (await storageContextFromDefaults({}));
    serviceContext = serviceContext ?? serviceContextFromDefaults({});
@@ -131,7 +131,7 @@ export class ListIndex extends BaseIndex<IndexList> {
    }

    const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
-    const index = await ListIndex.init({
+    const index = await SummaryIndex.init({
      nodes,
      storageContext,
      serviceContext,
@@ -139,14 +139,14 @@ export class ListIndex extends BaseIndex<IndexList> {
    return index;
  }

-  asRetriever(options?: { mode: ListRetrieverMode }): BaseRetriever {
-    const { mode = ListRetrieverMode.DEFAULT } = options ?? {};
+  asRetriever(options?: { mode: SummaryRetrieverMode }): BaseRetriever {
+    const { mode = SummaryRetrieverMode.DEFAULT } = options ?? {};

    switch (mode) {
-      case ListRetrieverMode.DEFAULT:
-        return new ListIndexRetriever(this);
-      case ListRetrieverMode.LLM:
-        return new ListIndexLLMRetriever(this);
+      case SummaryRetrieverMode.DEFAULT:
+        return new SummaryIndexRetriever(this);
+      case SummaryRetrieverMode.LLM:
+        return new SummaryIndexLLMRetriever(this);
      default:
        throw new Error(`Unknown retriever mode: ${mode}`);
    }
@@ -253,4 +253,5 @@ export class ListIndex extends BaseIndex<IndexList> {
 }

 // Legacy
-export type GPTListIndex = ListIndex;
+export type ListIndex = SummaryIndex;
+export type ListRetrieverMode = SummaryRetrieverMode;
@@ -1,25 +1,25 @@
-import { BaseRetriever } from "../../Retriever";
+import _ from "lodash";
+import { globalsHelper } from "../../GlobalsHelper";
 import { NodeWithScore } from "../../Node";
-import { ListIndex } from "./ListIndex";
+import { ChoiceSelectPrompt, defaultChoiceSelectPrompt } from "../../Prompt";
+import { BaseRetriever } from "../../Retriever";
 import { ServiceContext } from "../../ServiceContext";
+import { Event } from "../../callbacks/CallbackManager";
+import { SummaryIndex } from "./SummaryIndex";
 import {
-  NodeFormatterFunction,
  ChoiceSelectParserFunction,
+  NodeFormatterFunction,
  defaultFormatNodeBatchFn,
  defaultParseChoiceSelectAnswerFn,
 } from "./utils";
-import { SimplePrompt, defaultChoiceSelectPrompt } from "../../Prompt";
-import _ from "lodash";
-import { globalsHelper } from "../../GlobalsHelper";
-import { Event } from "../../callbacks/CallbackManager";

 /**
- * Simple retriever for ListIndex that returns all nodes
+ * Simple retriever for SummaryIndex that returns all nodes
 */
-export class ListIndexRetriever implements BaseRetriever {
-  index: ListIndex;
+export class SummaryIndexRetriever implements BaseRetriever {
+  index: SummaryIndex;

-  constructor(index: ListIndex) {
+  constructor(index: SummaryIndex) {
    this.index = index;
  }

@@ -51,23 +51,23 @@ export class ListIndexRetriever implements BaseRetriever {
 }

 /**
- * LLM retriever for ListIndex.
+ * LLM retriever for SummaryIndex which lets you select the most relevant chunks.
 */
-export class ListIndexLLMRetriever implements BaseRetriever {
-  index: ListIndex;
-  choiceSelectPrompt: SimplePrompt;
+export class SummaryIndexLLMRetriever implements BaseRetriever {
+  index: SummaryIndex;
+  choiceSelectPrompt: ChoiceSelectPrompt;
  choiceBatchSize: number;
  formatNodeBatchFn: NodeFormatterFunction;
  parseChoiceSelectAnswerFn: ChoiceSelectParserFunction;
  serviceContext: ServiceContext;

  constructor(
-    index: ListIndex,
-    choiceSelectPrompt?: SimplePrompt,
+    index: SummaryIndex,
+    choiceSelectPrompt?: ChoiceSelectPrompt,
    choiceBatchSize: number = 10,
    formatNodeBatchFn?: NodeFormatterFunction,
    parseChoiceSelectAnswerFn?: ChoiceSelectParserFunction,
-    serviceContext?: ServiceContext
+    serviceContext?: ServiceContext,
  ) {
    this.index = index;
    this.choiceSelectPrompt = choiceSelectPrompt || defaultChoiceSelectPrompt;
@@ -95,7 +95,7 @@ export class ListIndexLLMRetriever implements BaseRetriever {
      // parseResult is a map from doc number to relevance score
      const parseResult = this.parseChoiceSelectAnswerFn(
        rawResponse,
-        nodesBatch.length
+        nodesBatch.length,
      );
      const choiceNodeIds = nodeIdsBatch.filter((nodeId, idx) => {
        return `${idx}` in parseResult;
@@ -128,3 +128,7 @@ export class ListIndexLLMRetriever implements BaseRetriever {
    return this.serviceContext;
  }
 }
+
+// Legacy
+export type ListIndexRetriever = SummaryIndexRetriever;
+export type ListIndexLLMRetriever = SummaryIndexLLMRetriever;
@@ -0,0 +1,10 @@
+export { SummaryIndex, SummaryRetrieverMode } from "./SummaryIndex";
+export type { ListIndex, ListRetrieverMode } from "./SummaryIndex";
+export {
+  SummaryIndexLLMRetriever,
+  SummaryIndexRetriever,
+} from "./SummaryIndexRetriever";
+export type {
+  ListIndexLLMRetriever,
+  ListIndexRetriever,
+} from "./SummaryIndexRetriever";
@@ -1,9 +1,9 @@
-import { BaseNode, MetadataMode } from "../../Node";
 import _ from "lodash";
+import { BaseNode, MetadataMode } from "../../Node";

 export type NodeFormatterFunction = (summaryNodes: BaseNode[]) => string;
 export const defaultFormatNodeBatchFn: NodeFormatterFunction = (
-  summaryNodes: BaseNode[]
+  summaryNodes: BaseNode[],
 ): string => {
  return summaryNodes
    .map((node, idx) => {
@@ -20,13 +20,13 @@ export type ChoiceSelectParseResult = { [docNumber: number]: number };
 export type ChoiceSelectParserFunction = (
  answer: string,
  numChoices: number,
-  raiseErr?: boolean
+  raiseErr?: boolean,
 ) => ChoiceSelectParseResult;

 export const defaultParseChoiceSelectAnswerFn: ChoiceSelectParserFunction = (
  answer: string,
  numChoices: number,
-  raiseErr: boolean = false
+  raiseErr: boolean = false,
 ): ChoiceSelectParseResult => {
  // split the line into the answer number and relevance score portions
  const lineTokens: string[][] = answer
@@ -36,7 +36,7 @@ export const defaultParseChoiceSelectAnswerFn: ChoiceSelectParserFunction = (
      if (lineTokens.length !== 2) {
        if (raiseErr) {
          throw new Error(
-            `Invalid answer line: ${line}. Answer line must be of the form: answer_num: <int>, answer_relevance: <float>`
+            `Invalid answer line: ${line}. Answer line must be of the form: answer_num: <int>, answer_relevance: <float>`,
          );
        } else {
          return null;
@@ -55,7 +55,7 @@ export const defaultParseChoiceSelectAnswerFn: ChoiceSelectParserFunction = (
        if (docNum < 1 || docNum > numChoices) {
          if (raiseErr) {
            throw new Error(
-              `Invalid answer number: ${docNum}. Answer number must be between 1 and ${numChoices}`
+              `Invalid answer number: ${docNum}. Answer number must be between 1 and ${numChoices}`,
            );
          }
        } else {
@@ -68,6 +68,6 @@ export const defaultParseChoiceSelectAnswerFn: ChoiceSelectParserFunction = (
      }
      return parseResult;
    },
-    {}
+    {},
  );
 };
@@ -1,6 +1,6 @@
-import { VectorStoreIndex } from "./VectorStoreIndex";
 import { globalsHelper } from "../../GlobalsHelper";
 import { NodeWithScore } from "../../Node";
+import { BaseRetriever } from "../../Retriever";
 import { ServiceContext } from "../../ServiceContext";
 import { Event } from "../../callbacks/CallbackManager";
 import { DEFAULT_SIMILARITY_TOP_K } from "../../constants";
@@ -8,7 +8,7 @@ import {
  VectorStoreQuery,
  VectorStoreQueryMode,
 } from "../../storage/vectorStore/types";
-import { BaseRetriever } from "../../Retriever";
+import { VectorStoreIndex } from "./VectorStoreIndex";

 /**
 * VectorIndexRetriever retrieves nodes from a VectorIndex.
@@ -1,2 +1,2 @@
-export { VectorStoreIndex } from "./VectorStoreIndex";
 export { VectorIndexRetriever } from "./VectorIndexRetriever";
+export { VectorStoreIndex } from "./VectorStoreIndex";
@@ -1,6 +1,6 @@
 import Anthropic, {
-  ClientOptions,
  AI_PROMPT,
+  ClientOptions,
  HUMAN_PROMPT,
 } from "@anthropic-ai/sdk";
 import _ from "lodash";
@@ -38,7 +38,7 @@ const DEFAULT_API_VERSION = "2023-05-15";
 //^ NOTE: this will change over time, if you want to pin it, use a specific version

 export function getAzureConfigFromEnv(
-  init?: Partial<AzureOpenAIConfig> & { model?: string }
+  init?: Partial<AzureOpenAIConfig> & { model?: string },
 ): AzureOpenAIConfig {
  return {
    apiKey:
@@ -71,7 +71,7 @@ export function getAzureBaseUrl(config: AzureOpenAIConfig): string {

 export function getAzureModel(openAIModel: string) {
  for (const [key, value] of Object.entries(
-    ALL_AZURE_OPENAI_EMBEDDING_MODELS
+    ALL_AZURE_OPENAI_EMBEDDING_MODELS,
  )) {
    if (value.openAIModel === openAIModel) {
      return key;
@@ -1,5 +1,5 @@
-import OpenAI, { ClientOptions } from "openai";
 import _ from "lodash";
+import OpenAI, { ClientOptions } from "openai";

 export class AzureOpenAI extends OpenAI {
  protected override authHeaders() {
@@ -42,7 +42,7 @@ let defaultOpenAISession: { session: OpenAISession; options: ClientOptions }[] =
 * @returns
 */
 export function getOpenAISession(
-  options: ClientOptions & { azure?: boolean } = {}
+  options: ClientOptions & { azure?: boolean } = {},
 ) {
  let session = defaultOpenAISession.find((session) => {
    return _.isEqual(session.options, options);
@@ -11,7 +11,7 @@ export class ReplicateSession {
      this.replicateKey = process.env.REPLICATE_API_TOKEN;
    } else {
      throw new Error(
-        "Set Replicate token in REPLICATE_API_TOKEN env variable"
+        "Set Replicate token in REPLICATE_API_TOKEN env variable",
      );
    }

@@ -1,7 +1,7 @@
-import { DEFAULT_FS, GenericFileSystem } from "../storage/FileSystem";
 import Papa, { ParseConfig } from "papaparse";
-import { BaseReader } from "./base";
 import { Document } from "../Node";
+import { DEFAULT_FS, GenericFileSystem } from "../storage/FileSystem";
+import { BaseReader } from "./base";

 /**
 * papaparse-based csv parser
@@ -24,7 +24,7 @@ export class PapaCSVReader implements BaseReader {
    concatRows: boolean = true,
    colJoiner: string = ", ",
    rowJoiner: string = "\n",
-    papaConfig?: ParseConfig
+    papaConfig?: ParseConfig,
  ) {
    this.concatRows = concatRows;
    this.colJoiner = colJoiner;
@@ -40,7 +40,7 @@ export class PapaCSVReader implements BaseReader {
   */
  async loadData(
    file: string,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
  ): Promise<Document[]> {
    const fileContent: string = await fs.readFile(file, "utf-8");
    const result = Papa.parse(fileContent, this.papaConfig);
@@ -1,8 +1,8 @@
+import pdfParse from "pdf-parse";
 import { Document } from "../Node";
-import { BaseReader } from "./base";
 import { GenericFileSystem } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
-import pdfParse from "pdf-parse";
+import { BaseReader } from "./base";

 /**
 * Read the text of a PDF
@@ -10,7 +10,7 @@ import pdfParse from "pdf-parse";
 export class PDFReader implements BaseReader {
  async loadData(
    file: string,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
  ): Promise<Document[]> {
    const dataBuffer = (await fs.readFile(file)) as any;
    const data = await pdfParse(dataBuffer);
@@ -1,11 +1,11 @@
 import _ from "lodash";
 import { Document } from "../Node";
-import { BaseReader } from "./base";
 import { CompleteFileSystem, walk } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
-import { PDFReader } from "./PDFReader";
 import { PapaCSVReader } from "./CSVReader";
 import { MarkdownReader } from "./MarkdownReader";
+import { PDFReader } from "./PDFReader";
+import { BaseReader } from "./base";

 /**
 * Read a .txt file
@@ -13,7 +13,7 @@ import { MarkdownReader } from "./MarkdownReader";
 export class TextFileReader implements BaseReader {
  async loadData(
    file: string,
-    fs: CompleteFileSystem = DEFAULT_FS as CompleteFileSystem
+    fs: CompleteFileSystem = DEFAULT_FS as CompleteFileSystem,
  ): Promise<Document[]> {
    const dataBuffer = await fs.readFile(file, "utf-8");
    return [new Document({ text: dataBuffer, id_: file })];
@@ -73,7 +73,7 @@ export const DEFAULT_FS: GenericFileSystem | CompleteFileSystem =
 */
 export async function exists(
  fs: GenericFileSystem,
-  path: string
+  path: string,
 ): Promise<boolean> {
  try {
    await fs.access(path);
@@ -90,11 +90,11 @@ export async function exists(
 */
 export async function* walk(
  fs: WalkableFileSystem,
-  dirPath: string
+  dirPath: string,
 ): AsyncIterable<string> {
  if (fs instanceof InMemoryFileSystem) {
    throw new Error(
-      "The InMemoryFileSystem does not support directory traversal."
+      "The InMemoryFileSystem does not support directory traversal.",
    );
  }

@@ -1,15 +1,11 @@
-import { BaseDocumentStore } from "./docStore/types";
-import { BaseIndexStore } from "./indexStore/types";
-import { VectorStore } from "./vectorStore/types";
-import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore";
-import { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
-import { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
 import { GenericFileSystem } from "./FileSystem";
-import {
-  DEFAULT_PERSIST_DIR,
-  DEFAULT_FS,
-  DEFAULT_NAMESPACE,
-} from "./constants";
+import { DEFAULT_FS, DEFAULT_NAMESPACE } from "./constants";
+import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore";
+import { BaseDocumentStore } from "./docStore/types";
+import { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
+import { BaseIndexStore } from "./indexStore/types";
+import { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
+import { VectorStore } from "./vectorStore/types";

 export interface StorageContext {
  docStore: BaseDocumentStore;
@@ -43,7 +39,7 @@ export async function storageContextFromDefaults({
      (await SimpleDocumentStore.fromPersistDir(
        persistDir,
        DEFAULT_NAMESPACE,
-        fs
+        fs,
      ));
    indexStore =
      indexStore || (await SimpleIndexStore.fromPersistDir(persistDir, fs));
@@ -1,15 +1,15 @@
-import * as path from "path";
 import _ from "lodash";
-import { KVDocumentStore } from "./KVDocumentStore";
-import { SimpleKVStore } from "../kvStore/SimpleKVStore";
-import { BaseInMemoryKVStore } from "../kvStore/types";
+import * as path from "path";
 import { GenericFileSystem } from "../FileSystem";
 import {
-  DEFAULT_PERSIST_DIR,
-  DEFAULT_NAMESPACE,
  DEFAULT_DOC_STORE_PERSIST_FILENAME,
  DEFAULT_FS,
+  DEFAULT_NAMESPACE,
+  DEFAULT_PERSIST_DIR,
 } from "../constants";
+import { SimpleKVStore } from "../kvStore/SimpleKVStore";
+import { BaseInMemoryKVStore } from "../kvStore/types";
+import { KVDocumentStore } from "./KVDocumentStore";

 type SaveDict = Record<string, any>;

@@ -26,23 +26,23 @@ export class SimpleDocumentStore extends KVDocumentStore {
  static async fromPersistDir(
    persistDir: string = DEFAULT_PERSIST_DIR,
    namespace?: string,
-    fsModule?: GenericFileSystem
+    fsModule?: GenericFileSystem,
  ): Promise<SimpleDocumentStore> {
    const persistPath = path.join(
      persistDir,
-      DEFAULT_DOC_STORE_PERSIST_FILENAME
+      DEFAULT_DOC_STORE_PERSIST_FILENAME,
    );
    return await SimpleDocumentStore.fromPersistPath(
      persistPath,
      namespace,
-      fsModule
+      fsModule,
    );
  }

  static async fromPersistPath(
    persistPath: string,
    namespace?: string,
-    fs?: GenericFileSystem
+    fs?: GenericFileSystem,
  ): Promise<SimpleDocumentStore> {
    fs = fs || DEFAULT_FS;
    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
@@ -52,9 +52,9 @@ export class SimpleDocumentStore extends KVDocumentStore {
  async persist(
    persistPath: string = path.join(
      DEFAULT_PERSIST_DIR,
-      DEFAULT_DOC_STORE_PERSIST_FILENAME
+      DEFAULT_DOC_STORE_PERSIST_FILENAME,
    ),
-    fs?: GenericFileSystem
+    fs?: GenericFileSystem,
  ): Promise<void> {
    fs = fs || DEFAULT_FS;
    if (
@@ -1,4 +1,4 @@
-import { BaseNode, Document, TextNode, ObjectType } from "../../Node";
+import { BaseNode, Document, ObjectType, TextNode } from "../../Node";

 const TYPE_KEY = "__type__";
 const DATA_KEY = "__data__";
@@ -1,7 +1,11 @@
-export * from "./constants";
 export * from "./FileSystem";
 export * from "./StorageContext";
-export * from "./vectorStore/types";
+export * from "./constants";
 export { SimpleDocumentStore } from "./docStore/SimpleDocumentStore";
+export * from "./docStore/types";
 export { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
+export * from "./indexStore/types";
+export { SimpleKVStore } from "./kvStore/SimpleKVStore";
+export * from "./kvStore/types";
 export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
+export * from "./vectorStore/types";
@@ -1,7 +1,7 @@
-import { BaseKVStore } from "../kvStore/types";
-import { IndexStruct, jsonToIndexStruct } from "../../indices/BaseIndex";
 import _ from "lodash";
+import { IndexStruct, jsonToIndexStruct } from "../../indices/BaseIndex";
 import { DEFAULT_NAMESPACE } from "../constants";
+import { BaseKVStore } from "../kvStore/types";
 import { BaseIndexStore } from "./types";

 export class KVIndexStore extends BaseIndexStore {
@@ -1,14 +1,13 @@
 import * as path from "path";
-import * as _ from "lodash";
-import { BaseInMemoryKVStore } from "../kvStore/types";
-import { SimpleKVStore, DataType } from "../kvStore/SimpleKVStore";
-import { KVIndexStore } from "./KVIndexStore";
-import {
-  DEFAULT_PERSIST_DIR,
-  DEFAULT_INDEX_STORE_PERSIST_FILENAME,
-  DEFAULT_FS,
-} from "../constants";
 import { GenericFileSystem } from "../FileSystem";
+import {
+  DEFAULT_FS,
+  DEFAULT_INDEX_STORE_PERSIST_FILENAME,
+  DEFAULT_PERSIST_DIR,
+} from "../constants";
+import { DataType, SimpleKVStore } from "../kvStore/SimpleKVStore";
+import { BaseInMemoryKVStore } from "../kvStore/types";
+import { KVIndexStore } from "./KVIndexStore";

 export class SimpleIndexStore extends KVIndexStore {
  private kvStore: BaseInMemoryKVStore;
@@ -21,18 +20,18 @@ export class SimpleIndexStore extends KVIndexStore {

  static async fromPersistDir(
    persistDir: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
  ): Promise<SimpleIndexStore> {
    const persistPath = path.join(
      persistDir,
-      DEFAULT_INDEX_STORE_PERSIST_FILENAME
+      DEFAULT_INDEX_STORE_PERSIST_FILENAME,
    );
    return this.fromPersistPath(persistPath, fs);
  }

  static async fromPersistPath(
    persistPath: string,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
  ): Promise<SimpleIndexStore> {
    let simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
    return new SimpleIndexStore(simpleKVStore);
@@ -40,7 +39,7 @@ export class SimpleIndexStore extends KVIndexStore {

  async persist(
    persistPath: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = DEFAULT_FS
+    fs: GenericFileSystem = DEFAULT_FS,
  ): Promise<void> {
    await this.kvStore.persist(persistPath, fs);
  }
@@ -1,8 +1,8 @@
 import { IndexStruct } from "../../indices/BaseIndex";
 import { GenericFileSystem } from "../FileSystem";
 import {
-  DEFAULT_PERSIST_DIR,
  DEFAULT_INDEX_STORE_PERSIST_FILENAME,
+  DEFAULT_PERSIST_DIR,
 } from "../constants";

 const defaultPersistPath = `${DEFAULT_PERSIST_DIR}/${DEFAULT_INDEX_STORE_PERSIST_FILENAME}`;
@@ -18,7 +18,7 @@ export abstract class BaseIndexStore {

  async persist(
    persistPath: string = defaultPersistPath,
-    fs?: GenericFileSystem
+    fs?: GenericFileSystem,
  ): Promise<void> {
    // Persist the index store to disk.
  }
@@ -1,7 +1,7 @@
+import * as _ from "lodash";
 import * as path from "path";
 import { GenericFileSystem, exists } from "../FileSystem";
 import { DEFAULT_COLLECTION, DEFAULT_FS } from "../constants";
-import * as _ from "lodash";
 import { BaseKVStore } from "./types";

 export type DataType = Record<string, Record<string, any>>;
@@ -19,7 +19,7 @@ export class SimpleKVStore extends BaseKVStore {
  async put(
    key: string,
    val: any,
-    collection: string = DEFAULT_COLLECTION
+    collection: string = DEFAULT_COLLECTION,
  ): Promise<void> {
    if (!(collection in this.data)) {
      this.data[collection] = {};
@@ -33,7 +33,7 @@ export class SimpleKVStore extends BaseKVStore {

  async get(
    key: string,
-    collection: string = DEFAULT_COLLECTION
+    collection: string = DEFAULT_COLLECTION,
  ): Promise<any> {
    let collectionData = this.data[collection];
    if (_.isNil(collectionData)) {
@@ -51,7 +51,7 @@ export class SimpleKVStore extends BaseKVStore {

  async delete(
    key: string,
-    collection: string = DEFAULT_COLLECTION
+    collection: string = DEFAULT_COLLECTION,
  ): Promise<boolean> {
    if (key in this.data[collection]) {
      delete this.data[collection][key];
@@ -72,7 +72,7 @@ export class SimpleKVStore extends BaseKVStore {

  static async fromPersistPath(
    persistPath: string,
-    fs?: GenericFileSystem
+    fs?: GenericFileSystem,
  ): Promise<SimpleKVStore> {
    fs = fs || DEFAULT_FS;
    let dirPath = path.dirname(persistPath);
@@ -86,7 +86,7 @@ export class SimpleKVStore extends BaseKVStore {
      data = JSON.parse(fileData.toString());
    } catch (e) {
      console.error(
-        `No valid data found at path: ${persistPath} starting new store.`
+        `No valid data found at path: ${persistPath} starting new store.`,
      );
    }

@@ -7,7 +7,7 @@ export abstract class BaseKVStore {
  abstract put(
    key: string,
    val: Record<string, any>,
-    collection?: string
+    collection?: string,
  ): Promise<void>;
  abstract get(key: string, collection?: string): Promise<StoredValue>;
  abstract getAll(collection?: string): Promise<Record<string, StoredValue>>;
@@ -6,8 +6,8 @@ import {
  getTopKMMREmbeddings,
 } from "../../Embedding";
 import { BaseNode } from "../../Node";
+import { GenericFileSystem, exists } from "../FileSystem";
 import { DEFAULT_FS, DEFAULT_PERSIST_DIR } from "../constants";
-import { exists, GenericFileSystem } from "../FileSystem";
 import {
  VectorStore,
  VectorStoreQuery,
@@ -1,18 +1,18 @@
-import { VectorStoreIndex } from "../indices/vectorStore/VectorStoreIndex";
 import { OpenAIEmbedding } from "../Embedding";
-import { OpenAI } from "../llm/LLM";
 import { Document } from "../Node";
+import {
+  ResponseSynthesizer,
+  SimpleResponseBuilder,
+} from "../ResponseSynthesizer";
 import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext";
 import {
  CallbackManager,
  RetrievalCallbackResponse,
  StreamCallbackResponse,
 } from "../callbacks/CallbackManager";
-import { ListIndex, ListRetrieverMode } from "../indices/list";
-import {
-  ResponseSynthesizer,
-  SimpleResponseBuilder,
-} from "../ResponseSynthesizer";
+import { SummaryIndex } from "../indices/summary";
+import { VectorStoreIndex } from "../indices/vectorStore/VectorStoreIndex";
+import { OpenAI } from "../llm/LLM";
 import { mockEmbeddingModel, mockLlmGeneration } from "./utility/mockOpenAI";

 // Mock the OpenAI getOpenAISession function during testing
@@ -65,10 +65,9 @@ describe("CallbackManager: onLLMStream and onRetrieve", () => {
  });

  test("For VectorStoreIndex w/ a SimpleResponseBuilder", async () => {
-    const vectorStoreIndex = await VectorStoreIndex.fromDocuments(
-      [document],
-      { serviceContext }
-    );
+    const vectorStoreIndex = await VectorStoreIndex.fromDocuments([document], {
+      serviceContext,
+    });
    const queryEngine = vectorStoreIndex.asQueryEngine();
    const query = "What is the author's name?";
    const response = await queryEngine.query(query);
@@ -132,21 +131,20 @@ describe("CallbackManager: onLLMStream and onRetrieve", () => {
    // both retrieval and streaming should have
    // the same parent event
    expect(streamCallbackData[0].event.parentId).toBe(
-      retrieveCallbackData[0].event.parentId
+      retrieveCallbackData[0].event.parentId,
    );
  });

-  test("For ListIndex w/ a ListIndexRetriever", async () => {
-    const listIndex = await ListIndex.fromDocuments(
-      [document],
-      { serviceContext },
-    );
+  test("For SummaryIndex w/ a SummaryIndexRetriever", async () => {
+    const summaryIndex = await SummaryIndex.fromDocuments([document], {
+      serviceContext,
+    });
    const responseBuilder = new SimpleResponseBuilder(serviceContext);
    const responseSynthesizer = new ResponseSynthesizer({
      serviceContext: serviceContext,
      responseBuilder,
    });
-    const queryEngine = listIndex.asQueryEngine({
+    const queryEngine = summaryIndex.asQueryEngine({
      responseSynthesizer,
    });
    const query = "What is the author's name?";
@@ -211,7 +209,7 @@ describe("CallbackManager: onLLMStream and onRetrieve", () => {
    // both retrieval and streaming should have
    // the same parent event
    expect(streamCallbackData[0].event.parentId).toBe(
-      retrieveCallbackData[0].event.parentId
+      retrieveCallbackData[0].event.parentId,
    );
  });
 });
@@ -11,7 +11,7 @@ describe("similarity", () => {
    const embedding1 = [1, 2, 3];
    const embedding2 = [4, 5, 6];
    expect(() =>
-      similarity(embedding1, embedding2, "unknown" as SimilarityType)
+      similarity(embedding1, embedding2, "unknown" as SimilarityType),
    ).toThrow();
  });

@@ -19,7 +19,7 @@ describe("similarity", () => {
    const embedding1 = [1, 2, 3];
    const embedding2 = [4, 5, 6];
    expect(
-      similarity(embedding1, embedding2, SimilarityType.DOT_PRODUCT)
+      similarity(embedding1, embedding2, SimilarityType.DOT_PRODUCT),
    ).toEqual(32);
  });

@@ -27,7 +27,7 @@ describe("similarity", () => {
    const embedding1 = [1, 0];
    const embedding2 = [0, 1];
    expect(similarity(embedding1, embedding2, SimilarityType.DEFAULT)).toEqual(
-      0.0
+      0.0,
    );
  });

@@ -36,9 +36,9 @@ describe("similarity", () => {
    const docEmbedding1 = [0, 1]; // farther from query, distance 1.414
    const docEmbedding2 = [1, 1]; // closer to query distance 1
    expect(
-      similarity(queryEmbedding, docEmbedding1, SimilarityType.EUCLIDEAN)
+      similarity(queryEmbedding, docEmbedding1, SimilarityType.EUCLIDEAN),
    ).toBeLessThan(
-      similarity(queryEmbedding, docEmbedding2, SimilarityType.EUCLIDEAN)
+      similarity(queryEmbedding, docEmbedding2, SimilarityType.EUCLIDEAN),
    );
  });
 });
@@ -1,12 +1,12 @@
-import {
-  GenericFileSystem,
-  getNodeFS,
-  InMemoryFileSystem,
-  exists,
-  walk,
-} from "../storage/FileSystem";
 import os from "os";
 import path from "path";
+import {
+  GenericFileSystem,
+  InMemoryFileSystem,
+  exists,
+  getNodeFS,
+  walk,
+} from "../storage/FileSystem";

 type FileSystemUnderTest = {
  name: string;
@@ -61,7 +61,7 @@ describe.each<FileSystemUnderTest>([
    it("writes file to memory", async () => {
      await testFS.writeFile(`${tempDir}/test.txt`, "Hello, world!");
      expect(await testFS.readFile(`${tempDir}/test.txt`, "utf-8")).toBe(
-        "Hello, world!"
+        "Hello, world!",
      );
    });

@@ -69,7 +69,7 @@ describe.each<FileSystemUnderTest>([
      await testFS.writeFile(`${tempDir}/test.txt`, "Hello, world!");
      await testFS.writeFile(`${tempDir}/test.txt`, "Hello, again!");
      expect(await testFS.readFile(`${tempDir}/test.txt`, "utf-8")).toBe(
-        "Hello, again!"
+        "Hello, again!",
      );
    });
  });
@@ -77,7 +77,7 @@ describe.each<FileSystemUnderTest>([
  describe("readFile", () => {
    it("throws error for non-existing file", async () => {
      await expect(
-        testFS.readFile(`${tempDir}/not_exist.txt`, "utf-8")
+        testFS.readFile(`${tempDir}/not_exist.txt`, "utf-8"),
      ).rejects.toThrow();
    });
  });
@@ -1,8 +1,5 @@
 import { existsSync, rmSync } from "fs";
-import {
-  storageContextFromDefaults,
-  StorageContext,
-} from "../storage/StorageContext";
+import { storageContextFromDefaults } from "../storage/StorageContext";

 jest.spyOn(console, "error");

@@ -1,7 +1,7 @@
 import { OpenAIEmbedding } from "../../Embedding";
 import { globalsHelper } from "../../GlobalsHelper";
-import { ChatMessage, OpenAI } from "../../llm/LLM";
 import { CallbackManager, Event } from "../../callbacks/CallbackManager";
+import { ChatMessage, OpenAI } from "../../llm/LLM";

 export function mockLlmGeneration({
  languageModel,
@@ -57,7 +57,7 @@ export function mockLlmGeneration({
            },
          });
        });
-      }
+      },
    );
 }

@@ -2,11 +2,10 @@
 # For details on our config file, check out our docs at https://docs.sweep.dev

 # If you use this be sure to frequently sync your default branch(main, master) to dev.
-branch: 'main'
+branch: "main"
 # If you want to enable GitHub Actions for Sweep, set this to true.
 gha_enabled: False
 # This is the description of your project. It will be used by sweep when creating PRs. You can tell Sweep what's unique about your project, what frameworks you use, or anything else you want.
 # Here's an example: sweepai/sweep is a python project. The main api endpoints are in sweepai/api.py. Write code that adheres to PEP8.
-description: ''
-
+description: ""
 # Default Values: https://github.com/sweepai/sweep/blob/main/sweep.yaml
Author	SHA1	Message	Date
Yi Ding	40a8f0775e	updated exports for storage/index.ts	2023-08-30 12:06:58 -07:00
yisding	65aaebe2b5	Merge pull request #93 from andfk/patch-1 Update README.md for apps example	2023-08-30 06:20:40 -07:00
Andrés F	769559279f	Update README.md The commands should be inverted	2023-08-30 18:13:00 +08:00
Yi Ding	bb7fd38c46	removing the newline to space embedding conversion https://github.com/openai/openai-python/issues/418	2023-08-29 22:35:41 -07:00
Yi Ding	a734927a42	upgrade prettier and prettiered	2023-08-29 20:30:02 -07:00
Yi Ding	e21eca2a16	updated packages and README	2023-08-29 20:24:29 -07:00
Yi Ding	33c8c2fe47	documentation update for SummaryIndex	2023-08-29 14:06:12 -07:00
Yi Ding	c3048858e9	0.0.24	2023-08-29 12:34:21 -07:00
Yi Ding	259fe63ceb	strong types for prompts	2023-08-29 12:33:46 -07:00
Yi Ding	d1aa3b7982	more changes for the summary index	2023-08-29 09:41:23 -07:00
Yi Ding	e4af7b3a53	renamed listindex to summaryindex Better aligns with what the index is used for	2023-08-29 09:32:04 -07:00
Yi Ding	51bd392fed	0.0.23	2023-08-25 12:28:06 -07:00