changeset

Add create-llama CLI tool

.changeset/eight-dolphins-pump.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add multimodal support (thanks Marcus)

.changeset/nasty-seas-care.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Add notion loader (thank you @TomPenguin!)

.changeset/sixty-ladybugs-change.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Hello Create Llama (thanks Marcus!)

.gitignore

@@ -3,6 +3,7 @@
 # dependencies
 node_modules
 .pnp
+.pnpm-store
 .pnp.js
 
 # testing
@@ -36,3 +37,6 @@ yarn-error.log*
 .vercel
 
 dist/
+
+# vs code
+.vscode/launch.json

README.md

@@ -84,6 +84,26 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
 
 - [SimplePrompt](/packages/core/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
 
+## Note: NextJS:
+
+If you're using NextJS App Router, you'll need to use the NodeJS runtime (default) and add the follow config to your next.config.js to have it use imports/exports in the same way Node does.
+
+```js
+export const runtime = "nodejs" // default
+```
+
+```js
+// next.config.js
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  experimental: {
+    serverComponentsExternalPackages: ["pdf-parse"], // Puts pdf-parse in actual NodeJS mode with NextJS App Router
+  },
+};
+
+module.exports = nextConfig;
+```
+
 ## Supported LLMs:
 
 - OpenAI GPT-3.5-turbo and GPT-4

apps/docs/docs/end_to_end.md

@@ -14,7 +14,7 @@ Read a file and chat about it with the LLM.
 
 Create a vector index and query it. The vector index will use embeddings to fetch the top k most relevant nodes. By default, the top k is 2.
 
-## [Summary Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/summarIndex.ts)
+## [Summary Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/summaryIndex.ts)
 
 Create a list index and query it. This example also use the `LLMRetriever`, which will use the LLM to select the best nodes to use when generating answer.
 

apps/docs/docs/environments.md

@@ -0,0 +1,29 @@
+---
+sidebar_position: 5
+---
+
+# Environments
+
+LlamaIndex currently officially supports NodeJS 18 and NodeJS 20.
+
+## NextJS App Router
+
+If you're using NextJS App Router route handlers/serverless functions, you'll need to use the NodeJS mode:
+
+```js
+export const runtime = "nodejs" // default
+```
+
+and you'll need to add an exception for pdf-parse in your next.config.js
+
+```js
+// next.config.js
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  experimental: {
+    serverComponentsExternalPackages: ["pdf-parse"], // Puts pdf-parse in actual NodeJS mode with NextJS App Router
+  },
+};
+
+module.exports = nextConfig;
+```

apps/docs/docs/introduction.md

@@ -19,7 +19,7 @@ That's where **LlamaIndex.TS** comes in.
 
 LlamaIndex.TS provides the following tools:
 
-- **Data loading** ingest your existing `txt` and `pdf` data directly
+- **Data loading** ingest your existing `.txt`, `.pdf`, `.csv`, `.md` and `.docx` data directly
 - **Data indexes** structure your data in intermediate representations that are easy and performant for LLMs to consume.
 - **Engines** provide natural language access to your data. For example:
   - Query engines are powerful retrieval interfaces for knowledge-augmented output.

apps/docs/docs/modules/high_level/data_loader.md

@@ -4,7 +4,7 @@ sidebar_position: 1
 
 # Reader / Loader
 
-LlamaIndex.TS supports easy loading of files from folders using the `SimpleDirectoryReader` class. Currently, `.txt` and `.pdf` files are supported, with more planned in the future!
+LlamaIndex.TS supports easy loading of files from folders using the `SimpleDirectoryReader` class. Currently, `.txt`, `.pdf`, `.csv`, `.md` and `.docx` files are supported, with more planned in the future!
 
 ```typescript
 import { SimpleDirectoryReader } from "llamaindex";

apps/docs/package.json

@@ -15,24 +15,24 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "2.4.1",
-    "@docusaurus/preset-classic": "2.4.1",
-    "@docusaurus/remark-plugin-npm2yarn": "^2.4.1",
+    "@docusaurus/core": "2.4.3",
+    "@docusaurus/preset-classic": "2.4.3",
+    "@docusaurus/remark-plugin-npm2yarn": "^2.4.3",
     "@mdx-js/react": "^1.6.22",
     "clsx": "^1.2.1",
-    "postcss": "^8.4.28",
+    "postcss": "^8.4.31",
     "prism-react-renderer": "^1.3.5",
     "raw-loader": "^4.0.2",
     "react": "^17.0.2",
     "react-dom": "^17.0.2"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "2.4.1",
-    "@docusaurus/types": "^2.4.1",
-    "@tsconfig/docusaurus": "^1.0.7",
+    "@docusaurus/module-type-aliases": "2.4.3",
+    "@docusaurus/types": "^2.4.3",
+    "@tsconfig/docusaurus": "^2.0.1",
     "docusaurus-plugin-typedoc": "^0.19.2",
     "typedoc": "^0.24.8",
-    "typedoc-plugin-markdown": "^3.15.4",
+    "typedoc-plugin-markdown": "^3.16.0",
     "typescript": "^4.9.5"
   },
   "browserslist": {

apps/simple/CHANGELOG.md

@@ -1,5 +1,81 @@
 # simple
 
+## 0.0.32
+
+### Patch Changes
+
+- Updated dependencies [2a27e21]
+  - llamaindex@0.0.34
+
+## 0.0.31
+
+### Patch Changes
+
+- Updated dependencies [5e2e92c]
+  - llamaindex@0.0.33
+
+## 0.0.30
+
+### Patch Changes
+
+- Updated dependencies [90c0b83]
+- Updated dependencies [dfd22aa]
+  - llamaindex@0.0.32
+
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [6c55b2d]
+- Updated dependencies [8aa8c65]
+- Updated dependencies [6c55b2d]
+  - llamaindex@0.0.31
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [139abad]
+- Updated dependencies [139abad]
+- Updated dependencies [eb0e994]
+- Updated dependencies [eb0e994]
+- Updated dependencies [139abad]
+  - llamaindex@0.0.30
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [a52143b]
+- Updated dependencies [1b7fd95]
+- Updated dependencies [0db3f41]
+  - llamaindex@0.0.29
+
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [96bb657]
+- Updated dependencies [96bb657]
+- Updated dependencies [837854d]
+  - llamaindex@0.0.28
+
+## 0.0.25
+
+### Patch Changes
+
+- Updated dependencies [4a5591b]
+- Updated dependencies [4a5591b]
+- Updated dependencies [4a5591b]
+  - llamaindex@0.0.27
+
+## 0.0.24
+
+### Patch Changes
+
+- Updated dependencies [5bb55bc]
+  - llamaindex@0.0.26
+
 ## 0.0.23
 
 ### Patch Changes

apps/simple/directory.ts

@@ -0,0 +1,24 @@
+import { SimpleDirectoryReader } from "llamaindex";
+
+function callback(
+  category: string,
+  name: string,
+  status: any,
+  message?: string,
+): boolean {
+  console.log(category, name, status, message);
+  if (name.endsWith(".pdf")) {
+    console.log("I DON'T WANT PDF FILES!");
+    return false;
+  }
+  return true;
+}
+
+async function main() {
+  // Load page
+  const reader = new SimpleDirectoryReader(callback);
+  const params = { directoryPath: "./data" };
+  await reader.loadData(params);
+}
+
+main().catch(console.error);

apps/simple/gptllama.ts

@@ -6,7 +6,7 @@ import readline from "node:readline/promises";
 import { ChatMessage, LlamaDeuce, OpenAI } from "llamaindex";
 
 (async () => {
-  const gpt4 = new OpenAI({ model: "gpt-4", temperature: 0.9 });
+  const gpt4 = new OpenAI({ model: "gpt-4-vision-preview", temperature: 0.9 });
   const l2 = new LlamaDeuce({
     model: "Llama-2-70b-chat-4bit",
     temperature: 0.9,

apps/simple/html.ts

@@ -0,0 +1,21 @@
+import { HTMLReader, VectorStoreIndex } from "llamaindex";
+
+async function main() {
+  // Load page
+  const reader = new HTMLReader();
+  const documents = await reader.loadData("data/18-1_Changelog.html");
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  const index = await VectorStoreIndex.fromDocuments(documents);
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query(
+    "What were the notable changes in 18.1?",
+  );
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);

apps/simple/keywordIndex.ts

@@ -0,0 +1,32 @@
+import {
+  Document,
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "llamaindex";
+import essay from "./essay";
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+  const index = await KeywordTableIndex.fromDocuments([document]);
+
+  const allModes: KeywordTableRetrieverMode[] = [
+    KeywordTableRetrieverMode.DEFAULT,
+    KeywordTableRetrieverMode.SIMPLE,
+    KeywordTableRetrieverMode.RAKE,
+  ];
+  allModes.forEach(async (mode) => {
+    const queryEngine = index.asQueryEngine({
+      retriever: index.asRetriever({
+        mode,
+      }),
+    });
+    const response = await queryEngine.query(
+      "What did the author do growing up?",
+    );
+    console.log(response.toString());
+  });
+}
+
+main().catch((e: Error) => {
+  console.error(e, e.stack);
+});

apps/simple/llmStream.ts

@@ -0,0 +1,47 @@
+import { ChatMessage, OpenAI, SimpleChatEngine } from "llamaindex";
+import {Anthropic} from "../../packages/core/src/llm/LLM";
+import { stdin as input, stdout as output } from "node:process";
+import readline from "node:readline/promises";
+
+async function main() {
+  const query: string = `
+Where is Istanbul?
+  `;
+
+  // const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 });
+  const llm = new Anthropic();
+  const message: ChatMessage = { content: query, role: "user" };
+
+  //TODO: Add callbacks later
+
+  //Stream Complete
+  //Note: Setting streaming flag to true or false will auto-set your return type to
+  //either an AsyncGenerator or a Response.
+  // Omitting the streaming flag automatically sets streaming to false
+
+  const chatEngine: SimpleChatEngine = new SimpleChatEngine({
+    chatHistory: undefined,
+    llm: llm,
+  });
+
+  const rl = readline.createInterface({ input, output });
+  while (true) {
+    const query = await rl.question("Query: ");
+
+    if (!query) {
+      break;
+    }
+
+    //Case 1: .chat(query, undefined, true) => Stream
+    //Case 2: .chat(query, undefined, false) => Response object
+    //Case 3: .chat(query, undefined) => Response object
+    const chatStream = await chatEngine.chat(query, undefined, true);
+    var accumulated_result = "";
+    for await (const part of chatStream) {
+      accumulated_result += part;
+      process.stdout.write(part);
+    }
+  }
+}
+
+main();

apps/simple/mongo.ts

@@ -0,0 +1,68 @@
+import { MongoClient } from "mongodb";
+import { VectorStoreIndex } from "../../packages/core/src/indices";
+import { Document } from "../../packages/core/src/Node";
+import { SimpleMongoReader } from "../../packages/core/src/readers/SimpleMongoReader";
+
+import { stdin as input, stdout as output } from "node:process";
+import readline from "node:readline/promises";
+
+async function main() {
+  //Dummy test code
+  const query: object = { _id: "waldo" };
+  const options: object = {};
+  const projections: object = { embedding: 0 };
+  const limit: number = Infinity;
+  const uri: string = process.env.MONGODB_URI ?? "fake_uri";
+  const client: MongoClient = new MongoClient(uri);
+
+  //Where the real code starts
+  const MR = new SimpleMongoReader(client);
+  const documents: Document[] = await MR.loadData(
+    "data",
+    "posts",
+    1,
+    {},
+    options,
+    projections,
+  );
+
+  //
+  //If you need to look at low-level details of
+  // a queryEngine (for example, needing to check each individual node)
+  //
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  // var storageContext = await storageContextFromDefaults({});
+  // var serviceContext = serviceContextFromDefaults({});
+  // const docStore = storageContext.docStore;
+
+  // for (const doc of documents) {
+  //   docStore.setDocumentHash(doc.id_, doc.hash);
+  // }
+  // const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
+  // console.log(nodes);
+
+  //
+  //Making Vector Store from documents
+  //
+
+  const index = await VectorStoreIndex.fromDocuments(documents);
+  // Create query engine
+  const queryEngine = index.asQueryEngine();
+
+  const rl = readline.createInterface({ input, output });
+  while (true) {
+    const query = await rl.question("Query: ");
+
+    if (!query) {
+      break;
+    }
+
+    const response = await queryEngine.query(query);
+
+    // Output response
+    console.log(response.toString());
+  }
+}
+
+main();

apps/simple/openai.ts

@@ -1,7 +1,7 @@
 import { OpenAI } from "llamaindex";
 
 (async () => {
-  const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 });
+  const llm = new OpenAI({ model: "gpt-4-1106-preview", temperature: 0.1 });
 
   // complete api
   const response1 = await llm.complete("How are you?");
@@ -9,7 +9,7 @@ import { OpenAI } from "llamaindex";
 
   // chat api
   const response2 = await llm.chat([
-    { content: "Tell me a joke!", role: "user" },
+    { content: "Tell me a joke.", role: "user" },
   ]);
   console.log(response2.message.content);
 })();

apps/simple/package.json

@@ -1,14 +1,16 @@
 {
-  "version": "0.0.23",
+  "version": "0.0.32",
   "private": true,
   "name": "simple",
   "dependencies": {
-    "@notionhq/client": "^2.2.12",
-    "commander": "^11.0.0",
+    "@notionhq/client": "^2.2.13",
+    "@pinecone-database/pinecone": "^1.1.2",
+    "commander": "^11.1.0",
     "llamaindex": "workspace:*"
   },
   "devDependencies": {
-    "@types/node": "^18.17.12"
+    "@types/node": "^18.18.6",
+    "ts-node": "^10.9.1"
   },
   "scripts": {
     "lint": "eslint ."

apps/simple/portkey.ts

@@ -0,0 +1,23 @@
+import { Portkey } from "llamaindex";
+
+(async () => {
+  const llms = [{
+
+  }]
+  const portkey = new Portkey({
+    mode: "single",
+    llms: [{
+      provider:"anyscale",
+      virtual_key:"anyscale-3b3c04",
+      model: "meta-llama/Llama-2-13b-chat-hf",
+      max_tokens: 2000
+    }]
+  });
+  const result = portkey.stream_chat([
+    { role: "system", content: "You are a helpful assistant." },
+    { role: "user", content: "Tell me a joke." }
+  ]);
+  for await (const res of result) {
+    process.stdout.write(res)
+  }
+})();

apps/simple/vectorIndexCustomize.ts

@@ -3,6 +3,7 @@ import {
   OpenAI,
   RetrieverQueryEngine,
   serviceContextFromDefaults,
+  SimilarityPostprocessor,
   VectorStoreIndex,
 } from "llamaindex";
 import essay from "./essay";
@@ -21,8 +22,16 @@ async function main() {
 
   const retriever = index.asRetriever();
   retriever.similarityTopK = 5;
+  const nodePostprocessor = new SimilarityPostprocessor({
+    similarityCutoff: 0.7,
+  });
   // TODO: cannot pass responseSynthesizer into retriever query engine
-  const queryEngine = new RetrieverQueryEngine(retriever);
+  const queryEngine = new RetrieverQueryEngine(
+    retriever,
+    undefined,
+    undefined,
+    [nodePostprocessor],
+  );
 
   const response = await queryEngine.query(
     "What did the author do growing up?",

apps/simple/vectorIndexFromVectorStore.ts

@@ -0,0 +1,197 @@
+import {
+  OpenAI,
+  ResponseSynthesizer,
+  RetrieverQueryEngine,
+  serviceContextFromDefaults,
+  TextNode,
+  TreeSummarize,
+  VectorIndexRetriever,
+  VectorStore,
+  VectorStoreIndex,
+  VectorStoreQuery,
+  VectorStoreQueryResult,
+} from "llamaindex";
+
+import { Index, Pinecone, RecordMetadata } from "@pinecone-database/pinecone";
+
+/**
+ * Please do not use this class in production; it's only for demonstration purposes.
+ */
+class PineconeVectorStore<T extends RecordMetadata = RecordMetadata>
+  implements VectorStore
+{
+  storesText = true;
+  isEmbeddingQuery = false;
+
+  indexName!: string;
+  pineconeClient!: Pinecone;
+  index!: Index<T>;
+
+  constructor({ indexName, client }: { indexName: string; client: Pinecone }) {
+    this.indexName = indexName;
+    this.pineconeClient = client;
+    this.index = client.index<T>(indexName);
+  }
+
+  client() {
+    return this.pineconeClient;
+  }
+
+  async query(
+    query: VectorStoreQuery,
+    kwargs?: any,
+  ): Promise<VectorStoreQueryResult> {
+    let queryEmbedding: number[] = [];
+    if (query.queryEmbedding) {
+      if (typeof query.alpha === "number") {
+        const alpha = query.alpha;
+        queryEmbedding = query.queryEmbedding.map((v) => v * alpha);
+      } else {
+        queryEmbedding = query.queryEmbedding;
+      }
+    }
+
+    // Current LlamaIndexTS implementation only support exact match filter, so we use kwargs instead.
+    const filter = kwargs?.filter || {};
+
+    const response = await this.index.query({
+      filter,
+      vector: queryEmbedding,
+      topK: query.similarityTopK,
+      includeValues: true,
+      includeMetadata: true,
+    });
+
+    console.log(
+      `Numbers of vectors returned by Pinecone after preFilters are applied: ${
+        response?.matches?.length || 0
+      }.`,
+    );
+
+    const topKIds: string[] = [];
+    const topKNodes: TextNode[] = [];
+    const topKScores: number[] = [];
+
+    const metadataToNode = (metadata?: T): Partial<TextNode> => {
+      if (!metadata) {
+        throw new Error("metadata is undefined.");
+      }
+
+      const nodeContent = metadata["_node_content"];
+      if (!nodeContent) {
+        throw new Error("nodeContent is undefined.");
+      }
+
+      if (typeof nodeContent !== "string") {
+        throw new Error("nodeContent is not a string.");
+      }
+
+      return JSON.parse(nodeContent);
+    };
+
+    if (response.matches) {
+      for (const match of response.matches) {
+        const node = new TextNode({
+          ...metadataToNode(match.metadata),
+          embedding: match.values,
+        });
+
+        topKIds.push(match.id);
+        topKNodes.push(node);
+        topKScores.push(match.score ?? 0);
+      }
+    }
+
+    const result = {
+      ids: topKIds,
+      nodes: topKNodes,
+      similarities: topKScores,
+    };
+
+    return result;
+  }
+
+  add(): Promise<string[]> {
+    return Promise.resolve([]);
+  }
+
+  delete(): Promise<void> {
+    throw new Error("Method `delete` not implemented.");
+  }
+
+  persist(): Promise<void> {
+    throw new Error("Method `persist` not implemented.");
+  }
+}
+
+/**
+ * The goal of this example is to show how to use Pinecone as a vector store
+ * for LlamaIndexTS with(out) preFilters.
+ *
+ * It should not be used in production like that,
+ * as you might want to find a proper PineconeVectorStore implementation.
+ */
+async function main() {
+  process.env.PINECONE_API_KEY = "Your Pinecone API Key.";
+  process.env.PINECONE_ENVIRONMENT = "Your Pinecone Environment.";
+  process.env.PINECONE_PROJECT_ID = "Your Pinecone Project ID.";
+  process.env.PINECONE_INDEX_NAME = "Your Pinecone Index Name.";
+  process.env.OPENAI_API_KEY = "Your OpenAI API Key.";
+  process.env.OPENAI_API_ORGANIZATION = "Your OpenAI API Organization.";
+
+  const getPineconeVectorStore = async () => {
+    return new PineconeVectorStore({
+      indexName: process.env.PINECONE_INDEX_NAME || "index-name",
+      client: new Pinecone(),
+    });
+  };
+
+  const getServiceContext = () => {
+    const openAI = new OpenAI({
+      model: "gpt-4",
+      apiKey: process.env.OPENAI_API_KEY,
+    });
+
+    return serviceContextFromDefaults({
+      llm: openAI,
+    });
+  };
+
+  const getQueryEngine = async (filter: unknown) => {
+    const vectorStore = await getPineconeVectorStore();
+    const serviceContext = getServiceContext();
+
+    const vectorStoreIndex = await VectorStoreIndex.fromVectorStore(
+      vectorStore,
+      serviceContext,
+    );
+
+    const retriever = new VectorIndexRetriever({
+      index: vectorStoreIndex,
+      similarityTopK: 500,
+    });
+
+    const responseSynthesizer = new ResponseSynthesizer({
+      serviceContext,
+      responseBuilder: new TreeSummarize(serviceContext),
+    });
+
+    return new RetrieverQueryEngine(retriever, responseSynthesizer, {
+      filter,
+    });
+  };
+
+  // whatever is a key from your metadata
+  const queryEngine = await getQueryEngine({
+    whatever: {
+      $gte: 1,
+      $lte: 100,
+    },
+  });
+
+  const response = await queryEngine.query("How many results do you have?");
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);

apps/simple/vision.ts

@@ -0,0 +1,15 @@
+import { OpenAI } from "llamaindex";
+
+(async () => {
+  const llm = new OpenAI({ model: "gpt-4-vision-preview", temperature: 0.1 });
+
+  // complete api
+  const response1 = await llm.complete("How are you?");
+  console.log(response1.message.content);
+
+  // chat api
+  const response2 = await llm.chat([
+    { content: "Tell me a joke!", role: "user" },
+  ]);
+  console.log(response2.message.content);
+})();

examples/keywordIndex.ts

@@ -0,0 +1,32 @@
+import {
+  Document,
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "llamaindex";
+import essay from "./essay";
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+  const index = await KeywordTableIndex.fromDocuments([document]);
+
+  const allModes: KeywordTableRetrieverMode[] = [
+    KeywordTableRetrieverMode.DEFAULT,
+    KeywordTableRetrieverMode.SIMPLE,
+    KeywordTableRetrieverMode.RAKE,
+  ];
+  allModes.forEach(async (mode) => {
+    const queryEngine = index.asQueryEngine({
+      retriever: index.asRetriever({
+        mode,
+      }),
+    });
+    const response = await queryEngine.query(
+      "What did the author do growing up?",
+    );
+    console.log(response.toString());
+  });
+}
+
+main().catch((e: Error) => {
+  console.error(e, e.stack);
+});

examples/notion.ts

@@ -0,0 +1,89 @@
+import { Client } from "@notionhq/client";
+import { program } from "commander";
+import { NotionReader, VectorStoreIndex } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+
+program
+  .argument("[page]", "Notion page id (must be provided)")
+  .action(async (page, _options, command) => {
+    // Initializing a client
+
+    if (!process.env.NOTION_TOKEN) {
+      console.log(
+        "No NOTION_TOKEN found in environment variables. You will need to register an integration https://www.notion.com/my-integrations and put it in your NOTION_TOKEN environment variable.",
+      );
+      return;
+    }
+
+    const notion = new Client({
+      auth: process.env.NOTION_TOKEN,
+    });
+
+    if (!page) {
+      const response = await notion.search({
+        filter: {
+          value: "page",
+          property: "object",
+        },
+        sort: {
+          direction: "descending",
+          timestamp: "last_edited_time",
+        },
+      });
+
+      const { results } = response;
+
+      if (results.length === 0) {
+        console.log(
+          "No pages found. You will need to share it with your integration. (tap the three dots on the top right, find Add connections, and add your integration)",
+        );
+        return;
+      } else {
+        const pages = results
+          .map((result) => {
+            if (!("url" in result)) {
+              return null;
+            }
+
+            return {
+              id: result.id,
+              url: result.url,
+            };
+          })
+          .filter((page) => page !== null);
+        console.log("Found pages:");
+        console.table(pages);
+        console.log(`To run, run ts-node ${command.name()} [page id]`);
+        return;
+      }
+    }
+
+    const reader = new NotionReader({ client: notion });
+    const documents = await reader.loadData(page);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Query: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      // Output response
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

examples/openai.ts

@@ -1,7 +1,7 @@
 import { OpenAI } from "llamaindex";
 
 (async () => {
-  const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 });
+  const llm = new OpenAI({ model: "gpt-4-1106-preview", temperature: 0.1 });
 
   // complete api
   const response1 = await llm.complete("How are you?");
@@ -9,7 +9,7 @@ import { OpenAI } from "llamaindex";
 
   // chat api
   const response2 = await llm.chat([
-    { content: "Tell me a joke!", role: "user" },
+    { content: "Tell me a joke.", role: "user" },
   ]);
   console.log(response2.message.content);
 })();

examples/portkey.ts

@@ -0,0 +1,23 @@
+import { Portkey } from "llamaindex";
+
+(async () => {
+  const llms = [{
+
+  }]
+  const portkey = new Portkey({
+    mode: "single",
+    llms: [{
+      provider:"anyscale",
+      virtual_key:"anyscale-3b3c04",
+      model: "meta-llama/Llama-2-13b-chat-hf",
+      max_tokens: 2000
+    }]
+  });
+  const result = portkey.stream_chat([
+    { role: "system", content: "You are a helpful assistant." },
+    { role: "user", content: "Tell me a joke." }
+  ]);
+  for await (const res of result) {
+    process.stdout.write(res)
+  }
+})();

examples/readonly.ts

@@ -0,0 +1,37 @@
+import { execSync } from "child_process";
+import {
+  PDFReader,
+  serviceContextFromDefaults,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+
+const STORAGE_DIR = "./cache";
+
+async function main() {
+  // write the index to disk
+  const serviceContext = serviceContextFromDefaults({});
+  const storageContext = await storageContextFromDefaults({
+    persistDir: `${STORAGE_DIR}`,
+  });
+  const reader = new PDFReader();
+  const documents = await reader.loadData("data/brk-2022.pdf");
+  await VectorStoreIndex.fromDocuments(documents, {
+    storageContext,
+    serviceContext,
+  });
+  console.log("wrote index to disk - now trying to read it");
+  // make index dir read only
+  execSync(`chmod -R 555 ${STORAGE_DIR}`);
+  // reopen index
+  const readOnlyStorageContext = await storageContextFromDefaults({
+    persistDir: `${STORAGE_DIR}`,
+  });
+  await VectorStoreIndex.init({
+    storageContext: readOnlyStorageContext,
+    serviceContext,
+  });
+  console.log("read only index successfully opened");
+}
+
+main().catch(console.error);

package.json

@@ -11,24 +11,25 @@
     "publish-snapshot": "turbo run build lint test && changeset version --snapshot && changeset publish"
   },
   "devDependencies": {
-    "@turbo/gen": "^1.10.13",
-    "@types/jest": "^29.5.4",
-    "eslint": "^7.32.0",
+    "@turbo/gen": "^1.10.16",
+    "@types/jest": "^29.5.6",
+    "eslint": "^8.52.0",
     "eslint-config-custom": "workspace:*",
     "husky": "^8.0.3",
-    "jest": "^29.6.4",
+    "jest": "^29.7.0",
     "prettier": "^3.0.3",
     "prettier-plugin-organize-imports": "^3.2.3",
     "ts-jest": "^29.1.1",
-    "turbo": "^1.10.13"
+    "turbo": "^1.10.16"
   },
-  "packageManager": "pnpm@7.15.0",
+  "packageManager": "pnpm@8.10.4+sha256.df3202c6c8fd345be5ba6a4199297582d5bebf8963822aa3344f4cd2b8be8d43",
   "dependencies": {
     "@changesets/cli": "^2.26.2"
   },
   "pnpm": {
     "overrides": {
-      "trim": "1.0.1"
+      "trim": "1.0.1",
+      "@babel/traverse": "7.23.2"
     }
   }
 }

packages/core/CHANGELOG.md

@@ -1,5 +1,72 @@
 # llamaindex
 
+## 0.0.34
+
+### Patch Changes
+
+- 2a27e21: Add support for gpt-3.5-turbo-1106
+
+## 0.0.33
+
+### Patch Changes
+
+- 5e2e92c: gpt-4-1106-preview and gpt-4-vision-preview from OpenAI dev day
+
+## 0.0.32
+
+### Patch Changes
+
+- 90c0b83: Add HTMLReader (thanks @mtutty)
+- dfd22aa: Add observer/filter to the SimpleDirectoryReader (thanks @mtutty)
+
+## 0.0.31
+
+### Patch Changes
+
+- 6c55b2d: Give HistoryChatEngine pluggable options (thanks @marcusschiesser)
+- 8aa8c65: Add SimilarityPostProcessor (thanks @TomPenguin)
+- 6c55b2d: Added LLMMetadata (thanks @marcusschiesser)
+
+## 0.0.30
+
+### Patch Changes
+
+- 139abad: Streaming improvements including Anthropic (thanks @kkang2097)
+- 139abad: Portkey integration (Thank you @noble-varghese)
+- eb0e994: Add export for PromptHelper (thanks @zigamall)
+- eb0e994: Publish ESM module again
+- 139abad: Pinecone demo (thanks @Einsenhorn)
+
+## 0.0.29
+
+### Patch Changes
+
+- a52143b: Added DocxReader for Word documents (thanks @jayantasamaddar)
+- 1b7fd95: Updated OpenAI streaming (thanks @kkang2097)
+- 0db3f41: Migrated to Tiktoken lite, which hopefully fixes the Windows issue
+
+## 0.0.28
+
+### Patch Changes
+
+- 96bb657: Typesafe metadata (thanks @TomPenguin)
+- 96bb657: MongoReader (thanks @kkang2097)
+- 837854d: Make OutputParser less strict and add tests (Thanks @kkang2097)
+
+## 0.0.27
+
+### Patch Changes
+
+- 4a5591b: Chat History summarization (thanks @marcusschiesser)
+- 4a5591b: Notion database support (thanks @TomPenguin)
+- 4a5591b: KeywordIndex (thanks @swk777)
+
+## 0.0.26
+
+### Patch Changes
+
+- 5bb55bc: Add notion loader (thank you @TomPenguin!)
+
 ## 0.0.25
 
 ### Patch Changes

packages/core/package.json

@@ -1,36 +1,47 @@
 {
   "name": "llamaindex",
-  "version": "0.0.25",
+  "version": "0.0.34",
+  "license": "MIT",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.6.2",
-    "@notionhq/client": "^2.2.12",
+    "@anthropic-ai/sdk": "^0.9.0",
+    "@notionhq/client": "^2.2.13",
+    "js-tiktoken": "^1.0.7",
     "lodash": "^4.17.21",
+    "mammoth": "^1.6.0",
     "md-utils-ts": "^2.0.0",
-    "openai": "^4.3.1",
+    "mongodb": "^6.2.0",
+    "notion-md-crawler": "^0.0.2",
+    "openai": "^4.16.1",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
-    "replicate": "^0.16.1",
-    "tiktoken-node": "^0.0.6",
-    "uuid": "^9.0.0",
+    "portkey-ai": "^0.1.16",
+    "rake-modified": "^1.0.8",
+    "replicate": "^0.20.1",
+    "string-strip-html": "^13.4.3",
+    "uuid": "^9.0.1",
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
-    "@types/lodash": "^4.14.197",
-    "@types/node": "^18.17.12",
-    "@types/papaparse": "^5.3.8",
-    "@types/pdf-parse": "^1.1.1",
-    "@types/uuid": "^9.0.3",
+    "@types/lodash": "^4.14.200",
+    "@types/node": "^18.18.8",
+    "@types/papaparse": "^5.3.10",
+    "@types/pdf-parse": "^1.1.3",
+    "@types/uuid": "^9.0.6",
     "node-stdlib-browser": "^1.2.0",
-    "tsup": "^7.2.0"
+    "tsup": "^7.2.0",
+    "typescript": "^5.2.2"
   },
   "engines": {
     "node": ">=18.0.0"
   },
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "repository": "run-llama/LlamaIndexTS",
   "scripts": {
     "lint": "eslint .",
     "test": "jest",
-    "build": "tsup src/index.ts --format esm,cjs --dts"
+    "build": "tsup src/index.ts --format esm,cjs --dts",
+    "dev": "tsup src/index.ts --format esm,cjs --dts --watch"
   }
 }

packages/core/src/ChatEngine.ts

@@ -1,5 +1,6 @@
 import { v4 as uuidv4 } from "uuid";
-import { TextNode } from "./Node";
+import { ChatHistory } from "./ChatHistory";
+import { NodeWithScore, TextNode } from "./Node";
 import {
   CondenseQuestionPrompt,
   ContextSystemPrompt,
@@ -12,6 +13,7 @@ import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { Event } from "./callbacks/CallbackManager";
+import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
 import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 
 /**
@@ -22,8 +24,16 @@ export interface ChatEngine {
    * Send message along with the class's current chat history to the LLM.
    * @param message
    * @param chatHistory optional chat history if you want to customize the chat history
+   * @param streaming optional streaming flag, which auto-sets the return value if True.
    */
-  chat(message: string, chatHistory?: ChatMessage[]): Promise<Response>;
+  chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(
+    message: string,
+    chatHistory?: ChatMessage[],
+    streaming?: T,
+  ): Promise<R>;
 
   /**
    * Resets the chat history so that it's empty.
@@ -43,13 +53,45 @@ export class SimpleChatEngine implements ChatEngine {
     this.llm = init?.llm ?? new OpenAI();
   }
 
-  async chat(message: string, chatHistory?: ChatMessage[]): Promise<Response> {
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(message: string, chatHistory?: ChatMessage[], streaming?: T): Promise<R> {
+    //Streaming option
+    if (streaming) {
+      return this.streamChat(message, chatHistory) as R;
+    }
+
+    //Non-streaming option
     chatHistory = chatHistory ?? this.chatHistory;
     chatHistory.push({ content: message, role: "user" });
-    const response = await this.llm.chat(chatHistory);
+    const response = await this.llm.chat(chatHistory, undefined);
     chatHistory.push(response.message);
     this.chatHistory = chatHistory;
-    return new Response(response.message.content);
+    return new Response(response.message.content) as R;
+  }
+
+  protected async *streamChat(
+    message: string,
+    chatHistory?: ChatMessage[],
+  ): AsyncGenerator<string, void, unknown> {
+    chatHistory = chatHistory ?? this.chatHistory;
+    chatHistory.push({ content: message, role: "user" });
+    const response_generator = await this.llm.chat(
+      chatHistory,
+      undefined,
+      true,
+    );
+
+    var accumulator: string = "";
+    for await (const part of response_generator) {
+      accumulator += part;
+      yield part;
+    }
+
+    chatHistory.push({ content: accumulator, role: "assistant" });
+    this.chatHistory = chatHistory;
+    return;
   }
 
   reset() {
@@ -98,10 +140,14 @@ export class CondenseQuestionChatEngine implements ChatEngine {
     );
   }
 
-  async chat(
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(
     message: string,
     chatHistory?: ChatMessage[] | undefined,
-  ): Promise<Response> {
+    streaming?: T,
+  ): Promise<R> {
     chatHistory = chatHistory ?? this.chatHistory;
 
     const condensedQuestion = (
@@ -113,7 +159,7 @@ export class CondenseQuestionChatEngine implements ChatEngine {
     chatHistory.push({ content: message, role: "user" });
     chatHistory.push({ content: response.response, role: "assistant" });
 
-    return response;
+    return response as R;
   }
 
   reset() {
@@ -121,57 +167,117 @@ export class CondenseQuestionChatEngine implements ChatEngine {
   }
 }
 
+export interface Context {
+  message: ChatMessage;
+  nodes: NodeWithScore[];
+}
+
+export interface ContextGenerator {
+  generate(message: string, parentEvent?: Event): Promise<Context>;
+}
+
+export class DefaultContextGenerator implements ContextGenerator {
+  retriever: BaseRetriever;
+  contextSystemPrompt: ContextSystemPrompt;
+  nodePostprocessors: BaseNodePostprocessor[];
+
+  constructor(init: {
+    retriever: BaseRetriever;
+    contextSystemPrompt?: ContextSystemPrompt;
+    nodePostprocessors?: BaseNodePostprocessor[];
+  }) {
+    this.retriever = init.retriever;
+    this.contextSystemPrompt =
+      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
+    this.nodePostprocessors = init.nodePostprocessors || [];
+  }
+
+  private applyNodePostprocessors(nodes: NodeWithScore[]) {
+    return this.nodePostprocessors.reduce(
+      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
+      nodes,
+    );
+  }
+
+  async generate(message: string, parentEvent?: Event): Promise<Context> {
+    if (!parentEvent) {
+      parentEvent = {
+        id: uuidv4(),
+        type: "wrapper",
+        tags: ["final"],
+      };
+    }
+    const sourceNodesWithScore = await this.retriever.retrieve(
+      message,
+      parentEvent,
+    );
+
+    const nodes = this.applyNodePostprocessors(sourceNodesWithScore);
+
+    return {
+      message: {
+        content: this.contextSystemPrompt({
+          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
+        }),
+        role: "system",
+      },
+      nodes,
+    };
+  }
+}
+
 /**
  * ContextChatEngine uses the Index to get the appropriate context for each query.
  * The context is stored in the system prompt, and the chat history is preserved,
  * ideally allowing the appropriate context to be surfaced for each query.
  */
 export class ContextChatEngine implements ChatEngine {
-  retriever: BaseRetriever;
-  chatModel: OpenAI;
+  chatModel: LLM;
   chatHistory: ChatMessage[];
-  contextSystemPrompt: ContextSystemPrompt;
+  contextGenerator: ContextGenerator;
 
   constructor(init: {
     retriever: BaseRetriever;
-    chatModel?: OpenAI;
+    chatModel?: LLM;
     chatHistory?: ChatMessage[];
     contextSystemPrompt?: ContextSystemPrompt;
+    nodePostprocessors?: BaseNodePostprocessor[];
   }) {
-    this.retriever = init.retriever;
     this.chatModel =
       init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
     this.chatHistory = init?.chatHistory ?? [];
-    this.contextSystemPrompt =
-      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
+    this.contextGenerator = new DefaultContextGenerator({
+      retriever: init.retriever,
+      contextSystemPrompt: init?.contextSystemPrompt,
+    });
   }
 
-  async chat(message: string, chatHistory?: ChatMessage[] | undefined) {
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(
+    message: string,
+    chatHistory?: ChatMessage[] | undefined,
+    streaming?: T,
+  ): Promise<R> {
     chatHistory = chatHistory ?? this.chatHistory;
 
+    //Streaming option
+    if (streaming) {
+      return this.streamChat(message, chatHistory) as R;
+    }
+
     const parentEvent: Event = {
       id: uuidv4(),
       type: "wrapper",
       tags: ["final"],
     };
-    const sourceNodesWithScore = await this.retriever.retrieve(
-      message,
-      parentEvent,
-    );
-
-    const systemMessage: ChatMessage = {
-      content: this.contextSystemPrompt({
-        context: sourceNodesWithScore
-          .map((r) => (r.node as TextNode).text)
-          .join("\n\n"),
-      }),
-      role: "system",
-    };
+    const context = await this.contextGenerator.generate(message, parentEvent);
 
     chatHistory.push({ content: message, role: "user" });
 
     const response = await this.chatModel.chat(
-      [systemMessage, ...chatHistory],
+      [context.message, ...chatHistory],
       parentEvent,
     );
     chatHistory.push(response.message);
@@ -180,11 +286,146 @@ export class ContextChatEngine implements ChatEngine {
 
     return new Response(
       response.message.content,
-      sourceNodesWithScore.map((r) => r.node),
+      context.nodes.map((r) => r.node),
+    ) as R;
+  }
+
+  protected async *streamChat(
+    message: string,
+    chatHistory?: ChatMessage[] | undefined,
+  ): AsyncGenerator<string, void, unknown> {
+    chatHistory = chatHistory ?? this.chatHistory;
+
+    const parentEvent: Event = {
+      id: uuidv4(),
+      type: "wrapper",
+      tags: ["final"],
+    };
+    const context = await this.contextGenerator.generate(message, parentEvent);
+
+    chatHistory.push({ content: message, role: "user" });
+
+    const response_stream = await this.chatModel.chat(
+      [context.message, ...chatHistory],
+      parentEvent,
+      true,
     );
+    var accumulator: string = "";
+    for await (const part of response_stream) {
+      accumulator += part;
+      yield part;
+    }
+
+    chatHistory.push({ content: accumulator, role: "assistant" });
+
+    this.chatHistory = chatHistory;
+
+    return;
   }
 
   reset() {
     this.chatHistory = [];
   }
 }
+
+export interface MessageContentDetail {
+  type: "text" | "image_url";
+  text: string;
+  image_url: { url: string };
+}
+
+/**
+ * Extended type for the content of a message that allows for multi-modal messages.
+ */
+export type MessageContent = string | MessageContentDetail[];
+
+/**
+ * HistoryChatEngine is a ChatEngine that uses a `ChatHistory` object
+ * to keeps track of chat's message history.
+ * A `ChatHistory` object is passed as a parameter for each call to the `chat` method,
+ * so the state of the chat engine is preserved between calls.
+ * Optionally, a `ContextGenerator` can be used to generate an additional context for each call to `chat`.
+ */
+export class HistoryChatEngine {
+  llm: LLM;
+  contextGenerator?: ContextGenerator;
+
+  constructor(init?: Partial<HistoryChatEngine>) {
+    this.llm = init?.llm ?? new OpenAI();
+    this.contextGenerator = init?.contextGenerator;
+  }
+
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(
+    message: MessageContent,
+    chatHistory: ChatHistory,
+    streaming?: T,
+  ): Promise<R> {
+    //Streaming option
+    if (streaming) {
+      return this.streamChat(message, chatHistory) as R;
+    }
+    const requestMessages = await this.prepareRequestMessages(
+      message,
+      chatHistory,
+    );
+    const response = await this.llm.chat(requestMessages);
+    chatHistory.addMessage(response.message);
+    return new Response(response.message.content) as R;
+  }
+
+  protected async *streamChat(
+    message: MessageContent,
+    chatHistory: ChatHistory,
+  ): AsyncGenerator<string, void, unknown> {
+    const requestMessages = await this.prepareRequestMessages(
+      message,
+      chatHistory,
+    );
+    const response_stream = await this.llm.chat(
+      requestMessages,
+      undefined,
+      true,
+    );
+
+    var accumulator = "";
+    for await (const part of response_stream) {
+      accumulator += part;
+      yield part;
+    }
+    chatHistory.addMessage({
+      content: accumulator,
+      role: "assistant",
+    });
+    return;
+  }
+
+  private async prepareRequestMessages(
+    message: MessageContent,
+    chatHistory: ChatHistory,
+  ) {
+    chatHistory.addMessage({
+      content: message,
+      role: "user",
+    });
+    let requestMessages;
+    let context;
+    if (this.contextGenerator) {
+      if (Array.isArray(message)) {
+        // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
+        // so we can pass them to the context generator
+        message = (message as MessageContentDetail[])
+          .filter((c) => c.type === "text")
+          .map((c) => c.text)
+          .join("\n\n");
+      }
+      context = await this.contextGenerator.generate(message);
+    }
+    requestMessages = await chatHistory.requestMessages(
+      context ? [context.message] : undefined,
+    );
+    return requestMessages;
+  }
+}

packages/core/src/ChatHistory.ts

@@ -0,0 +1,200 @@
+import { ChatMessage, LLM, MessageType, OpenAI } from "./llm/LLM";
+import {
+  defaultSummaryPrompt,
+  messagesToHistoryStr,
+  SummaryPrompt,
+} from "./Prompt";
+
+/**
+ * A ChatHistory is used to keep the state of back and forth chat messages
+ */
+export interface ChatHistory {
+  messages: ChatMessage[];
+  /**
+   * Adds a message to the chat history.
+   * @param message
+   */
+  addMessage(message: ChatMessage): void;
+
+  /**
+   * Returns the messages that should be used as input to the LLM.
+   */
+  requestMessages(transientMessages?: ChatMessage[]): Promise<ChatMessage[]>;
+
+  /**
+   * Resets the chat history so that it's empty.
+   */
+  reset(): void;
+
+  /**
+   * Returns the new messages since the last call to this function (or since calling the constructor)
+   */
+  newMessages(): ChatMessage[];
+}
+
+export class SimpleChatHistory implements ChatHistory {
+  messages: ChatMessage[];
+  private messagesBefore: number;
+
+  constructor(init?: Partial<SimpleChatHistory>) {
+    this.messages = init?.messages ?? [];
+    this.messagesBefore = this.messages.length;
+  }
+
+  addMessage(message: ChatMessage) {
+    this.messages.push(message);
+  }
+
+  async requestMessages(transientMessages?: ChatMessage[]) {
+    return [...(transientMessages ?? []), ...this.messages];
+  }
+
+  reset() {
+    this.messages = [];
+  }
+
+  newMessages() {
+    const newMessages = this.messages.slice(this.messagesBefore);
+    this.messagesBefore = this.messages.length;
+    return newMessages;
+  }
+}
+
+export class SummaryChatHistory implements ChatHistory {
+  tokensToSummarize: number;
+  messages: ChatMessage[];
+  summaryPrompt: SummaryPrompt;
+  llm: LLM;
+  private messagesBefore: number;
+
+  constructor(init?: Partial<SummaryChatHistory>) {
+    this.messages = init?.messages ?? [];
+    this.messagesBefore = this.messages.length;
+    this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
+    this.llm = init?.llm ?? new OpenAI();
+    if (!this.llm.metadata.maxTokens) {
+      throw new Error(
+        "LLM maxTokens is not set. Needed so the summarizer ensures the context window size of the LLM.",
+      );
+    }
+    this.tokensToSummarize =
+      this.llm.metadata.contextWindow - this.llm.metadata.maxTokens;
+  }
+
+  private async summarize(): Promise<ChatMessage> {
+    // get the conversation messages to create summary
+    const messagesToSummarize = this.calcConversationMessages();
+
+    let promptMessages;
+    do {
+      promptMessages = [
+        {
+          content: this.summaryPrompt({
+            context: messagesToHistoryStr(messagesToSummarize),
+          }),
+          role: "user" as MessageType,
+        },
+      ];
+      // remove oldest message until the chat history is short enough for the context window
+      messagesToSummarize.shift();
+    } while (this.llm.tokens(promptMessages) > this.tokensToSummarize);
+
+    const response = await this.llm.chat(promptMessages);
+    return { content: response.message.content, role: "memory" };
+  }
+
+  addMessage(message: ChatMessage) {
+    this.messages.push(message);
+  }
+
+  // Find last summary message
+  private getLastSummaryIndex(): number | null {
+    const reversedMessages = this.messages.slice().reverse();
+    const index = reversedMessages.findIndex(
+      (message) => message.role === "memory",
+    );
+    if (index === -1) {
+      return null;
+    }
+    return this.messages.length - 1 - index;
+  }
+
+  private get systemMessages() {
+    // get array of all system messages
+    return this.messages.filter((message) => message.role === "system");
+  }
+
+  private get nonSystemMessages() {
+    // get array of all non-system messages
+    return this.messages.filter((message) => message.role !== "system");
+  }
+
+  /**
+   * Calculates the messages that describe the conversation so far.
+   * If there's no memory, all non-system messages are used.
+   * If there's a memory, uses all messages after the last summary message.
+   */
+  private calcConversationMessages(transformSummary?: boolean): ChatMessage[] {
+    const lastSummaryIndex = this.getLastSummaryIndex();
+    if (!lastSummaryIndex) {
+      // there's no memory, so just use all non-system messages
+      return this.nonSystemMessages;
+    } else {
+      // there's a memory, so use all messages after the last summary message
+      // and convert summary message so it can be send to the LLM
+      const summaryMessage: ChatMessage = transformSummary
+        ? {
+            content: `Summary of the conversation so far: ${this.messages[lastSummaryIndex].content}`,
+            role: "system",
+          }
+        : this.messages[lastSummaryIndex];
+      return [summaryMessage, ...this.messages.slice(lastSummaryIndex + 1)];
+    }
+  }
+
+  private calcCurrentRequestMessages(transientMessages?: ChatMessage[]) {
+    // TODO: check order: currently, we're sending:
+    // system messages first, then transient messages and then the messages that describe the conversation so far
+    return [
+      ...this.systemMessages,
+      ...(transientMessages ? transientMessages : []),
+      ...this.calcConversationMessages(true),
+    ];
+  }
+
+  async requestMessages(transientMessages?: ChatMessage[]) {
+    const requestMessages = this.calcCurrentRequestMessages(transientMessages);
+
+    // get tokens of current request messages and the transient messages
+    const tokens = this.llm.tokens(requestMessages);
+    if (tokens > this.tokensToSummarize) {
+      // if there are too many tokens for the next request, call summarize
+      const memoryMessage = await this.summarize();
+      const lastMessage = this.messages.at(-1);
+      if (lastMessage && lastMessage.role === "user") {
+        // if last message is a user message, ensure that it's sent after the new memory message
+        this.messages.pop();
+        this.messages.push(memoryMessage);
+        this.messages.push(lastMessage);
+      } else {
+        // otherwise just add the memory message
+        this.messages.push(memoryMessage);
+      }
+      // TODO: we still might have too many tokens
+      // e.g. too large system messages or transient messages
+      // how should we deal with that?
+      return this.calcCurrentRequestMessages(transientMessages);
+    }
+    return requestMessages;
+  }
+
+  reset() {
+    this.messages = [];
+  }
+
+  newMessages() {
+    const newMessages = this.messages.slice(this.messagesBefore);
+    this.messagesBefore = this.messages.length;
+    return newMessages;
+  }
+}

packages/core/src/GlobalsHelper.ts

@@ -1,30 +1,56 @@
+import { encodingForModel, TiktokenModel } from "js-tiktoken";
+
 import { v4 as uuidv4 } from "uuid";
 import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
 
+export enum Tokenizers {
+  CL100K_BASE = "cl100k_base",
+}
+
 /**
  * Helper class singleton
  */
 class GlobalsHelper {
   defaultTokenizer: {
-    encode: (text: string) => number[];
-    decode: (tokens: number[]) => string;
+    encode: (text: string) => Uint32Array;
+    decode: (tokens: Uint32Array) => string;
   } | null = null;
 
-  tokenizer() {
-    if (!this.defaultTokenizer) {
-      const tiktoken = require("tiktoken-node");
-      this.defaultTokenizer = tiktoken.getEncoding("gpt2");
-    }
+  private initDefaultTokenizer() {
+    const encoding = encodingForModel("text-embedding-ada-002"); // cl100k_base
 
-    return this.defaultTokenizer!.encode.bind(this.defaultTokenizer);
+    this.defaultTokenizer = {
+      encode: (text: string) => {
+        return new Uint32Array(encoding.encode(text));
+      },
+      decode: (tokens: Uint32Array) => {
+        const numberArray = Array.from(tokens);
+        const text = encoding.decode(numberArray);
+        const uint8Array = new TextEncoder().encode(text);
+        return new TextDecoder().decode(uint8Array);      
+      },
+    };
   }
 
-  tokenizerDecoder() {
-    if (!this.defaultTokenizer) {
-      const tiktoken = require("tiktoken-node");
-      this.defaultTokenizer = tiktoken.getEncoding("gpt2");
+  tokenizer(encoding?: string) {
+    if (encoding && encoding !== Tokenizers.CL100K_BASE) {
+      throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
     }
-
+    if (!this.defaultTokenizer) {
+      this.initDefaultTokenizer();
+    }
+  
+    return this.defaultTokenizer!.encode.bind(this.defaultTokenizer);
+  }
+  
+  tokenizerDecoder(encoding?: string) {
+    if (encoding && encoding !== Tokenizers.CL100K_BASE) {
+      throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
+    }
+    if (!this.defaultTokenizer) {
+      this.initDefaultTokenizer();
+    }
+  
     return this.defaultTokenizer!.decode.bind(this.defaultTokenizer);
   }
 

packages/core/src/Node.ts

@@ -23,19 +23,23 @@ export enum MetadataMode {
   NONE = "NONE",
 }
 
-export interface RelatedNodeInfo {
+export type Metadata = Record<string, any>;
+
+export interface RelatedNodeInfo<T extends Metadata = Metadata> {
   nodeId: string;
   nodeType?: ObjectType;
-  metadata: Record<string, any>;
+  metadata: T;
   hash?: string;
 }
 
-export type RelatedNodeType = RelatedNodeInfo | RelatedNodeInfo[];
+export type RelatedNodeType<T extends Metadata = Metadata> =
+  | RelatedNodeInfo<T>
+  | RelatedNodeInfo<T>[];
 
 /**
  * Generic abstract class for retrievable nodes
  */
-export abstract class BaseNode {
+export abstract class BaseNode<T extends Metadata = Metadata> {
   /**
    * The unique ID of the Node/Document. The trailing underscore is here
    * to avoid collisions with the id keyword in Python.
@@ -46,13 +50,13 @@ export abstract class BaseNode {
   embedding?: number[];
 
   // Metadata fields
-  metadata: Record<string, any> = {};
+  metadata: T = {} as T;
   excludedEmbedMetadataKeys: string[] = [];
   excludedLlmMetadataKeys: string[] = [];
-  relationships: Partial<Record<NodeRelationship, RelatedNodeType>> = {};
+  relationships: Partial<Record<NodeRelationship, RelatedNodeType<T>>> = {};
   hash: string = "";
 
-  constructor(init?: Partial<BaseNode>) {
+  constructor(init?: Partial<BaseNode<T>>) {
     Object.assign(this, init);
   }
 
@@ -62,7 +66,7 @@ export abstract class BaseNode {
   abstract getMetadataStr(metadataMode: MetadataMode): string;
   abstract setContent(value: any): void;
 
-  get sourceNode(): RelatedNodeInfo | undefined {
+  get sourceNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.SOURCE];
 
     if (Array.isArray(relationship)) {
@@ -72,7 +76,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get prevNode(): RelatedNodeInfo | undefined {
+  get prevNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.PREVIOUS];
 
     if (Array.isArray(relationship)) {
@@ -84,7 +88,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get nextNode(): RelatedNodeInfo | undefined {
+  get nextNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.NEXT];
 
     if (Array.isArray(relationship)) {
@@ -94,7 +98,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get parentNode(): RelatedNodeInfo | undefined {
+  get parentNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.PARENT];
 
     if (Array.isArray(relationship)) {
@@ -104,7 +108,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get childNodes(): RelatedNodeInfo[] | undefined {
+  get childNodes(): RelatedNodeInfo<T>[] | undefined {
     const relationship = this.relationships[NodeRelationship.CHILD];
 
     if (!Array.isArray(relationship)) {
@@ -126,7 +130,7 @@ export abstract class BaseNode {
     return this.embedding;
   }
 
-  asRelatedNodeInfo(): RelatedNodeInfo {
+  asRelatedNodeInfo(): RelatedNodeInfo<T> {
     return {
       nodeId: this.id_,
       metadata: this.metadata,
@@ -146,7 +150,7 @@ export abstract class BaseNode {
 /**
  * TextNode is the default node type for text. Most common node type in LlamaIndex.TS
  */
-export class TextNode extends BaseNode {
+export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
   text: string = "";
   startCharIdx?: number;
   endCharIdx?: number;
@@ -154,7 +158,7 @@ export class TextNode extends BaseNode {
   // metadataTemplate: NOTE write your own formatter if needed
   metadataSeparator: string = "\n";
 
-  constructor(init?: Partial<TextNode>) {
+  constructor(init?: Partial<TextNode<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -233,10 +237,10 @@ export class TextNode extends BaseNode {
 //   }
 // }
 
-export class IndexNode extends TextNode {
+export class IndexNode<T extends Metadata = Metadata> extends TextNode<T> {
   indexId: string = "";
 
-  constructor(init?: Partial<IndexNode>) {
+  constructor(init?: Partial<IndexNode<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -253,8 +257,8 @@ export class IndexNode extends TextNode {
 /**
  * A document is just a special text node with a docId.
  */
-export class Document extends TextNode {
-  constructor(init?: Partial<Document>) {
+export class Document<T extends Metadata = Metadata> extends TextNode<T> {
+  constructor(init?: Partial<Document<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -292,7 +296,7 @@ export function jsonToNode(json: any) {
 /**
  * A node with a similarity score
  */
-export interface NodeWithScore {
-  node: BaseNode;
-  score: number;
+export interface NodeWithScore<T extends Metadata = Metadata> {
+  node: BaseNode<T>;
+  score?: number;
 }

packages/core/src/OutputParser.ts

@@ -53,30 +53,31 @@ class OutputParserError extends Error {
  * @param text A markdown block with JSON
  * @returns parsed JSON object
  */
-function parseJsonMarkdown(text: string) {
+export function parseJsonMarkdown(text: string) {
   text = text.trim();
 
-  const beginDelimiter = "```json";
-  const endDelimiter = "```";
+  const left_square = text.indexOf("[");
+  const left_brace = text.indexOf("{");
 
-  const beginIndex = text.indexOf(beginDelimiter);
-  const endIndex = text.indexOf(
-    endDelimiter,
-    beginIndex + beginDelimiter.length,
-  );
-  if (beginIndex === -1 || endIndex === -1) {
-    throw new OutputParserError("Not a json markdown", { output: text });
+  var left: number;
+  var right: number;
+  if (left_square < left_brace && left_square != -1) {
+    left = left_square;
+    right = text.lastIndexOf("]");
+  } else {
+    left = left_brace;
+    right = text.lastIndexOf("}");
   }
-
-  const jsonText = text.substring(beginIndex + beginDelimiter.length, endIndex);
-
+  const jsonText = text.substring(left, right + 1);
   try {
+    //Single JSON object case
+    if (left_square === -1) {
+      return [JSON.parse(jsonText)];
+    }
+    //Multiple JSON object case.
     return JSON.parse(jsonText);
   } catch (e) {
-    throw new OutputParserError("Not a valid json", {
-      cause: e as Error,
-      output: text,
-    });
+    throw new OutputParserError("Not a json markdown", { output: text });
   }
 }
 

packages/core/src/Prompt.ts

@@ -356,3 +356,34 @@ ${context}
 };
 
 export type ContextSystemPrompt = typeof defaultContextSystemPrompt;
+
+export const defaultKeywordExtractPrompt = ({
+  context = "",
+  maxKeywords = 10,
+}) => {
+  return `
+Some text is provided below. Given the text, extract up to ${maxKeywords} keywords from the text. Avoid stopwords.
+---------------------
+${context}
+---------------------
+Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'
+`;
+};
+
+export type KeywordExtractPrompt = typeof defaultKeywordExtractPrompt;
+
+export const defaultQueryKeywordExtractPrompt = ({
+  question = "",
+  maxKeywords = 10,
+}) => {
+  return `(
+  "A question is provided below. Given the question, extract up to ${maxKeywords} "
+  "keywords from the text. Focus on extracting the keywords that we can use "
+  "to best lookup answers to the question. Avoid stopwords."
+  "---------------------"
+  "${question}"
+  "---------------------"
+  "Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'"
+)`;
+};
+export type QueryKeywordExtractPrompt = typeof defaultQueryKeywordExtractPrompt;

packages/core/src/PromptHelper.ts

@@ -34,7 +34,7 @@ export class PromptHelper {
   numOutput = DEFAULT_NUM_OUTPUTS;
   chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO;
   chunkSizeLimit?: number;
-  tokenizer: (text: string) => number[];
+  tokenizer: (text: string) => Uint32Array;
   separator = " ";
 
   constructor(
@@ -42,7 +42,7 @@ export class PromptHelper {
     numOutput = DEFAULT_NUM_OUTPUTS,
     chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO,
     chunkSizeLimit?: number,
-    tokenizer?: (text: string) => number[],
+    tokenizer?: (text: string) => Uint32Array,
     separator = " ",
   ) {
     this.contextWindow = contextWindow;

packages/core/src/QueryEngine.ts

@@ -1,4 +1,6 @@
 import { v4 as uuidv4 } from "uuid";
+import { Event } from "./callbacks/CallbackManager";
+import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   BaseQuestionGenerator,
@@ -10,7 +12,6 @@ import { CompactAndRefine, ResponseSynthesizer } from "./ResponseSynthesizer";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { QueryEngineTool, ToolMetadata } from "./Tool";
-import { Event } from "./callbacks/CallbackManager";
 
 /**
  * A query engine is a question answerer that can use one or more steps.
@@ -30,16 +31,39 @@ export interface BaseQueryEngine {
 export class RetrieverQueryEngine implements BaseQueryEngine {
   retriever: BaseRetriever;
   responseSynthesizer: ResponseSynthesizer;
+  nodePostprocessors: BaseNodePostprocessor[];
+  preFilters?: unknown;
 
   constructor(
     retriever: BaseRetriever,
     responseSynthesizer?: ResponseSynthesizer,
+    preFilters?: unknown,
+    nodePostprocessors?: BaseNodePostprocessor[],
   ) {
     this.retriever = retriever;
     const serviceContext: ServiceContext | undefined =
       this.retriever.getServiceContext();
     this.responseSynthesizer =
       responseSynthesizer || new ResponseSynthesizer({ serviceContext });
+    this.preFilters = preFilters;
+    this.nodePostprocessors = nodePostprocessors || [];
+  }
+
+  private applyNodePostprocessors(nodes: NodeWithScore[]) {
+    return this.nodePostprocessors.reduce(
+      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
+      nodes,
+    );
+  }
+
+  private async retrieve(query: string, parentEvent: Event) {
+    const nodes = await this.retriever.retrieve(
+      query,
+      parentEvent,
+      this.preFilters,
+    );
+
+    return this.applyNodePostprocessors(nodes);
   }
 
   async query(query: string, parentEvent?: Event) {
@@ -48,7 +72,7 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
       type: "wrapper",
       tags: ["final"],
     };
-    const nodes = await this.retriever.retrieve(query, _parentEvent);
+    const nodes = await this.retrieve(query, _parentEvent);
     return this.responseSynthesizer.synthesize(query, nodes, _parentEvent);
   }
 }

packages/core/src/ResponseSynthesizer.ts

@@ -1,18 +1,18 @@
+import { Event } from "./callbacks/CallbackManager";
+import { LLM } from "./llm/LLM";
 import { MetadataMode, NodeWithScore } from "./Node";
 import {
+  defaultRefinePrompt,
+  defaultTextQaPrompt,
+  defaultTreeSummarizePrompt,
   RefinePrompt,
   SimplePrompt,
   TextQaPrompt,
   TreeSummarizePrompt,
-  defaultRefinePrompt,
-  defaultTextQaPrompt,
-  defaultTreeSummarizePrompt,
 } from "./Prompt";
 import { getBiggestPrompt } from "./PromptHelper";
 import { Response } from "./Response";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { Event } from "./callbacks/CallbackManager";
-import { LLM } from "./llm/LLM";
 
 /**
  * Response modes of the response synthesizer
@@ -231,6 +231,7 @@ export class TreeSummarize implements BaseResponseBuilder {
       throw new Error("Must have at least one text chunk");
     }
 
+    // Should we send the query here too?
     const packedTextChunks = this.serviceContext.promptHelper.repack(
       this.summaryTemplate,
       textChunks,
@@ -241,6 +242,7 @@ export class TreeSummarize implements BaseResponseBuilder {
         await this.serviceContext.llm.complete(
           this.summaryTemplate({
             context: packedTextChunks[0],
+            query,
           }),
           parentEvent,
         )
@@ -251,6 +253,7 @@ export class TreeSummarize implements BaseResponseBuilder {
           this.serviceContext.llm.complete(
             this.summaryTemplate({
               context: chunk,
+              query,
             }),
             parentEvent,
           ),

packages/core/src/Retriever.ts

@@ -1,11 +1,15 @@
+import { Event } from "./callbacks/CallbackManager";
 import { NodeWithScore } from "./Node";
 import { ServiceContext } from "./ServiceContext";
-import { Event } from "./callbacks/CallbackManager";
 
 /**
  * Retrievers retrieve the nodes that most closely match our query in similarity.
  */
 export interface BaseRetriever {
-  retrieve(query: string, parentEvent?: Event): Promise<NodeWithScore[]>;
+  retrieve(
+    query: string,
+    parentEvent?: Event,
+    preFilters?: unknown,
+  ): Promise<NodeWithScore[]>;
   getServiceContext(): ServiceContext;
 }

packages/core/src/callbacks/CallbackManager.ts

@@ -20,7 +20,8 @@ interface BaseCallbackResponse {
   event: Event;
 }
 
-export interface StreamToken {
+//Specify StreamToken per mainstream LLM
+export interface DefaultStreamToken {
   id: string;
   object: string;
   created: number;
@@ -29,16 +30,34 @@ export interface StreamToken {
     index: number;
     delta: {
       content?: string | null;
-      role?: "user" | "assistant" | "system" | "function";
+      role?: "user" | "assistant" | "system" | "function" | "tool";
     };
     finish_reason: string | null;
   }[];
 }
 
+//OpenAI stream token schema is the default.
+//Note: Anthropic and Replicate also use similar token schemas.
+export type OpenAIStreamToken = DefaultStreamToken;
+export type AnthropicStreamToken = {
+  completion: string;
+  model: string;
+  stop_reason: string | undefined;
+  stop?: boolean | undefined;
+  log_id?: string;
+};
+
+//
+//Callback Responses
+//
+//TODO: Write Embedding Callbacks
+
+//StreamCallbackResponse should let practitioners implement callbacks out of the box...
+//When custom streaming LLMs are involved, people are expected to write their own StreamCallbackResponses
 export interface StreamCallbackResponse extends BaseCallbackResponse {
   index: number;
   isDone?: boolean;
-  token?: StreamToken;
+  token?: DefaultStreamToken;
 }
 
 export interface RetrievalCallbackResponse extends BaseCallbackResponse {

packages/core/src/callbacks/utility/handleOpenAIStream.ts

@@ -1,45 +0,0 @@
-import { ChatCompletionChunk } from "openai/resources/chat";
-import { Stream } from "openai/streaming";
-import { globalsHelper } from "../../GlobalsHelper";
-import { MessageType } from "../../llm/LLM";
-import { Event, StreamCallbackResponse } from "../CallbackManager";
-
-/**
- * Handles the OpenAI streaming interface and pipes it to the callback function
- * @param response - The response from the OpenAI API.
- * @param onLLMStream - A callback function to handle the LLM stream.
- * @param parentEvent - An optional parent event.
- * @returns A promise that resolves to an object with a message and a role.
- */
-export async function handleOpenAIStream({
-  response,
-  onLLMStream,
-  parentEvent,
-}: {
-  response: Stream<ChatCompletionChunk>;
-  onLLMStream: (data: StreamCallbackResponse) => void;
-  parentEvent?: Event;
-}): Promise<{ message: string; role: MessageType }> {
-  const event = globalsHelper.createEvent({
-    parentEvent,
-    type: "llmPredict",
-  });
-  let index = 0;
-  let cumulativeText = "";
-  let messageRole: MessageType = "assistant";
-  for await (const part of response) {
-    const { content = "", role = "assistant" } = part.choices[0].delta;
-
-    // ignore the first token
-    if (!content && role === "assistant" && index === 0) {
-      continue;
-    }
-
-    cumulativeText += content;
-    messageRole = role;
-    onLLMStream?.({ event, index, token: part });
-    index++;
-  }
-  onLLMStream?.({ event, index, isDone: true });
-  return { message: cumulativeText, role: messageRole };
-}

packages/core/src/index.ts

@@ -1,30 +1,29 @@
+export * from "./callbacks/CallbackManager";
 export * from "./ChatEngine";
+export * from "./ChatHistory";
+export * from "./constants";
 export * from "./Embedding";
 export * from "./GlobalsHelper";
+export * from "./indices";
+export * from "./llm/LLM";
 export * from "./Node";
 export * from "./NodeParser";
 export * from "./OutputParser";
 export * from "./Prompt";
+export * from "./PromptHelper";
 export * from "./QueryEngine";
 export * from "./QuestionGenerator";
-export * from "./Response";
-export * from "./ResponseSynthesizer";
-export * from "./Retriever";
-export * from "./ServiceContext";
-export * from "./TextSplitter";
-export * from "./Tool";
-export * from "./constants";
-export * from "./llm/LLM";
-
-export * from "./indices";
-
-export * from "./callbacks/CallbackManager";
-
+export * from "./readers/base";
 export * from "./readers/CSVReader";
 export * from "./readers/MarkdownReader";
 export * from "./readers/NotionReader";
 export * from "./readers/PDFReader";
+export * from "./readers/HTMLReader";
 export * from "./readers/SimpleDirectoryReader";
-export * from "./readers/base";
-
+export * from "./Response";
+export * from "./ResponseSynthesizer";
+export * from "./Retriever";
+export * from "./ServiceContext";
 export * from "./storage";
+export * from "./TextSplitter";
+export * from "./Tool";

packages/core/src/indices/BaseIndex.ts

@@ -39,6 +39,7 @@ export abstract class IndexStruct {
 export enum IndexStructType {
   SIMPLE_DICT = "simple_dict",
   LIST = "list",
+  KEYWORD_TABLE = "keyword_table",
 }
 
 export class IndexDict extends IndexStruct {
@@ -106,6 +107,36 @@ export class IndexList extends IndexStruct {
   }
 }
 
+// A table of keywords mapping keywords to text chunks.
+export class KeywordTable extends IndexStruct {
+  table: Map<string, Set<string>> = new Map();
+  type: IndexStructType = IndexStructType.KEYWORD_TABLE;
+  addNode(keywords: string[], nodeId: string): void {
+    keywords.forEach((keyword) => {
+      if (!this.table.has(keyword)) {
+        this.table.set(keyword, new Set());
+      }
+      this.table.get(keyword)!.add(nodeId);
+    });
+  }
+
+  deleteNode(keywords: string[], nodeId: string) {
+    keywords.forEach((keyword) => {
+      if (this.table.has(keyword)) {
+        this.table.get(keyword)!.delete(nodeId);
+      }
+    });
+  }
+
+  toJson(): Record<string, unknown> {
+    return {
+      ...super.toJson(),
+      table: this.table,
+      type: this.type,
+    };
+  }
+}
+
 export interface BaseIndexInit<T> {
   serviceContext: ServiceContext;
   storageContext: StorageContext;

packages/core/src/indices/BaseNodePostprocessor.ts

@@ -0,0 +1,20 @@
+import { NodeWithScore } from "../Node";
+
+export interface BaseNodePostprocessor {
+  postprocessNodes: (nodes: NodeWithScore[]) => NodeWithScore[];
+}
+
+export class SimilarityPostprocessor implements BaseNodePostprocessor {
+  similarityCutoff?: number;
+
+  constructor(options?: { similarityCutoff?: number }) {
+    this.similarityCutoff = options?.similarityCutoff;
+  }
+
+  postprocessNodes(nodes: NodeWithScore[]) {
+    if (this.similarityCutoff === undefined) return nodes;
+
+    const cutoff = this.similarityCutoff || 0;
+    return nodes.filter((node) => node.score && node.score >= cutoff);
+  }
+}

packages/core/src/indices/index.ts

@@ -1,3 +1,5 @@
 export * from "./BaseIndex";
+export * from "./BaseNodePostprocessor";
+export * from "./keyword";
 export * from "./summary";
 export * from "./vectorStore";

packages/core/src/indices/keyword/KeywordTableIndex.ts

@@ -0,0 +1,274 @@
+import { BaseNode, Document, MetadataMode } from "../../Node";
+import { defaultKeywordExtractPrompt } from "../../Prompt";
+import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
+import { ResponseSynthesizer } from "../../ResponseSynthesizer";
+import { BaseRetriever } from "../../Retriever";
+import {
+  ServiceContext,
+  serviceContextFromDefaults,
+} from "../../ServiceContext";
+import { StorageContext, storageContextFromDefaults } from "../../storage";
+import { BaseDocumentStore } from "../../storage/docStore/types";
+import {
+  BaseIndex,
+  BaseIndexInit,
+  IndexStructType,
+  KeywordTable,
+} from "../BaseIndex";
+import { BaseNodePostprocessor } from "../BaseNodePostprocessor";
+import {
+  KeywordTableLLMRetriever,
+  KeywordTableRAKERetriever,
+  KeywordTableSimpleRetriever,
+} from "./KeywordTableIndexRetriever";
+import { extractKeywordsGivenResponse } from "./utils";
+
+export interface KeywordIndexOptions {
+  nodes?: BaseNode[];
+  indexStruct?: KeywordTable;
+  indexId?: string;
+  serviceContext?: ServiceContext;
+  storageContext?: StorageContext;
+}
+export enum KeywordTableRetrieverMode {
+  DEFAULT = "DEFAULT",
+  SIMPLE = "SIMPLE",
+  RAKE = "RAKE",
+}
+
+const KeywordTableRetrieverMap = {
+  [KeywordTableRetrieverMode.DEFAULT]: KeywordTableLLMRetriever,
+  [KeywordTableRetrieverMode.SIMPLE]: KeywordTableSimpleRetriever,
+  [KeywordTableRetrieverMode.RAKE]: KeywordTableRAKERetriever,
+};
+
+/**
+ * The KeywordTableIndex, an index that extracts keywords from each Node and builds a mapping from each keyword to the corresponding Nodes of that keyword.
+ */
+export class KeywordTableIndex extends BaseIndex<KeywordTable> {
+  constructor(init: BaseIndexInit<KeywordTable>) {
+    super(init);
+  }
+
+  static async init(options: KeywordIndexOptions): Promise<KeywordTableIndex> {
+    const storageContext =
+      options.storageContext ?? (await storageContextFromDefaults({}));
+    const serviceContext =
+      options.serviceContext ?? serviceContextFromDefaults({});
+    const { docStore, indexStore } = storageContext;
+
+    // Setup IndexStruct from storage
+    let indexStructs = (await indexStore.getIndexStructs()) as KeywordTable[];
+    let indexStruct: KeywordTable | null;
+
+    if (options.indexStruct && indexStructs.length > 0) {
+      throw new Error(
+        "Cannot initialize index with both indexStruct and indexStore",
+      );
+    }
+
+    if (options.indexStruct) {
+      indexStruct = options.indexStruct;
+    } else if (indexStructs.length == 1) {
+      indexStruct = indexStructs[0];
+    } else if (indexStructs.length > 1 && options.indexId) {
+      indexStruct = (await indexStore.getIndexStruct(
+        options.indexId,
+      )) as KeywordTable;
+    } else {
+      indexStruct = null;
+    }
+
+    // check indexStruct type
+    if (indexStruct && indexStruct.type !== IndexStructType.KEYWORD_TABLE) {
+      throw new Error(
+        "Attempting to initialize KeywordTableIndex with non-keyword table indexStruct",
+      );
+    }
+
+    if (indexStruct) {
+      if (options.nodes) {
+        throw new Error(
+          "Cannot initialize KeywordTableIndex with both nodes and indexStruct",
+        );
+      }
+    } else {
+      if (!options.nodes) {
+        throw new Error(
+          "Cannot initialize KeywordTableIndex without nodes or indexStruct",
+        );
+      }
+      indexStruct = await KeywordTableIndex.buildIndexFromNodes(
+        options.nodes,
+        storageContext.docStore,
+        serviceContext,
+      );
+
+      await indexStore.addIndexStruct(indexStruct);
+    }
+
+    return new KeywordTableIndex({
+      storageContext,
+      serviceContext,
+      docStore,
+      indexStore,
+      indexStruct,
+    });
+  }
+
+  asRetriever(options?: any): BaseRetriever {
+    const { mode = KeywordTableRetrieverMode.DEFAULT, ...otherOptions } =
+      options ?? {};
+    const KeywordTableRetriever =
+      KeywordTableRetrieverMap[mode as KeywordTableRetrieverMode];
+    if (KeywordTableRetriever) {
+      return new KeywordTableRetriever({ index: this, ...otherOptions });
+    }
+    throw new Error(`Unknown retriever mode: ${mode}`);
+  }
+
+  asQueryEngine(options?: {
+    retriever?: BaseRetriever;
+    responseSynthesizer?: ResponseSynthesizer;
+    preFilters?: unknown;
+    nodePostprocessors?: BaseNodePostprocessor[];
+  }): BaseQueryEngine {
+    const { retriever, responseSynthesizer } = options ?? {};
+    return new RetrieverQueryEngine(
+      retriever ?? this.asRetriever(),
+      responseSynthesizer,
+      options?.preFilters,
+      options?.nodePostprocessors,
+    );
+  }
+
+  static async extractKeywords(
+    text: string,
+    serviceContext: ServiceContext,
+  ): Promise<Set<string>> {
+    const response = await serviceContext.llm.complete(
+      defaultKeywordExtractPrompt({
+        context: text,
+      }),
+    );
+    return extractKeywordsGivenResponse(response.message.content, "KEYWORDS:");
+  }
+
+  /**
+   * High level API: split documents, get keywords, and build index.
+   * @param documents
+   * @param storageContext
+   * @param serviceContext
+   * @returns
+   */
+  static async fromDocuments(
+    documents: Document[],
+    args: {
+      storageContext?: StorageContext;
+      serviceContext?: ServiceContext;
+    } = {},
+  ): Promise<KeywordTableIndex> {
+    let { storageContext, serviceContext } = args;
+    storageContext = storageContext ?? (await storageContextFromDefaults({}));
+    serviceContext = serviceContext ?? serviceContextFromDefaults({});
+    const docStore = storageContext.docStore;
+
+    docStore.addDocuments(documents, true);
+    for (const doc of documents) {
+      docStore.setDocumentHash(doc.id_, doc.hash);
+    }
+
+    const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
+    const index = await KeywordTableIndex.init({
+      nodes,
+      storageContext,
+      serviceContext,
+    });
+    return index;
+  }
+
+  /**
+   * Get keywords for nodes and place them into the index.
+   * @param nodes
+   * @param serviceContext
+   * @param vectorStore
+   * @returns
+   */
+  static async buildIndexFromNodes(
+    nodes: BaseNode[],
+    docStore: BaseDocumentStore,
+    serviceContext: ServiceContext,
+  ): Promise<KeywordTable> {
+    const indexStruct = new KeywordTable();
+    await docStore.addDocuments(nodes, true);
+    for (const node of nodes) {
+      const keywords = await KeywordTableIndex.extractKeywords(
+        node.getContent(MetadataMode.LLM),
+        serviceContext,
+      );
+      indexStruct.addNode([...keywords], node.id_);
+    }
+    return indexStruct;
+  }
+
+  async insertNodes(nodes: BaseNode[]) {
+    for (let node of nodes) {
+      const keywords = await KeywordTableIndex.extractKeywords(
+        node.getContent(MetadataMode.LLM),
+        this.serviceContext,
+      );
+      this.indexStruct.addNode([...keywords], node.id_);
+    }
+  }
+
+  deleteNode(nodeId: string): void {
+    const keywordsToDelete: Set<string> = new Set();
+    for (const [keyword, existingNodeIds] of Object.entries(
+      this.indexStruct.table,
+    )) {
+      const index = existingNodeIds.indexOf(nodeId);
+      if (index !== -1) {
+        existingNodeIds.splice(index, 1);
+
+        // Delete keywords that have zero nodes
+        if (existingNodeIds.length === 0) {
+          keywordsToDelete.add(keyword);
+        }
+      }
+    }
+    this.indexStruct.deleteNode([...keywordsToDelete], nodeId);
+  }
+
+  async deleteNodes(nodeIds: string[], deleteFromDocStore: boolean) {
+    nodeIds.forEach((nodeId) => {
+      this.deleteNode(nodeId);
+    });
+
+    if (deleteFromDocStore) {
+      for (const nodeId of nodeIds) {
+        await this.docStore.deleteDocument(nodeId, false);
+      }
+    }
+
+    await this.storageContext.indexStore.addIndexStruct(this.indexStruct);
+  }
+
+  async deleteRefDoc(
+    refDocId: string,
+    deleteFromDocStore?: boolean,
+  ): Promise<void> {
+    const refDocInfo = await this.docStore.getRefDocInfo(refDocId);
+
+    if (!refDocInfo) {
+      return;
+    }
+
+    await this.deleteNodes(refDocInfo.nodeIds, false);
+
+    if (deleteFromDocStore) {
+      await this.docStore.deleteRefDoc(refDocId, false);
+    }
+
+    return;
+  }
+}

packages/core/src/indices/keyword/KeywordTableIndexRetriever.ts

@@ -0,0 +1,119 @@
+import { NodeWithScore } from "../../Node";
+import {
+  defaultKeywordExtractPrompt,
+  defaultQueryKeywordExtractPrompt,
+  KeywordExtractPrompt,
+  QueryKeywordExtractPrompt,
+} from "../../Prompt";
+import { BaseRetriever } from "../../Retriever";
+import { ServiceContext } from "../../ServiceContext";
+import { BaseDocumentStore } from "../../storage/docStore/types";
+import { KeywordTable } from "../BaseIndex";
+import { KeywordTableIndex } from "./KeywordTableIndex";
+import {
+  extractKeywordsGivenResponse,
+  rakeExtractKeywords,
+  simpleExtractKeywords,
+} from "./utils";
+
+// Base Keyword Table Retriever
+abstract class BaseKeywordTableRetriever implements BaseRetriever {
+  protected index: KeywordTableIndex;
+  protected indexStruct: KeywordTable;
+  protected docstore: BaseDocumentStore;
+  protected serviceContext: ServiceContext;
+
+  protected maxKeywordsPerQuery: number; // Maximum number of keywords to extract from query.
+  protected numChunksPerQuery: number; // Maximum number of text chunks to query.
+  protected keywordExtractTemplate: KeywordExtractPrompt; // A Keyword Extraction Prompt
+  protected queryKeywordExtractTemplate: QueryKeywordExtractPrompt; // A Query Keyword Extraction Prompt
+
+  constructor({
+    index,
+    keywordExtractTemplate,
+    queryKeywordExtractTemplate,
+    maxKeywordsPerQuery = 10,
+    numChunksPerQuery = 10,
+  }: {
+    index: KeywordTableIndex;
+    keywordExtractTemplate?: KeywordExtractPrompt;
+    queryKeywordExtractTemplate?: QueryKeywordExtractPrompt;
+    maxKeywordsPerQuery: number;
+    numChunksPerQuery: number;
+  }) {
+    this.index = index;
+    this.indexStruct = index.indexStruct;
+    this.docstore = index.docStore;
+    this.serviceContext = index.serviceContext;
+
+    this.maxKeywordsPerQuery = maxKeywordsPerQuery;
+    this.numChunksPerQuery = numChunksPerQuery;
+    this.keywordExtractTemplate =
+      keywordExtractTemplate || defaultKeywordExtractPrompt;
+    this.queryKeywordExtractTemplate =
+      queryKeywordExtractTemplate || defaultQueryKeywordExtractPrompt;
+  }
+
+  abstract getKeywords(query: string): Promise<string[]>;
+
+  async retrieve(query: string): Promise<NodeWithScore[]> {
+    const keywords = await this.getKeywords(query);
+    const chunkIndicesCount: { [key: string]: number } = {};
+    const filteredKeywords = keywords.filter((keyword) =>
+      this.indexStruct.table.has(keyword),
+    );
+
+    for (let keyword of filteredKeywords) {
+      for (let nodeId of this.indexStruct.table.get(keyword) || []) {
+        chunkIndicesCount[nodeId] = (chunkIndicesCount[nodeId] ?? 0) + 1;
+      }
+    }
+
+    const sortedChunkIndices = Object.keys(chunkIndicesCount)
+      .sort((a, b) => chunkIndicesCount[b] - chunkIndicesCount[a])
+      .slice(0, this.numChunksPerQuery);
+
+    const sortedNodes = await this.docstore.getNodes(sortedChunkIndices);
+
+    return sortedNodes.map((node) => ({ node }));
+  }
+
+  getServiceContext(): ServiceContext {
+    return this.index.serviceContext;
+  }
+}
+
+// Extracts keywords using LLMs.
+export class KeywordTableLLMRetriever extends BaseKeywordTableRetriever {
+  async getKeywords(query: string): Promise<string[]> {
+    const response = await this.serviceContext.llm.complete(
+      this.queryKeywordExtractTemplate({
+        question: query,
+        maxKeywords: this.maxKeywordsPerQuery,
+      }),
+    );
+    const keywords = extractKeywordsGivenResponse(
+      response.message.content,
+      "KEYWORDS:",
+    );
+    return [...keywords];
+  }
+}
+
+// Extracts keywords using simple regex-based keyword extractor.
+export class KeywordTableSimpleRetriever extends BaseKeywordTableRetriever {
+  getKeywords(query: string): Promise<string[]> {
+    return Promise.resolve([
+      ...simpleExtractKeywords(query, this.maxKeywordsPerQuery),
+    ]);
+  }
+}
+
+// Extracts keywords using RAKE keyword extractor
+export class KeywordTableRAKERetriever extends BaseKeywordTableRetriever {
+  getKeywords(query: string): Promise<string[]> {
+    return Promise.resolve([
+      ...rakeExtractKeywords(query, this.maxKeywordsPerQuery),
+    ]);
+  }
+}

packages/core/src/indices/keyword/index.ts

@@ -0,0 +1,9 @@
+export {
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "./KeywordTableIndex";
+export {
+  KeywordTableLLMRetriever,
+  KeywordTableRAKERetriever,
+  KeywordTableSimpleRetriever,
+} from "./KeywordTableIndexRetriever";

packages/core/src/indices/keyword/utils.ts

@@ -0,0 +1,81 @@
+// @ts-ignore
+import rake from "rake-modified";
+
+// Get subtokens from a list of tokens., filtering for stopwords.
+export function expandTokensWithSubtokens(tokens: Set<string>): Set<string> {
+  const results: Set<string> = new Set();
+  const regex: RegExp = /\w+/g;
+
+  for (let token of tokens) {
+    results.add(token);
+    const subTokens: RegExpMatchArray | null = token.match(regex);
+    if (subTokens && subTokens.length > 1) {
+      for (let w of subTokens) {
+        results.add(w);
+      }
+    }
+  }
+  return results;
+}
+
+export function extractKeywordsGivenResponse(
+  response: string,
+  startToken: string = "",
+  lowercase: boolean = true,
+): Set<string> {
+  const results: string[] = [];
+  response = response.trim();
+
+  if (response.startsWith(startToken)) {
+    response = response.substring(startToken.length);
+  }
+
+  const keywords: string[] = response.split(",");
+  for (let k of keywords) {
+    let rk: string = k;
+    if (lowercase) {
+      rk = rk.toLowerCase();
+    }
+    results.push(rk.trim());
+  }
+
+  return expandTokensWithSubtokens(new Set(results));
+}
+
+export function simpleExtractKeywords(
+  textChunk: string,
+  maxKeywords?: number,
+): Set<string> {
+  const regex: RegExp = /\w+/g;
+  let tokens: string[] = [...textChunk.matchAll(regex)].map((token) =>
+    token[0].toLowerCase().trim(),
+  );
+
+  // Creating a frequency map
+  const valueCounts: { [key: string]: number } = {};
+  for (let token of tokens) {
+    valueCounts[token] = (valueCounts[token] || 0) + 1;
+  }
+
+  // Sorting tokens by frequency
+  const sortedTokens: string[] = Object.keys(valueCounts).sort(
+    (a, b) => valueCounts[b] - valueCounts[a],
+  );
+
+  const keywords: string[] = maxKeywords
+    ? sortedTokens.slice(0, maxKeywords)
+    : sortedTokens;
+
+  return new Set(keywords);
+}
+
+export function rakeExtractKeywords(
+  textChunk: string,
+  maxKeywords?: number,
+): Set<string> {
+  const keywords = Object.keys(rake(textChunk));
+  const limitedKeywords = maxKeywords
+    ? keywords.slice(0, maxKeywords)
+    : keywords;
+  return new Set(limitedKeywords);
+}

packages/core/src/indices/summary/SummaryIndex.ts

@@ -10,17 +10,18 @@ import {
   ServiceContext,
   serviceContextFromDefaults,
 } from "../../ServiceContext";
+import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   StorageContext,
   storageContextFromDefaults,
 } from "../../storage/StorageContext";
-import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   BaseIndex,
   BaseIndexInit,
   IndexList,
   IndexStructType,
 } from "../BaseIndex";
+import { BaseNodePostprocessor } from "../BaseNodePostprocessor";
 import {
   SummaryIndexLLMRetriever,
   SummaryIndexRetriever,
@@ -155,6 +156,8 @@ export class SummaryIndex extends BaseIndex<IndexList> {
   asQueryEngine(options?: {
     retriever?: BaseRetriever;
     responseSynthesizer?: ResponseSynthesizer;
+    preFilters?: unknown;
+    nodePostprocessors?: BaseNodePostprocessor[];
   }): BaseQueryEngine {
     let { retriever, responseSynthesizer } = options ?? {};
 
@@ -170,7 +173,12 @@ export class SummaryIndex extends BaseIndex<IndexList> {
       });
     }
 
-    return new RetrieverQueryEngine(retriever, responseSynthesizer);
+    return new RetrieverQueryEngine(
+      retriever,
+      responseSynthesizer,
+      options?.preFilters,
+      options?.nodePostprocessors,
+    );
   }
 
   static async buildIndexFromNodes(

packages/core/src/indices/vectorStore/VectorIndexRetriever.ts

@@ -1,9 +1,9 @@
+import { Event } from "../../callbacks/CallbackManager";
+import { DEFAULT_SIMILARITY_TOP_K } from "../../constants";
 import { globalsHelper } from "../../GlobalsHelper";
 import { NodeWithScore } from "../../Node";
 import { BaseRetriever } from "../../Retriever";
 import { ServiceContext } from "../../ServiceContext";
-import { Event } from "../../callbacks/CallbackManager";
-import { DEFAULT_SIMILARITY_TOP_K } from "../../constants";
 import {
   VectorStoreQuery,
   VectorStoreQueryMode,
@@ -32,7 +32,7 @@ export class VectorIndexRetriever implements BaseRetriever {
     this.similarityTopK = similarityTopK ?? DEFAULT_SIMILARITY_TOP_K;
   }
 
-  async retrieve(query: string, parentEvent?: Event): Promise<NodeWithScore[]> {
+  async retrieve(query: string, parentEvent?: Event, preFilters?: unknown): Promise<NodeWithScore[]> {
     const queryEmbedding =
       await this.serviceContext.embedModel.getQueryEmbedding(query);
 
@@ -41,10 +41,15 @@ export class VectorIndexRetriever implements BaseRetriever {
       mode: VectorStoreQueryMode.DEFAULT,
       similarityTopK: this.similarityTopK,
     };
-    const result = await this.index.vectorStore.query(q);
+    const result = await this.index.vectorStore.query(q, preFilters);
 
     let nodesWithScores: NodeWithScore[] = [];
     for (let i = 0; i < result.ids.length; i++) {
+      const nodeFromResult = result.nodes?.[i];
+      if (!this.index.indexStruct.nodesDict[result.ids[i]] && nodeFromResult) {
+        this.index.indexStruct.nodesDict[result.ids[i]] = nodeFromResult;
+      }
+
       const node = this.index.indexStruct.nodesDict[result.ids[i]];
       nodesWithScores.push({
         node: node,

packages/core/src/indices/vectorStore/VectorStoreIndex.ts

@@ -18,6 +18,7 @@ import {
   IndexDict,
   IndexStructType,
 } from "../BaseIndex";
+import { BaseNodePostprocessor } from "../BaseNodePostprocessor";
 import { VectorIndexRetriever } from "./VectorIndexRetriever";
 
 export interface VectorIndexOptions {
@@ -87,24 +88,23 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
       );
     }
 
-    if (!indexStruct && !options.nodes) {
+    if (options.nodes) {
+      // If nodes are passed in, then we need to update the index
+      indexStruct = await VectorStoreIndex.buildIndexFromNodes(
+        options.nodes,
+        serviceContext,
+        vectorStore,
+        docStore,
+        indexStruct,
+      );
+
+      await indexStore.addIndexStruct(indexStruct);
+    } else if (!indexStruct) {
       throw new Error(
         "Cannot initialize VectorStoreIndex without nodes or indexStruct",
       );
     }
 
-    const nodes = options.nodes ?? [];
-
-    indexStruct = await VectorStoreIndex.buildIndexFromNodes(
-      nodes,
-      serviceContext,
-      vectorStore,
-      docStore,
-      indexStruct,
-    );
-
-    await indexStore.addIndexStruct(indexStruct);
-
     return new VectorStoreIndex({
       storageContext,
       serviceContext,
@@ -219,6 +219,27 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
     return index;
   }
 
+  static async fromVectorStore(
+    vectorStore: VectorStore,
+    serviceContext: ServiceContext,
+  ) {
+    if (!vectorStore.storesText) {
+      throw new Error(
+        "Cannot initialize from a vector store that does not store text",
+      );
+    }
+
+    const storageContext = await storageContextFromDefaults({ vectorStore });
+
+    const index = await VectorStoreIndex.init({
+      nodes: [],
+      storageContext,
+      serviceContext,
+    });
+
+    return index;
+  }
+
   asRetriever(options?: any): VectorIndexRetriever {
     return new VectorIndexRetriever({ index: this, ...options });
   }
@@ -226,11 +247,15 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
   asQueryEngine(options?: {
     retriever?: BaseRetriever;
     responseSynthesizer?: ResponseSynthesizer;
+    preFilters?: unknown;
+    nodePostprocessors?: BaseNodePostprocessor[];
   }): BaseQueryEngine {
     const { retriever, responseSynthesizer } = options ?? {};
     return new RetrieverQueryEngine(
       retriever ?? this.asRetriever(),
       responseSynthesizer,
+      options?.preFilters,
+      options?.nodePostprocessors,
     );
   }
 

packages/core/src/llm/LLM.ts

@@ -1,6 +1,16 @@
 import OpenAILLM, { ClientOptions as OpenAIClientOptions } from "openai";
-import { CallbackManager, Event } from "../callbacks/CallbackManager";
-import { handleOpenAIStream } from "../callbacks/utility/handleOpenAIStream";
+import {
+  AnthropicStreamToken,
+  CallbackManager,
+  Event,
+  EventType,
+  OpenAIStreamToken,
+  StreamCallbackResponse,
+} from "../callbacks/CallbackManager";
+
+import { ChatCompletionMessageParam } from "openai/resources";
+import { LLMOptions } from "portkey-ai";
+import { globalsHelper, Tokenizers } from "../GlobalsHelper";
 import {
   ANTHROPIC_AI_PROMPT,
   ANTHROPIC_HUMAN_PROMPT,
@@ -14,7 +24,8 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure";
-import { OpenAISession, getOpenAISession } from "./openai";
+import { getOpenAISession, OpenAISession } from "./openai";
+import { getPortkeySession, PortkeySession } from "./portkey";
 import { ReplicateSession } from "./replicate";
 
 export type MessageType =
@@ -22,10 +33,11 @@ export type MessageType =
   | "assistant"
   | "system"
   | "generic"
-  | "function";
+  | "function"
+  | "memory";
 
 export interface ChatMessage {
-  content: string;
+  content: any;
   role: MessageType;
 }
 
@@ -38,31 +50,67 @@ export interface ChatResponse {
 // NOTE in case we need CompletionResponse to diverge from ChatResponse in the future
 export type CompletionResponse = ChatResponse;
 
+export interface LLMMetadata {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  contextWindow: number;
+  tokenizer: Tokenizers | undefined;
+}
+
 /**
  * Unified language model interface
  */
 export interface LLM {
+  metadata: LLMMetadata;
+  // Whether a LLM has streaming support
+  hasStreaming: boolean;
   /**
    * Get a chat response from the LLM
    * @param messages
+   *
+   * The return type of chat() and complete() are set by the "streaming" parameter being set to True.
    */
-  chat(messages: ChatMessage[], parentEvent?: Event): Promise<ChatResponse>;
+  chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
+    messages: ChatMessage[],
+    parentEvent?: Event,
+    streaming?: T,
+  ): Promise<R>;
 
   /**
    * Get a prompt completion from the LLM
    * @param prompt the prompt to complete
    */
-  complete(prompt: string, parentEvent?: Event): Promise<CompletionResponse>;
+  complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
+    prompt: string,
+    parentEvent?: Event,
+    streaming?: T,
+  ): Promise<R>;
+
+  /**
+   * Calculates the number of tokens needed for the given chat messages
+   */
+  tokens(messages: ChatMessage[]): number;
 }
 
 export const GPT4_MODELS = {
   "gpt-4": { contextWindow: 8192 },
   "gpt-4-32k": { contextWindow: 32768 },
+  "gpt-4-1106-preview": { contextWindow: 128000 },
+  "gpt-4-vision-preview": { contextWindow: 8192 },
 };
 
-export const TURBO_MODELS = {
+export const GPT35_MODELS = {
   "gpt-3.5-turbo": { contextWindow: 4096 },
   "gpt-3.5-turbo-16k": { contextWindow: 16384 },
+  "gpt-3.5-turbo-1106": { contextWindow: 16384 },
 };
 
 /**
@@ -70,20 +118,22 @@ export const TURBO_MODELS = {
  */
 export const ALL_AVAILABLE_OPENAI_MODELS = {
   ...GPT4_MODELS,
-  ...TURBO_MODELS,
+  ...GPT35_MODELS,
 };
 
 /**
  * OpenAI LLM implementation
  */
 export class OpenAI implements LLM {
+  hasStreaming: boolean = true;
+
   // Per completion OpenAI params
   model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS;
   temperature: number;
   topP: number;
   maxTokens?: number;
   additionalChatOptions?: Omit<
-    Partial<OpenAILLM.Chat.CompletionCreateParams>,
+    Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
     "max_tokens" | "messages" | "model" | "temperature" | "top_p" | "streaming"
   >;
 
@@ -153,6 +203,32 @@ export class OpenAI implements LLM {
     this.callbackManager = init?.callbackManager;
   }
 
+  get metadata() {
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow: ALL_AVAILABLE_OPENAI_MODELS[this.model].contextWindow,
+      tokenizer: Tokenizers.CL100K_BASE,
+    };
+  }
+
+  tokens(messages: ChatMessage[]): number {
+    // for latest OpenAI models, see https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+    const tokenizer = globalsHelper.tokenizer(this.metadata.tokenizer);
+    const tokensPerMessage = 3;
+    let numTokens = 0;
+    for (const message of messages) {
+      numTokens += tokensPerMessage;
+      for (const value of Object.values(message)) {
+        numTokens += tokenizer(value).length;
+      }
+    }
+    numTokens += 3; // every reply is primed with <|im_start|>assistant<|im_sep|>
+    return numTokens;
+  }
+
   mapMessageType(
     messageType: MessageType,
   ): "user" | "assistant" | "system" | "function" {
@@ -170,52 +246,124 @@ export class OpenAI implements LLM {
     }
   }
 
-  async chat(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-  ): Promise<ChatResponse> {
-    const baseRequestParams: OpenAILLM.Chat.CompletionCreateParams = {
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(messages: ChatMessage[], parentEvent?: Event, streaming?: T): Promise<R> {
+    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
       model: this.model,
       temperature: this.temperature,
       max_tokens: this.maxTokens,
-      messages: messages.map((message) => ({
-        role: this.mapMessageType(message.role),
-        content: message.content,
-      })),
+      messages: messages.map(
+        (message) =>
+          ({
+            role: this.mapMessageType(message.role),
+            content: message.content,
+          }) as ChatCompletionMessageParam,
+      ),
+      top_p: this.topP,
+      ...this.additionalChatOptions,
+    };
+    // Streaming
+    if (streaming) {
+      if (!this.hasStreaming) {
+        throw Error("No streaming support for this LLM.");
+      }
+      return this.streamChat(messages, parentEvent) as R;
+    }
+    // Non-streaming
+    const response = await this.session.openai.chat.completions.create({
+      ...baseRequestParams,
+      stream: false,
+    });
+
+    const content = response.choices[0].message?.content ?? "";
+    return {
+      message: { content, role: response.choices[0].message.role },
+    } as R;
+  }
+
+  async complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
+    return this.chat(
+      [{ content: prompt, role: "user" }],
+      parentEvent,
+      streaming,
+    );
+  }
+
+  //We can wrap a stream in a generator to add some additional logging behavior
+  //For future edits: syntax for generator type is <typeof Yield, typeof Return, typeof Accept>
+  //"typeof Accept" refers to what types you'll accept when you manually call generator.next(<AcceptType>)
+  protected async *streamChat(
+    messages: ChatMessage[],
+    parentEvent?: Event,
+  ): AsyncGenerator<string, void, unknown> {
+    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
+      model: this.model,
+      temperature: this.temperature,
+      max_tokens: this.maxTokens,
+      messages: messages.map(
+        (message) =>
+          ({
+            role: this.mapMessageType(message.role),
+            content: message.content,
+          }) as ChatCompletionMessageParam,
+      ),
       top_p: this.topP,
       ...this.additionalChatOptions,
     };
 
-    if (this.callbackManager?.onLLMStream) {
-      // Streaming
-      const response = await this.session.openai.chat.completions.create({
+    //Now let's wrap our stream in a callback
+    const onLLMStream = this.callbackManager?.onLLMStream
+      ? this.callbackManager.onLLMStream
+      : () => {};
+
+    const chunk_stream: AsyncIterable<OpenAIStreamToken> =
+      await this.session.openai.chat.completions.create({
         ...baseRequestParams,
         stream: true,
       });
 
-      const { message, role } = await handleOpenAIStream({
-        response,
-        onLLMStream: this.callbackManager.onLLMStream,
-        parentEvent,
-      });
-      return { message: { content: message, role } };
-    } else {
-      // Non-streaming
-      const response = await this.session.openai.chat.completions.create({
-        ...baseRequestParams,
-        stream: false,
-      });
+    const event: Event = parentEvent
+      ? parentEvent
+      : {
+          id: "unspecified",
+          type: "llmPredict" as EventType,
+        };
 
-      const content = response.choices[0].message?.content ?? "";
-      return { message: { content, role: response.choices[0].message.role } };
+    //Indices
+    var idx_counter: number = 0;
+    for await (const part of chunk_stream) {
+      //Increment
+      part.choices[0].index = idx_counter;
+      const is_done: boolean =
+        part.choices[0].finish_reason === "stop" ? true : false;
+      //onLLMStream Callback
+
+      const stream_callback: StreamCallbackResponse = {
+        event: event,
+        index: idx_counter,
+        isDone: is_done,
+        token: part,
+      };
+      onLLMStream(stream_callback);
+
+      idx_counter++;
+
+      yield part.choices[0].delta.content ? part.choices[0].delta.content : "";
     }
+    return;
   }
 
-  async complete(
-    prompt: string,
+  //streamComplete doesn't need to be async because it's child function is already async
+  protected streamComplete(
+    query: string,
     parentEvent?: Event,
-  ): Promise<CompletionResponse> {
-    return this.chat([{ content: prompt, role: "user" }], parentEvent);
+  ): AsyncGenerator<string, void, unknown> {
+    return this.streamChat([{ content: query, role: "user" }], parentEvent);
   }
 }
 
@@ -279,6 +427,7 @@ export class LlamaDeuce implements LLM {
   topP: number;
   maxTokens?: number;
   replicateSession: ReplicateSession;
+  hasStreaming: boolean;
 
   constructor(init?: Partial<LlamaDeuce>) {
     this.model = init?.model ?? "Llama-2-70b-chat-4bit";
@@ -293,6 +442,22 @@ export class LlamaDeuce implements LLM {
       init?.maxTokens ??
       ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model].contextWindow; // For Replicate, the default is 500 tokens which is too low.
     this.replicateSession = init?.replicateSession ?? new ReplicateSession();
+    this.hasStreaming = init?.hasStreaming ?? false;
+  }
+
+  tokens(messages: ChatMessage[]): number {
+    throw new Error("Method not implemented.");
+  }
+
+  get metadata() {
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow: ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model].contextWindow,
+      tokenizer: undefined,
+    };
   }
 
   mapMessagesToPrompt(messages: ChatMessage[]) {
@@ -399,10 +564,10 @@ If a question does not make any sense, or is not factually coherent, explain why
     };
   }
 
-  async chat(
-    messages: ChatMessage[],
-    _parentEvent?: Event,
-  ): Promise<ChatResponse> {
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(messages: ChatMessage[], _parentEvent?: Event, streaming?: T): Promise<R> {
     const api = ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model]
       .replicateApi as `${string}/${string}:${string}`;
 
@@ -423,6 +588,9 @@ If a question does not make any sense, or is not factually coherent, explain why
       replicateOptions.input.max_length = this.maxTokens;
     }
 
+    //TODO: Add streaming for this
+
+    //Non-streaming
     const response = await this.replicateSession.replicate.run(
       api,
       replicateOptions,
@@ -433,24 +601,32 @@ If a question does not make any sense, or is not factually coherent, explain why
         //^ We need to do this because Replicate returns a list of strings (for streaming functionality which is not exposed by the run function)
         role: "assistant",
       },
-    };
+    } as R;
   }
 
-  async complete(
-    prompt: string,
-    parentEvent?: Event,
-  ): Promise<CompletionResponse> {
+  async complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
     return this.chat([{ content: prompt, role: "user" }], parentEvent);
   }
 }
 
+export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
+  // both models have 100k context window, see https://docs.anthropic.com/claude/reference/selecting-a-model
+  "claude-2": { contextWindow: 100000 },
+  "claude-instant-1": { contextWindow: 100000 },
+};
+
 /**
  * Anthropic LLM implementation
  */
 
 export class Anthropic implements LLM {
+  hasStreaming: boolean = true;
+
   // Per completion Anthropic params
-  model: string;
+  model: keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS;
   temperature: number;
   topP: number;
   maxTokens?: number;
@@ -483,6 +659,21 @@ export class Anthropic implements LLM {
     this.callbackManager = init?.callbackManager;
   }
 
+  tokens(messages: ChatMessage[]): number {
+    throw new Error("Method not implemented.");
+  }
+
+  get metadata() {
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow: ALL_AVAILABLE_ANTHROPIC_MODELS[this.model].contextWindow,
+      tokenizer: undefined,
+    };
+  }
+
   mapMessagesToPrompt(messages: ChatMessage[]) {
     return (
       messages.reduce((acc, message) => {
@@ -498,10 +689,22 @@ export class Anthropic implements LLM {
     );
   }
 
-  async chat(
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
     messages: ChatMessage[],
     parentEvent?: Event | undefined,
-  ): Promise<ChatResponse> {
+    streaming?: T,
+  ): Promise<R> {
+    //Streaming
+    if (streaming) {
+      if (!this.hasStreaming) {
+        throw Error("No streaming support for this LLM.");
+      }
+      return this.streamChat(messages, parentEvent) as R;
+    }
+    //Non-streaming
     const response = await this.session.anthropic.completions.create({
       model: this.model,
       prompt: this.mapMessagesToPrompt(messages),
@@ -514,12 +717,182 @@ export class Anthropic implements LLM {
       message: { content: response.completion.trimStart(), role: "assistant" },
       //^ We're trimming the start because Anthropic often starts with a space in the response
       // That space will be re-added when we generate the next prompt.
-    };
+    } as R;
   }
-  async complete(
+
+  protected async *streamChat(
+    messages: ChatMessage[],
+    parentEvent?: Event | undefined,
+  ): AsyncGenerator<string, void, unknown> {
+    // AsyncIterable<AnthropicStreamToken>
+    const stream: AsyncIterable<AnthropicStreamToken> =
+      await this.session.anthropic.completions.create({
+        model: this.model,
+        prompt: this.mapMessagesToPrompt(messages),
+        max_tokens_to_sample: this.maxTokens ?? 100000,
+        temperature: this.temperature,
+        top_p: this.topP,
+        stream: true,
+      });
+
+    var idx_counter: number = 0;
+    for await (const part of stream) {
+      //TODO: LLM Stream Callback, pending re-work.
+
+      idx_counter++;
+      yield part.completion;
+    }
+    return;
+  }
+
+  async complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
     prompt: string,
     parentEvent?: Event | undefined,
-  ): Promise<CompletionResponse> {
-    return this.chat([{ content: prompt, role: "user" }], parentEvent);
+    streaming?: T,
+  ): Promise<R> {
+    if (streaming) {
+      return this.streamComplete(prompt, parentEvent) as R;
+    }
+    return this.chat(
+      [{ content: prompt, role: "user" }],
+      parentEvent,
+      streaming,
+    ) as R;
+  }
+
+  protected streamComplete(
+    prompt: string,
+    parentEvent?: Event | undefined,
+  ): AsyncGenerator<string, void, unknown> {
+    return this.streamChat([{ content: prompt, role: "user" }], parentEvent);
+  }
+}
+
+export class Portkey implements LLM {
+  hasStreaming: boolean = true;
+
+  apiKey?: string = undefined;
+  baseURL?: string = undefined;
+  mode?: string = undefined;
+  llms?: [LLMOptions] | null = undefined;
+  session: PortkeySession;
+  callbackManager?: CallbackManager;
+
+  constructor(init?: Partial<Portkey>) {
+    this.apiKey = init?.apiKey;
+    this.baseURL = init?.baseURL;
+    this.mode = init?.mode;
+    this.llms = init?.llms;
+    this.session = getPortkeySession({
+      apiKey: this.apiKey,
+      baseURL: this.baseURL,
+      llms: this.llms,
+      mode: this.mode,
+    });
+    this.callbackManager = init?.callbackManager;
+  }
+
+  tokens(messages: ChatMessage[]): number {
+    throw new Error("Method not implemented.");
+  }
+
+  get metadata(): LLMMetadata {
+    throw new Error("metadata not implemented for Portkey");
+  }
+
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
+    messages: ChatMessage[],
+    parentEvent?: Event | undefined,
+    streaming?: T,
+    params?: Record<string, any>,
+  ): Promise<R> {
+    if (streaming) {
+      return this.streamChat(messages, parentEvent, params) as R;
+    } else {
+      const resolvedParams = params || {};
+      const response = await this.session.portkey.chatCompletions.create({
+        messages,
+        ...resolvedParams,
+      });
+
+      const content = response.choices[0].message?.content ?? "";
+      const role = response.choices[0].message?.role || "assistant";
+      return { message: { content, role: role as MessageType } } as R;
+    }
+  }
+
+  async complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
+    prompt: string,
+    parentEvent?: Event | undefined,
+    streaming?: T,
+  ): Promise<R> {
+    return this.chat(
+      [{ content: prompt, role: "user" }],
+      parentEvent,
+      streaming,
+    );
+  }
+
+  async *streamChat(
+    messages: ChatMessage[],
+    parentEvent?: Event,
+    params?: Record<string, any>,
+  ): AsyncGenerator<string, void, unknown> {
+    // Wrapping the stream in a callback.
+    const onLLMStream = this.callbackManager?.onLLMStream
+      ? this.callbackManager.onLLMStream
+      : () => {};
+
+    const chunkStream = await this.session.portkey.chatCompletions.create({
+      messages,
+      ...params,
+      stream: true,
+    });
+
+    const event: Event = parentEvent
+      ? parentEvent
+      : {
+          id: "unspecified",
+          type: "llmPredict" as EventType,
+        };
+
+    //Indices
+    var idx_counter: number = 0;
+    for await (const part of chunkStream) {
+      //Increment
+      part.choices[0].index = idx_counter;
+      const is_done: boolean =
+        part.choices[0].finish_reason === "stop" ? true : false;
+      //onLLMStream Callback
+
+      const stream_callback: StreamCallbackResponse = {
+        event: event,
+        index: idx_counter,
+        isDone: is_done,
+        // token: part,
+      };
+      onLLMStream(stream_callback);
+
+      idx_counter++;
+
+      yield part.choices[0].delta?.content ?? "";
+    }
+    return;
+  }
+
+  streamComplete(
+    query: string,
+    parentEvent?: Event,
+  ): AsyncGenerator<string, void, unknown> {
+    return this.streamChat([{ content: query, role: "user" }], parentEvent);
   }
 }

packages/core/src/llm/openai.ts

@@ -24,7 +24,10 @@ export class OpenAISession {
     if (options.azure) {
       this.openai = new AzureOpenAI(options);
     } else {
-      this.openai = new OpenAI(options);
+      this.openai = new OpenAI({
+        ...options,
+        // defaultHeaders: { "OpenAI-Beta": "assistants=v1" },
+      });
     }
   }
 }

packages/core/src/llm/portkey.ts

@@ -0,0 +1,62 @@
+import _ from "lodash";
+import { LLMOptions, Portkey } from "portkey-ai";
+
+export const readEnv = (env: string, default_val?: string): string | undefined => {
+  if (typeof process !== 'undefined') {
+      return process.env?.[env] ?? default_val;
+  }
+  return default_val;
+};
+
+interface PortkeyOptions {
+  apiKey?: string;
+  baseURL?: string;
+  mode?: string;
+  llms?: [LLMOptions] | null
+}
+
+export class PortkeySession {
+  portkey: Portkey;
+
+  constructor(options:PortkeyOptions = {}) {
+    if (!options.apiKey) {
+      options.apiKey = readEnv('PORTKEY_API_KEY')
+    }
+
+    if (!options.baseURL) {
+      options.baseURL = readEnv('PORTKEY_BASE_URL', "https://api.portkey.ai")
+    }
+
+    this.portkey = new Portkey({});
+    this.portkey.llms = [{}]
+    if (!options.apiKey) {
+      throw new Error("Set Portkey ApiKey in PORTKEY_API_KEY env variable");
+    }
+
+    this.portkey = new Portkey(options);
+  }
+}
+
+let defaultPortkeySession: {
+  session: PortkeySession;
+  options: PortkeyOptions;
+}[] = [];
+
+/**
+ * Get a session for the Portkey API. If one already exists with the same options,
+ * it will be returned. Otherwise, a new session will be created.
+ * @param options
+ * @returns
+ */
+export function getPortkeySession(options: PortkeyOptions = {}) {
+  let session = defaultPortkeySession.find((session) => {
+    return _.isEqual(session.options, options);
+  })?.session;
+
+  if (!session) {
+    session = new PortkeySession(options);
+    defaultPortkeySession.push({ session, options });
+  }
+  return session;
+}
+

packages/core/src/readers/DocxReader.ts

@@ -0,0 +1,17 @@
+import mammoth from "mammoth";
+import { Document } from "../Node";
+import { DEFAULT_FS } from "../storage/constants";
+import { GenericFileSystem } from "../storage/FileSystem";
+import { BaseReader } from "./base";
+
+export class DocxReader implements BaseReader {
+  /** DocxParser */
+  async loadData(
+    file: string,
+    fs: GenericFileSystem = DEFAULT_FS,
+  ): Promise<Document[]> {
+    const dataBuffer = (await fs.readFile(file)) as any;
+    const { value } = await mammoth.extractRawText({ buffer: dataBuffer });
+    return [new Document({ text: value, id_: file })];
+  }
+}

packages/core/src/readers/HTMLReader.ts

@@ -0,0 +1,77 @@
+import { Document } from "../Node";
+import { DEFAULT_FS } from "../storage/constants";
+import { GenericFileSystem } from "../storage/FileSystem";
+import { BaseReader } from "./base";
+
+/**
+ * Extract the significant text from an arbitrary HTML document.
+ * The contents of any head, script, style, and xml tags are removed completely.
+ * The URLs for a[href] tags are extracted, along with the inner text of the tag.
+ * All other tags are removed, and the inner text is kept intact.
+ * Html entities (e.g., &) are not decoded.
+ */
+export class HTMLReader implements BaseReader {
+  /**
+   * Public method for this reader.
+   * Required by BaseReader interface.
+   * @param file Path/name of the file to be loaded.
+   * @param fs fs wrapper interface for getting the file content.
+   * @returns Promise<Document[]> A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
+   */
+  async loadData(
+    file: string,
+    fs: GenericFileSystem = DEFAULT_FS,
+  ): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file, "utf-8");
+    const htmlOptions = this.getOptions();
+    const content = await this.parseContent(dataBuffer, htmlOptions);
+    return [new Document({ text: content, id_: file })];
+  }
+
+  /**
+   * Wrapper for string-strip-html usage.
+   * @param html Raw HTML content to be parsed.
+   * @param options An object of options for the underlying library
+   * @see getOptions
+   * @returns The HTML content, stripped of unwanted tags and attributes
+   */
+  async parseContent(html: string, options: any = {}): Promise<string> {
+    const { stripHtml } = await import("string-strip-html"); // ESM only
+    return stripHtml(html).result;
+  }
+
+  /**
+   * Wrapper for our configuration options passed to string-strip-html library
+   * @see https://codsen.com/os/string-strip-html/examples
+   * @returns An object of options for the underlying library
+   */
+  getOptions() {
+    return {
+      skipHtmlDecoding: true,
+      stripTogetherWithTheirContents: [
+        "script", // default
+        "style", // default
+        "xml", // default
+        "head", // <-- custom-added
+      ],
+      // Keep the URLs for embedded links
+      // cb: (tag: any, deleteFrom: number, deleteTo: number, insert: string, rangesArr: any, proposedReturn: string) => {
+      //   let temp;
+      //   if (
+      //     tag.name === "a" &&
+      //     tag.attributes &&
+      //     tag.attributes.some((attr: any) => {
+      //       if (attr.name === "href") {
+      //         temp = attr.value;
+      //         return true;
+      //       }
+      //     })
+      //   ) {
+      //     rangesArr.push([deleteFrom, deleteTo, `${temp} ${insert || ""}`]);
+      //   } else {
+      //     rangesArr.push(proposedReturn);
+      //   }
+      // },
+    };
+  }
+}

packages/core/src/readers/NotionReader.ts

@@ -1,271 +1,67 @@
-import { Client, collectPaginatedAPI } from "@notionhq/client";
-import * as md from "md-utils-ts";
+import { Client } from "@notionhq/client";
+import { crawler, Crawler, Pages, pageToString } from "notion-md-crawler";
 import { Document } from "../Node";
 import { BaseReader } from "./base";
 
-type NotionClient = InstanceType<typeof Client>;
+type OptionalSerializers = Parameters<Crawler>[number]["serializers"];
 
-// Notion Page
-type NotionPageRetrieveMethod = NotionClient["pages"]["retrieve"];
-type NotionPartialPageObjectResponse = Awaited<
-  ReturnType<NotionPageRetrieveMethod>
->;
-
-// Notion Block
-type NotionBlockListMethod = NotionClient["blocks"]["children"]["list"];
-type NotionBlockListResponse = Awaited<ReturnType<NotionBlockListMethod>>;
-type NotionBlockObjectResponse = NotionBlockListResponse["results"][number];
-type ExtractBlockObjectResponse<T> = T extends { type: string } ? T : never;
-type NotionBlock = ExtractBlockObjectResponse<NotionBlockObjectResponse>;
-type NotionChildPageBlock = Extract<NotionBlock, { type: "child_page" }>;
-type NotionParagraphBlock = Extract<NotionBlock, { type: "paragraph" }>;
-type NotionTableRowBlock = Extract<NotionBlock, { type: "table_row" }>;
-type NotionRichText = NotionParagraphBlock["paragraph"]["rich_text"];
-type NotionAnnotations = NotionRichText[number]["annotations"];
-
-const fetchNotionBlocks = (client: Client) => async (blockId: string) =>
-  collectPaginatedAPI(client.blocks.children.list, {
-    block_id: blockId,
-  });
-
-const fetchNotionPage = (client: Client) => (pageId: string) =>
-  client.pages.retrieve({ page_id: pageId });
-
-type Page = {
-  metadata: {
-    id: string;
-    title: string;
-    createdTime: string;
-    lastEditedTime: string;
-    parentId?: string;
-  };
-  lines: string[];
+/**
+ * Options for initializing the NotionReader class
+ * @typedef {Object} NotionReaderOptions
+ * @property {Client} client - The Notion Client object for API interactions
+ * @property {OptionalSerializers} [serializers] - Option to customize serialization. See [the url](https://github.com/TomPenguin/notion-md-crawler/tree/main) for details.
+ */
+type NotionReaderOptions = {
+  client: Client;
+  serializers?: OptionalSerializers;
 };
 
-type Pages = Record<string, Page>;
-
-const hasType = (block: NotionBlockObjectResponse): block is NotionBlock =>
-  "type" in block;
-
-const blockIs = <T extends NotionBlock["type"]>(
-  block: NotionBlock,
-  type: T,
-): block is Extract<NotionBlock, { type: T }> => block.type === type;
-
-const getCursor = (
-  pageBlock: NotionChildPageBlock,
-  parentId?: string,
-): Page => ({
-  metadata: {
-    id: pageBlock.id,
-    title: pageBlock.child_page.title,
-    createdTime: pageBlock.created_time,
-    lastEditedTime: pageBlock.last_edited_time,
-    parentId,
-  },
-  lines: [],
-});
-
-const annotateText = (text: string, annotations: NotionAnnotations) => {
-  if (annotations.code) text = md.inlineCode(text);
-  if (annotations.bold) text = md.bold(text);
-  if (annotations.italic) text = md.italic(text);
-  if (annotations.strikethrough) text = md.del(text);
-  if (annotations.underline) text = md.underline(text);
-
-  return text;
-};
-
-const richTextToString = (richText: NotionRichText) =>
-  richText
-    .map(({ plain_text, annotations, href }) => {
-      if (plain_text.match(/^\s*$/)) return plain_text;
-
-      const leadingSpaceMatch = plain_text.match(/^(\s*)/);
-      const trailingSpaceMatch = plain_text.match(/(\s*)$/);
-
-      const leading_space = leadingSpaceMatch ? leadingSpaceMatch[0] : "";
-      const trailing_space = trailingSpaceMatch ? trailingSpaceMatch[0] : "";
-
-      const text = plain_text.trim();
-
-      if (text === "") return leading_space + trailing_space;
-
-      const annotatedText = annotateText(text, annotations);
-      const linkedText = href ? md.anchor(annotatedText, href) : annotatedText;
-
-      return leading_space + linkedText + trailing_space;
-    })
-    .join("");
-
-const tableRowToString = (block: NotionTableRowBlock) =>
-  `| ${block.table_row.cells
-    .flatMap((row) => row.map((column) => richTextToString([column])))
-    .join(" | ")} |`;
-
-const blockToString = (block: NotionBlock): string => {
-  switch (block.type) {
-    case "divider":
-      return md.hr();
-    case "equation":
-      return md.equationBlock(block.equation.expression);
-    case "bookmark":
-      return md.anchor(
-        richTextToString(block.bookmark.caption),
-        block.bookmark.url,
-      );
-    case "link_preview":
-      return md.anchor(block.type, block.link_preview.url);
-    case "link_to_page":
-      const href =
-        block.link_to_page.type === "page_id" ? block.link_to_page.page_id : "";
-      return md.anchor(block.type, href);
-    case "child_page":
-      return `[${block.child_page.title}]`;
-    case "child_database":
-      return `[${block.child_database.title}]`;
-    case "paragraph":
-      return richTextToString(block.paragraph.rich_text);
-    case "heading_1":
-      return md.h1(richTextToString(block.heading_1.rich_text));
-    case "heading_2":
-      return md.h2(richTextToString(block.heading_2.rich_text));
-    case "heading_3":
-      return md.h3(richTextToString(block.heading_3.rich_text));
-    case "bulleted_list_item":
-      return md.bullet(richTextToString(block.bulleted_list_item.rich_text));
-    case "numbered_list_item":
-      return md.bullet(richTextToString(block.numbered_list_item.rich_text), 1);
-    case "quote":
-      return md.quote(richTextToString(block.quote.rich_text));
-    case "table_row":
-      return tableRowToString(block);
-    case "to_do":
-      return md.todo(
-        richTextToString(block.to_do.rich_text),
-        block.to_do.checked,
-      );
-    case "template":
-      return richTextToString(block.template.rich_text);
-    case "code":
-      return md.codeBlock(block.code.language)(
-        richTextToString(block.code.rich_text),
-      );
-    case "callout":
-      return md.quote(richTextToString(block.callout.rich_text));
-
-    case "image":
-    case "video":
-    case "audio":
-    case "file":
-    case "pdf":
-    case "table":
-    case "embed":
-    case "breadcrumb":
-    case "synced_block":
-    case "table_of_contents":
-    case "unsupported":
-    default:
-      return "";
-  }
-};
-
-const getNest = (block: NotionBlock, baseNest: number) => {
-  switch (block.type) {
-    // Reset nest
-    case "child_page":
-      return 0;
-
-    // Eliminates unnecessary nests due to NotionBlock structure
-    case "table_row":
-    case "column_list":
-    case "column":
-    case "synced_block":
-      return baseNest;
-
-    default:
-      return baseNest + 1;
-  }
-};
-
-const crawlPages =
-  (client: Client) =>
-  async (
-    blocks: NotionBlockObjectResponse[],
-    cursor: Page,
-    pages: Pages = {},
-    nest = 0,
-  ): Promise<Pages> => {
-    pages[cursor.metadata.id] = pages[cursor.metadata.id] || cursor;
-
-    for (const block of blocks) {
-      if (!hasType(block)) continue;
-
-      const line = md.indent()(blockToString(block), nest);
-      cursor.lines.push(line);
-
-      if (block.has_children) {
-        const blockId = blockIs(block, "synced_block")
-          ? block.synced_block.synced_from?.block_id || block.id
-          : block.id;
-        const childBlocks = await fetchNotionBlocks(client)(blockId);
-        const nextCursor = blockIs(block, "child_page")
-          ? getCursor(block, cursor.metadata.id)
-          : cursor;
-        const childPages = await crawlPages(client)(
-          childBlocks,
-          nextCursor,
-          pages,
-          getNest(block, nest),
-        );
-        pages = { ...pages, ...childPages };
-      }
-    }
-
-    return pages;
-  };
-
-const extractPageTitle = (page: NotionPartialPageObjectResponse) => {
-  if (!("properties" in page)) return "";
-
-  if (page.properties.title.type !== "title") return "";
-
-  return page.properties.title.title[0].plain_text;
-};
-
-const nestHeading = (text: string) => (text.match(/^#+\s/) ? "#" + text : text);
-
-const pagesToDocuments = (pages: Pages): Document[] =>
-  Object.entries(pages).map(([, { lines, metadata }]) => {
-    const title = md.h1(metadata.title);
-    const body = lines.map(nestHeading);
-    const text = [title, ...body].join("\n");
-    return new Document({ text, metadata });
-  });
-
+/**
+ * Notion pages are retrieved recursively and converted to Document objects.
+ * Notion Database can also be loaded, and [the serialization method can be customized](https://github.com/TomPenguin/notion-md-crawler/tree/main).
+ *
+ * [Note] To use this reader, must be created the Notion integration must be created in advance
+ * Please refer to [this document](https://www.notion.so/help/create-integrations-with-the-notion-api) for details.
+ */
 export class NotionReader implements BaseReader {
-  private client: Client;
+  private crawl: ReturnType<Crawler>;
 
-  constructor(options: { client: Client }) {
-    this.client = options.client;
+  /**
+   * Constructor for the NotionReader class
+   * @param {NotionReaderOptions} options - Configuration options for the reader
+   */
+  constructor({ client, serializers }: NotionReaderOptions) {
+    this.crawl = crawler({ client, serializers });
   }
 
-  async loadData(pageId: string): Promise<Document[]> {
-    const rootPage = (await fetchNotionPage(this.client)(pageId)) as any;
-    const rootPageTitle = extractPageTitle(rootPage);
-    const rootBlocks = await fetchNotionBlocks(this.client)(rootPage.id);
+  /**
+   * Converts Pages to an array of Document objects
+   * @param {Pages} pages - The Notion pages to convert (Return value of `loadPages`)
+   * @returns {Document[]} An array of Document objects
+   */
+  toDocuments(pages: Pages): Document[] {
+    return Object.values(pages).map((page) => {
+      const text = pageToString(page);
+      return new Document({ text, metadata: page.metadata });
+    });
+  }
 
-    const cursor: Page = {
-      metadata: {
-        id: rootPage.id,
-        title: rootPageTitle,
-        createdTime: rootPage.created_time,
-        lastEditedTime: rootPage.last_edited_time,
-      },
-      lines: [],
-    };
-    const pages = await crawlPages(this.client)(rootBlocks, cursor);
+  /**
+   * Loads recursively the Notion page with the specified root page ID.
+   * @param {string} rootPageId - The root Notion page ID
+   * @returns {Promise<Pages>} A Promise that resolves to a Pages object(Convertible with the `toDocuments` method)
+   */
+  async loadPages(rootPageId: string): Promise<Pages> {
+    return this.crawl(rootPageId);
+  }
 
-    return pagesToDocuments(pages);
+  /**
+   * Loads recursively Notion pages and converts them to an array of Document objects
+   * @param {string} rootPageId - The root Notion page ID
+   * @returns {Promise<Document[]>} A Promise that resolves to an array of Document objects
+   */
+  async loadData(rootPageId: string): Promise<Document[]> {
+    const pages = await this.loadPages(rootPageId);
+    return this.toDocuments(pages);
   }
 }

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -3,10 +3,24 @@ import { Document } from "../Node";
 import { CompleteFileSystem, walk } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
 import { PapaCSVReader } from "./CSVReader";
+import { DocxReader } from "./DocxReader";
+import { HTMLReader } from "./HTMLReader";
 import { MarkdownReader } from "./MarkdownReader";
 import { PDFReader } from "./PDFReader";
 import { BaseReader } from "./base";
 
+type ReaderCallback = (
+  category: "file" | "directory",
+  name: string,
+  status: ReaderStatus,
+  message?: string,
+) => boolean;
+enum ReaderStatus {
+  STARTED = 0,
+  COMPLETE,
+  ERROR,
+}
+
 /**
  * Read a .txt file
  */
@@ -20,11 +34,14 @@ export class TextFileReader implements BaseReader {
   }
 }
 
-const FILE_EXT_TO_READER: Record<string, BaseReader> = {
+export const FILE_EXT_TO_READER: Record<string, BaseReader> = {
   txt: new TextFileReader(),
   pdf: new PDFReader(),
   csv: new PapaCSVReader(),
   md: new MarkdownReader(),
+  docx: new DocxReader(),
+  htm: new HTMLReader(),
+  html: new HTMLReader(),
 };
 
 export type SimpleDirectoryReaderLoadDataProps = {
@@ -35,20 +52,37 @@ export type SimpleDirectoryReaderLoadDataProps = {
 };
 
 /**
- * Read all of the documents in a directory. Currently supports PDF and TXT files.
+ * Read all of the documents in a directory.
+ * By default, supports the list of file types
+ * in the FILE_EXIT_TO_READER map.
  */
 export class SimpleDirectoryReader implements BaseReader {
+  constructor(private observer?: ReaderCallback) {}
+
   async loadData({
     directoryPath,
     fs = DEFAULT_FS as CompleteFileSystem,
     defaultReader = new TextFileReader(),
     fileExtToReader = FILE_EXT_TO_READER,
   }: SimpleDirectoryReaderLoadDataProps): Promise<Document[]> {
+    // Observer can decide to skip the directory
+    if (
+      !this.doObserverCheck("directory", directoryPath, ReaderStatus.STARTED)
+    ) {
+      return [];
+    }
+
     let docs: Document[] = [];
     for await (const filePath of walk(fs, directoryPath)) {
       try {
         const fileExt = _.last(filePath.split(".")) || "";
 
+        // Observer can decide to skip each file
+        if (!this.doObserverCheck("file", filePath, ReaderStatus.STARTED)) {
+          // Skip this file
+          continue;
+        }
+
         let reader = null;
 
         if (fileExt in fileExtToReader) {
@@ -56,16 +90,52 @@ export class SimpleDirectoryReader implements BaseReader {
         } else if (!_.isNil(defaultReader)) {
           reader = defaultReader;
         } else {
-          console.warn(`No reader for file extension of ${filePath}`);
+          const msg = `No reader for file extension of ${filePath}`;
+          console.warn(msg);
+
+          // In an error condition, observer's false cancels the whole process.
+          if (
+            !this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)
+          ) {
+            return [];
+          }
+
           continue;
         }
 
         const fileDocs = await reader.loadData(filePath, fs);
-        docs.push(...fileDocs);
+
+        // Observer can still cancel addition of the resulting docs from this file
+        if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) {
+          docs.push(...fileDocs);
+        }
       } catch (e) {
-        console.error(`Error reading file ${filePath}: ${e}`);
+        const msg = `Error reading file ${filePath}: ${e}`;
+        console.error(msg);
+
+        // In an error condition, observer's false cancels the whole process.
+        if (!this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)) {
+          return [];
+        }
       }
     }
+
+    // After successful import of all files, directory completion
+    // is only a notification for observer, cannot be cancelled.
+    this.doObserverCheck("directory", directoryPath, ReaderStatus.COMPLETE);
+
     return docs;
   }
+
+  private doObserverCheck(
+    category: "file" | "directory",
+    name: string,
+    status: ReaderStatus,
+    message?: string,
+  ): boolean {
+    if (this.observer) {
+      return this.observer(category, name, status, message);
+    }
+    return true;
+  }
 }

packages/core/src/readers/SimpleMongoReader.ts

@@ -0,0 +1,51 @@
+import { MongoClient } from "mongodb";
+import { Document } from "../Node";
+import { BaseReader } from "./base";
+
+/**
+ * Read in from MongoDB
+ */
+export class SimpleMongoReader implements BaseReader {
+  private client: MongoClient;
+
+  constructor(client: MongoClient) {
+    this.client = client;
+  }
+
+  /**
+   * Loads data from MongoDB collection
+   * @param {string} db_name - The name of the database to load.
+   * @param {string} collection_name - The name of the collection to load.
+   * @param {Number} [max_docs = 0] - Maximum number of documents to return. 0 means no limit.
+   * @param {Record<string, any>} [query_dict={}] - Specific query, as specified by MongoDB NodeJS documentation.
+   * @param {Record<string, any>} [query_options={}] - Specific query options, as specified by MongoDB NodeJS documentation.
+   * @param {Record<string, any>} [projection = {}] - Projection options, as specified by MongoDB NodeJS documentation.
+   * @returns {Promise<Document[]>}
+   */
+  async loadData(
+    db_name: string,
+    collection_name: string,
+    max_docs = 0,
+    //For later: Think about whether we want to pass generic objects in...
+    query_dict: Record<string, any> = {},
+    query_options: Record<string, any> = {},
+    projection: Record<string, any> = {},
+  ): Promise<Document[]> {
+    //Get items from collection using built-in functions
+    const cursor: Partial<Document>[] = await this.client
+      .db(db_name)
+      .collection(collection_name)
+      .find(query_dict, query_options)
+      .limit(max_docs)
+      .project(projection)
+      .toArray();
+
+    //Aggregate results and return
+    const documents: Document[] = [];
+    cursor.forEach((element: Partial<Document>) => {
+      //For later: Metadata filtering
+      documents.push(new Document({ text: JSON.stringify(element) }));
+    });
+    return documents;
+  }
+}

packages/core/src/storage/vectorStore/types.ts

@@ -63,6 +63,9 @@ export interface VectorStore {
   client(): any;
   add(embeddingResults: BaseNode[]): Promise<string[]>;
   delete(refDocId: string, deleteKwargs?: any): Promise<void>;
-  query(query: VectorStoreQuery, kwargs?: any): Promise<VectorStoreQueryResult>;
+  query(
+    query: VectorStoreQuery,
+    options?: any,
+  ): Promise<VectorStoreQueryResult>;
   persist(persistPath: string, fs?: GenericFileSystem): Promise<void>;
 }

packages/core/src/tests/Keyword.test.ts

@@ -0,0 +1,84 @@
+import {
+  rakeExtractKeywords,
+  simpleExtractKeywords,
+} from "../indices/keyword/utils";
+describe("SimpleExtractKeywords", () => {
+  test("should extract unique keywords", () => {
+    const text = "apple banana apple cherry";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set(["apple", "banana", "cherry"]));
+  });
+
+  test("should handle empty string", () => {
+    const text = "";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set());
+  });
+
+  test("should handle case sensitivity", () => {
+    const text = "Apple apple";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set(["apple"]));
+  });
+
+  test("should order keywords by frequency", () => {
+    const text = "apple banana apple cherry banana apple";
+    const result = simpleExtractKeywords(text);
+    expect([...result]).toEqual(["apple", "banana", "cherry"]);
+  });
+
+  test("should respect the maxKeywords parameter", () => {
+    const text = "apple banana apple cherry banana apple orange";
+    const result = simpleExtractKeywords(text, 2);
+    expect(result).toEqual(new Set(["apple", "banana"]));
+  });
+
+  test("should handle non-alphabetic characters", () => {
+    const text = "apple! banana... apple? cherry, orange;";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set(["apple", "banana", "cherry", "orange"]));
+  });
+});
+
+describe("RakeExtractKeywords", () => {
+  const sampleText = `Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.`;
+  test("should return all keywords if maxKeywords is not provided", () => {
+    const result = rakeExtractKeywords(sampleText);
+    expect(result).toEqual(
+      new Set([
+        "strong feelings",
+        "beginning writers",
+        "short stories",
+        "write essays",
+        "stories",
+        "write",
+        "deep",
+        "imagined",
+        "characters",
+        "plot",
+        "hardly",
+        "awful",
+        "probably",
+        "supposed",
+        "wrote",
+        "didn",
+        "programming",
+        "writing",
+        "school",
+        "outside",
+        "main",
+        "college",
+      ]),
+    );
+  });
+
+  test("should respect the maxKeywords parameter", () => {
+    const result = rakeExtractKeywords(sampleText, 2);
+    expect(result).toEqual(new Set(["strong feelings", "beginning writers"]));
+  });
+
+  test("should handle empty return from rake", () => {
+    const result = rakeExtractKeywords("");
+    expect(result).toEqual(new Set());
+  });
+});

packages/core/src/tests/OutputParser.test.ts

@@ -0,0 +1,90 @@
+import { SubQuestionOutputParser } from "../OutputParser";
+
+//This parser is really important, so make sure to add tests
+// as the parser sees through more iterations.
+describe("SubQuestionOutputParser", () => {
+  test("parses expected", () => {
+    const parser = new SubQuestionOutputParser();
+
+    const data = [
+      {
+        name: "uber_10k",
+        description: "Provides information about Uber financials for year 2021",
+      },
+      {
+        name: "lyft_10k",
+        description: "Provides information about Lyft financials for year 2021",
+      },
+    ];
+
+    const data_str: string = JSON.stringify(data);
+    const full_string = `\`\`\`json
+${data_str}
+\`\`\``;
+
+    const real_answer = { parsedOutput: data, rawOutput: full_string };
+
+    expect(parser.parse(full_string)).toEqual(real_answer);
+  });
+
+  //This is in case our LLM outputs a list response, but without ```json.
+  test("parses without ```json", () => {
+    const parser = new SubQuestionOutputParser();
+
+    const data = [
+      {
+        name: "uber_10k",
+        description: "Provides information about Uber financials for year 2021",
+      },
+      {
+        name: "lyft_10k",
+        description: "Provides information about Lyft financials for year 2021",
+      },
+    ];
+
+    const data_str: string = JSON.stringify(data);
+    const full_string = `${data_str}`;
+
+    const real_answer = { parsedOutput: data, rawOutput: full_string };
+
+    expect(parser.parse(JSON.stringify(data))).toEqual(real_answer);
+  });
+
+  test("parses null single response", () => {
+    const parser = new SubQuestionOutputParser();
+    const data_str =
+      "[\n" +
+      "    {\n" +
+      `        "subQuestion": "Sorry, I don't have any relevant information to answer your question",\n` +
+      '        "toolName": ""\n' +
+      "    }\n" +
+      "]";
+    const data = [
+      {
+        subQuestion:
+          "Sorry, I don't have any relevant information to answer your question",
+        toolName: "",
+      },
+    ];
+    const real_answer = { parsedOutput: data, rawOutput: data_str };
+    expect(parser.parse(data_str)).toEqual(real_answer);
+  });
+
+  test("Single JSON object case", () => {
+    const parser = new SubQuestionOutputParser();
+    const data_str =
+      "    {\n" +
+      `        "subQuestion": "Sorry, I don't have any relevant information to answer your question",\n` +
+      '        "toolName": ""\n' +
+      "    }\n";
+    const data = [
+      {
+        subQuestion:
+          "Sorry, I don't have any relevant information to answer your question",
+        toolName: "",
+      },
+    ];
+    const real_answer = { parsedOutput: data, rawOutput: data_str };
+    expect(parser.parse(data_str)).toEqual(real_answer);
+  });
+});

packages/core/src/tests/TextSplitter.test.ts

@@ -73,6 +73,7 @@ describe("SentenceSplitter", () => {
     let splits = sentenceSplitter.splitText(
       "This is a sentence. This is another sentence. 1.0",
     );
+
     expect(splits).toEqual([
       "This is a sentence.",
       "This is another sentence.",

packages/core/tsconfig.json

@@ -3,13 +3,14 @@
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
     "isolatedModules": true,
+    "module": "esnext",
     "moduleResolution": "node",
     "preserveWatchOutput": true,
     "skipLibCheck": true,
-    "noEmit": true,
     "strict": true,
     "lib": ["es2015", "dom"],
-    "target": "ES2015"
+    "target": "ES2015",
+    "resolveJsonModule": true
   },
   "exclude": ["node_modules"]
 }

packages/create-llama/README.md

@@ -0,0 +1,54 @@
+# Create LlamaIndex App
+
+The easiest way to get started with LlamaIndex is by using `create-llama`. This CLI tool enables you to quickly start building a new LlamaIndex application, with everything set up for you. 
+To get started, use the following command:
+
+### Interactive
+
+You can create a new project interactively by running:
+
+```bash
+npx create-llama@latest
+# or
+npm create llama
+# or
+yarn create llama
+# or
+pnpm create llama
+```
+
+You will be asked for the name of your project, and then which framework you want to use
+create a TypeScript project:
+
+```bash
+✔ Which framework would you like to use? › NextJS
+```
+
+You can choose between NextJS and Express.
+
+### Non-interactive
+
+You can also pass command line arguments to set up a new project
+non-interactively. See `create-llama --help`:
+
+```bash
+create-llama <project-directory> [options]
+
+Options:
+  -V, --version                      output the version number
+ 
+
+  --use-npm
+
+    Explicitly tell the CLI to bootstrap the app using npm
+
+  --use-pnpm
+
+    Explicitly tell the CLI to bootstrap the app using pnpm
+
+  --use-yarn
+
+    Explicitly tell the CLI to bootstrap the app using Yarn
+
+```
+

packages/create-llama/create-app.ts

@@ -0,0 +1,109 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import path from "path";
+import { green } from "picocolors";
+import { tryGitInit } from "./helpers/git";
+import { isFolderEmpty } from "./helpers/is-folder-empty";
+import { getOnline } from "./helpers/is-online";
+import { isWriteable } from "./helpers/is-writeable";
+import { makeDir } from "./helpers/make-dir";
+
+import fs from "fs";
+import terminalLink from "terminal-link";
+import type { InstallTemplateArgs } from "./templates";
+import { installTemplate } from "./templates";
+
+export async function createApp({
+  template,
+  framework,
+  engine,
+  ui,
+  appPath,
+  packageManager,
+  eslint,
+  frontend,
+  openAIKey,
+}: Omit<
+  InstallTemplateArgs,
+  "appName" | "root" | "isOnline" | "customApiPath"
+> & {
+  appPath: string;
+  frontend: boolean;
+}): Promise<void> {
+  const root = path.resolve(appPath);
+
+  if (!(await isWriteable(path.dirname(root)))) {
+    console.error(
+      "The application path is not writable, please check folder permissions and try again.",
+    );
+    console.error(
+      "It is likely you do not have write permissions for this folder.",
+    );
+    process.exit(1);
+  }
+
+  const appName = path.basename(root);
+
+  await makeDir(root);
+  if (!isFolderEmpty(root, appName)) {
+    process.exit(1);
+  }
+
+  const useYarn = packageManager === "yarn";
+  const isOnline = !useYarn || (await getOnline());
+
+  console.log(`Creating a new LlamaIndex app in ${green(root)}.`);
+  console.log();
+
+  const args = {
+    appName,
+    root,
+    template,
+    framework,
+    engine,
+    ui,
+    packageManager,
+    isOnline,
+    eslint,
+    openAIKey,
+  };
+
+  if (frontend) {
+    // install backend
+    const backendRoot = path.join(root, "backend");
+    await makeDir(backendRoot);
+    await installTemplate({ ...args, root: backendRoot, backend: true });
+    // install frontend
+    const frontendRoot = path.join(root, "frontend");
+    await makeDir(frontendRoot);
+    await installTemplate({
+      ...args,
+      root: frontendRoot,
+      framework: "nextjs",
+      customApiPath: "http://localhost:8000/api/chat",
+      backend: false,
+    });
+    // copy readme for fullstack
+    await fs.promises.copyFile(
+      path.join(__dirname, "templates", "README-fullstack.md"),
+      path.join(root, "README.md"),
+    );
+  } else {
+    await installTemplate({ ...args, backend: true });
+  }
+
+  process.chdir(root);
+  if (tryGitInit(root)) {
+    console.log("Initialized a git repository.");
+    console.log();
+  }
+
+  console.log(`${green("Success!")} Created ${appName} at ${appPath}`);
+
+  console.log(
+    `Now have a look at the ${terminalLink(
+      "README.md",
+      `file://${appName}/README.md`,
+    )} and learn how to get started.`,
+  );
+  console.log();
+}

packages/create-llama/helpers/copy.ts

@@ -0,0 +1,50 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { async as glob } from "fast-glob";
+import fs from "fs";
+import path from "path";
+
+interface CopyOption {
+  cwd?: string;
+  rename?: (basename: string) => string;
+  parents?: boolean;
+}
+
+const identity = (x: string) => x;
+
+export const copy = async (
+  src: string | string[],
+  dest: string,
+  { cwd, rename = identity, parents = true }: CopyOption = {},
+) => {
+  const source = typeof src === "string" ? [src] : src;
+
+  if (source.length === 0 || !dest) {
+    throw new TypeError("`src` and `dest` are required");
+  }
+
+  const sourceFiles = await glob(source, {
+    cwd,
+    dot: true,
+    absolute: false,
+    stats: false,
+  });
+
+  const destRelativeToCwd = cwd ? path.resolve(cwd, dest) : dest;
+
+  return Promise.all(
+    sourceFiles.map(async (p) => {
+      const dirname = path.dirname(p);
+      const basename = rename(path.basename(p));
+
+      const from = cwd ? path.resolve(cwd, p) : p;
+      const to = parents
+        ? path.join(destRelativeToCwd, dirname, basename)
+        : path.join(destRelativeToCwd, basename);
+
+      // Ensure the destination directory exists
+      await fs.promises.mkdir(path.dirname(to), { recursive: true });
+
+      return fs.promises.copyFile(from, to);
+    }),
+  );
+};

packages/create-llama/helpers/get-pkg-manager.ts

@@ -0,0 +1,15 @@
+export type PackageManager = "npm" | "pnpm" | "yarn";
+
+export function getPkgManager(): PackageManager {
+  const userAgent = process.env.npm_config_user_agent || "";
+
+  if (userAgent.startsWith("yarn")) {
+    return "yarn";
+  }
+
+  if (userAgent.startsWith("pnpm")) {
+    return "pnpm";
+  }
+
+  return "npm";
+}

packages/create-llama/helpers/git.ts

@@ -0,0 +1,58 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { execSync } from "child_process";
+import fs from "fs";
+import path from "path";
+
+function isInGitRepository(): boolean {
+  try {
+    execSync("git rev-parse --is-inside-work-tree", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+function isInMercurialRepository(): boolean {
+  try {
+    execSync("hg --cwd . root", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+function isDefaultBranchSet(): boolean {
+  try {
+    execSync("git config init.defaultBranch", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function tryGitInit(root: string): boolean {
+  let didInit = false;
+  try {
+    execSync("git --version", { stdio: "ignore" });
+    if (isInGitRepository() || isInMercurialRepository()) {
+      return false;
+    }
+
+    execSync("git init", { stdio: "ignore" });
+    didInit = true;
+
+    if (!isDefaultBranchSet()) {
+      execSync("git checkout -b main", { stdio: "ignore" });
+    }
+
+    execSync("git add -A", { stdio: "ignore" });
+    execSync('git commit -m "Initial commit from Create Next App"', {
+      stdio: "ignore",
+    });
+    return true;
+  } catch (e) {
+    if (didInit) {
+      try {
+        fs.rmSync(path.join(root, ".git"), { recursive: true, force: true });
+      } catch (_) {}
+    }
+    return false;
+  }
+}

packages/create-llama/helpers/install.ts

@@ -0,0 +1,50 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import spawn from "cross-spawn";
+import { yellow } from "picocolors";
+import type { PackageManager } from "./get-pkg-manager";
+
+/**
+ * Spawn a package manager installation based on user preference.
+ *
+ * @returns A Promise that resolves once the installation is finished.
+ */
+export async function callPackageManager(
+  /** Indicate which package manager to use. */
+  packageManager: PackageManager,
+  /** Indicate whether there is an active Internet connection.*/
+  isOnline: boolean,
+  args: string[] = ["install"],
+): Promise<void> {
+  if (!isOnline) {
+    console.log(
+      yellow("You appear to be offline.\nFalling back to the local cache."),
+    );
+    args.push("--offline");
+  }
+  /**
+   * Return a Promise that resolves once the installation is finished.
+   */
+  return new Promise((resolve, reject) => {
+    /**
+     * Spawn the installation process.
+     */
+    const child = spawn(packageManager, args, {
+      stdio: "inherit",
+      env: {
+        ...process.env,
+        ADBLOCK: "1",
+        // we set NODE_ENV to development as pnpm skips dev
+        // dependencies when production
+        NODE_ENV: "development",
+        DISABLE_OPENCOLLECTIVE: "1",
+      },
+    });
+    child.on("close", (code) => {
+      if (code !== 0) {
+        reject({ command: `${packageManager} ${args.join(" ")}` });
+        return;
+      }
+      resolve();
+    });
+  });
+}

packages/create-llama/helpers/is-folder-empty.ts

@@ -0,0 +1,62 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import fs from "fs";
+import path from "path";
+import { blue, green } from "picocolors";
+
+export function isFolderEmpty(root: string, name: string): boolean {
+  const validFiles = [
+    ".DS_Store",
+    ".git",
+    ".gitattributes",
+    ".gitignore",
+    ".gitlab-ci.yml",
+    ".hg",
+    ".hgcheck",
+    ".hgignore",
+    ".idea",
+    ".npmignore",
+    ".travis.yml",
+    "LICENSE",
+    "Thumbs.db",
+    "docs",
+    "mkdocs.yml",
+    "npm-debug.log",
+    "yarn-debug.log",
+    "yarn-error.log",
+    "yarnrc.yml",
+    ".yarn",
+  ];
+
+  const conflicts = fs
+    .readdirSync(root)
+    .filter((file) => !validFiles.includes(file))
+    // Support IntelliJ IDEA-based editors
+    .filter((file) => !/\.iml$/.test(file));
+
+  if (conflicts.length > 0) {
+    console.log(
+      `The directory ${green(name)} contains files that could conflict:`,
+    );
+    console.log();
+    for (const file of conflicts) {
+      try {
+        const stats = fs.lstatSync(path.join(root, file));
+        if (stats.isDirectory()) {
+          console.log(`  ${blue(file)}/`);
+        } else {
+          console.log(`  ${file}`);
+        }
+      } catch {
+        console.log(`  ${file}`);
+      }
+    }
+    console.log();
+    console.log(
+      "Either try using a new directory name, or remove the files listed above.",
+    );
+    console.log();
+    return false;
+  }
+
+  return true;
+}

packages/create-llama/helpers/is-online.ts

@@ -0,0 +1,40 @@
+import { execSync } from "child_process";
+import dns from "dns";
+import url from "url";
+
+function getProxy(): string | undefined {
+  if (process.env.https_proxy) {
+    return process.env.https_proxy;
+  }
+
+  try {
+    const httpsProxy = execSync("npm config get https-proxy").toString().trim();
+    return httpsProxy !== "null" ? httpsProxy : undefined;
+  } catch (e) {
+    return;
+  }
+}
+
+export function getOnline(): Promise<boolean> {
+  return new Promise((resolve) => {
+    dns.lookup("registry.yarnpkg.com", (registryErr) => {
+      if (!registryErr) {
+        return resolve(true);
+      }
+
+      const proxy = getProxy();
+      if (!proxy) {
+        return resolve(false);
+      }
+
+      const { hostname } = url.parse(proxy);
+      if (!hostname) {
+        return resolve(false);
+      }
+
+      dns.lookup(hostname, (proxyErr) => {
+        resolve(proxyErr == null);
+      });
+    });
+  });
+}

packages/create-llama/helpers/is-url.ts

@@ -0,0 +1,8 @@
+export function isUrl(url: string): boolean {
+  try {
+    new URL(url);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}

packages/create-llama/helpers/is-writeable.ts

@@ -0,0 +1,10 @@
+import fs from "fs";
+
+export async function isWriteable(directory: string): Promise<boolean> {
+  try {
+    await fs.promises.access(directory, (fs.constants || fs).W_OK);
+    return true;
+  } catch (err) {
+    return false;
+  }
+}

packages/create-llama/helpers/make-dir.ts

@@ -0,0 +1,8 @@
+import fs from "fs";
+
+export function makeDir(
+  root: string,
+  options = { recursive: true },
+): Promise<string | undefined> {
+  return fs.promises.mkdir(root, options);
+}

packages/create-llama/helpers/validate-pkg.ts

@@ -0,0 +1,20 @@
+// eslint-disable-next-line import/no-extraneous-dependencies
+import validateProjectName from "validate-npm-package-name";
+
+export function validateNpmName(name: string): {
+  valid: boolean;
+  problems?: string[];
+} {
+  const nameValidation = validateProjectName(name);
+  if (nameValidation.validForNewPackages) {
+    return { valid: true };
+  }
+
+  return {
+    valid: false,
+    problems: [
+      ...(nameValidation.errors || []),
+      ...(nameValidation.warnings || []),
+    ],
+  };
+}

packages/create-llama/index.ts

@@ -0,0 +1,399 @@
+#!/usr/bin/env node
+/* eslint-disable import/no-extraneous-dependencies */
+import ciInfo from "ci-info";
+import Commander from "commander";
+import Conf from "conf";
+import fs from "fs";
+import path from "path";
+import { blue, bold, cyan, green, red, yellow } from "picocolors";
+import prompts from "prompts";
+import checkForUpdate from "update-check";
+import { createApp } from "./create-app";
+import { getPkgManager } from "./helpers/get-pkg-manager";
+import { isFolderEmpty } from "./helpers/is-folder-empty";
+import { validateNpmName } from "./helpers/validate-pkg";
+import packageJson from "./package.json";
+
+let projectPath: string = "";
+
+const handleSigTerm = () => process.exit(0);
+
+process.on("SIGINT", handleSigTerm);
+process.on("SIGTERM", handleSigTerm);
+
+const onPromptState = (state: any) => {
+  if (state.aborted) {
+    // If we don't re-enable the terminal cursor before exiting
+    // the program, the cursor will remain hidden
+    process.stdout.write("\x1B[?25h");
+    process.stdout.write("\n");
+    process.exit(1);
+  }
+};
+
+const program = new Commander.Command(packageJson.name)
+  .version(packageJson.version)
+  .arguments("<project-directory>")
+  .usage(`${green("<project-directory>")} [options]`)
+  .action((name) => {
+    projectPath = name;
+  })
+  .option(
+    "--eslint",
+    `
+
+  Initialize with eslint config.
+`,
+  )
+  .option(
+    "--use-npm",
+    `
+
+  Explicitly tell the CLI to bootstrap the application using npm
+`,
+  )
+  .option(
+    "--use-pnpm",
+    `
+
+  Explicitly tell the CLI to bootstrap the application using pnpm
+`,
+  )
+  .option(
+    "--use-yarn",
+    `
+
+  Explicitly tell the CLI to bootstrap the application using Yarn
+`,
+  )
+  .option(
+    "--reset-preferences",
+    `
+
+  Explicitly tell the CLI to reset any stored preferences
+`,
+  )
+  .allowUnknownOption()
+  .parse(process.argv);
+
+const packageManager = !!program.useNpm
+  ? "npm"
+  : !!program.usePnpm
+  ? "pnpm"
+  : !!program.useYarn
+  ? "yarn"
+  : getPkgManager();
+
+async function run(): Promise<void> {
+  const conf = new Conf({ projectName: "create-llama" });
+
+  if (program.resetPreferences) {
+    conf.clear();
+    console.log(`Preferences reset successfully`);
+    return;
+  }
+
+  if (typeof projectPath === "string") {
+    projectPath = projectPath.trim();
+  }
+
+  if (!projectPath) {
+    const res = await prompts({
+      onState: onPromptState,
+      type: "text",
+      name: "path",
+      message: "What is your project named?",
+      initial: "my-app",
+      validate: (name) => {
+        const validation = validateNpmName(path.basename(path.resolve(name)));
+        if (validation.valid) {
+          return true;
+        }
+        return "Invalid project name: " + validation.problems![0];
+      },
+    });
+
+    if (typeof res.path === "string") {
+      projectPath = res.path.trim();
+    }
+  }
+
+  if (!projectPath) {
+    console.log(
+      "\nPlease specify the project directory:\n" +
+        `  ${cyan(program.name())} ${green("<project-directory>")}\n` +
+        "For example:\n" +
+        `  ${cyan(program.name())} ${green("my-next-app")}\n\n` +
+        `Run ${cyan(`${program.name()} --help`)} to see all options.`,
+    );
+    process.exit(1);
+  }
+
+  const resolvedProjectPath = path.resolve(projectPath);
+  const projectName = path.basename(resolvedProjectPath);
+
+  const { valid, problems } = validateNpmName(projectName);
+  if (!valid) {
+    console.error(
+      `Could not create a project called ${red(
+        `"${projectName}"`,
+      )} because of npm naming restrictions:`,
+    );
+
+    problems!.forEach((p) => console.error(`    ${red(bold("*"))} ${p}`));
+    process.exit(1);
+  }
+
+  /**
+   * Verify the project dir is empty or doesn't exist
+   */
+  const root = path.resolve(resolvedProjectPath);
+  const appName = path.basename(root);
+  const folderExists = fs.existsSync(root);
+
+  if (folderExists && !isFolderEmpty(root, appName)) {
+    process.exit(1);
+  }
+
+  const preferences = (conf.get("preferences") || {}) as Record<
+    string,
+    boolean | string
+  >;
+
+  const defaults: typeof preferences = {
+    template: "simple",
+    framework: "nextjs",
+    engine: "simple",
+    ui: "html",
+    eslint: true,
+    frontend: false,
+    openAIKey: "",
+  };
+  const getPrefOrDefault = (field: string) =>
+    preferences[field] ?? defaults[field];
+
+  const handlers = {
+    onCancel: () => {
+      console.error("Exiting.");
+      process.exit(1);
+    },
+  };
+
+  if (!program.template) {
+    if (ciInfo.isCI) {
+      program.template = getPrefOrDefault("template");
+    } else {
+      const { template } = await prompts(
+        {
+          type: "select",
+          name: "template",
+          message: "Which template would you like to use?",
+          choices: [
+            { title: "Chat without streaming", value: "simple" },
+            { title: "Chat with streaming", value: "streaming" },
+          ],
+          initial: 1,
+        },
+        handlers,
+      );
+      program.template = template;
+      preferences.template = template;
+    }
+  }
+
+  if (!program.framework) {
+    if (ciInfo.isCI) {
+      program.framework = getPrefOrDefault("framework");
+    } else {
+      const { framework } = await prompts(
+        {
+          type: "select",
+          name: "framework",
+          message: "Which framework would you like to use?",
+          choices: [
+            { title: "NextJS", value: "nextjs" },
+            { title: "Express", value: "express" },
+            { title: "FastAPI (Python)", value: "fastapi" },
+          ],
+          initial: 0,
+        },
+        handlers,
+      );
+      program.framework = framework;
+      preferences.framework = framework;
+    }
+  }
+
+  if (program.framework === "express" || program.framework === "fastapi") {
+    // if a backend-only framework is selected, ask whether we should create a frontend
+    if (!program.frontend) {
+      if (ciInfo.isCI) {
+        program.frontend = getPrefOrDefault("frontend");
+      } else {
+        const styledNextJS = blue("NextJS");
+        const styledBackend = green(
+          program.framework === "express"
+            ? "Express "
+            : program.framework === "fastapi"
+            ? "FastAPI (Python) "
+            : "",
+        );
+        const { frontend } = await prompts({
+          onState: onPromptState,
+          type: "toggle",
+          name: "frontend",
+          message: `Would you like to generate a ${styledNextJS} frontend for your ${styledBackend}backend?`,
+          initial: getPrefOrDefault("frontend"),
+          active: "Yes",
+          inactive: "No",
+        });
+        program.frontend = Boolean(frontend);
+        preferences.frontend = Boolean(frontend);
+      }
+    }
+  }
+
+  if (program.framework === "nextjs" || program.frontend) {
+    if (!program.ui) {
+      if (ciInfo.isCI) {
+        program.ui = getPrefOrDefault("ui");
+      } else {
+        const { ui } = await prompts(
+          {
+            type: "select",
+            name: "ui",
+            message: "Which UI would you like to use?",
+            choices: [
+              { title: "Just HTML", value: "html" },
+              { title: "Shadcn", value: "shadcn" },
+            ],
+            initial: 0,
+          },
+          handlers,
+        );
+        program.ui = ui;
+        preferences.ui = ui;
+      }
+    }
+  }
+
+  if (program.framework === "express" || program.framework === "nextjs") {
+    if (!program.engine) {
+      if (ciInfo.isCI) {
+        program.engine = getPrefOrDefault("engine");
+      } else {
+        const { engine } = await prompts(
+          {
+            type: "select",
+            name: "engine",
+            message: "Which chat engine would you like to use?",
+            choices: [
+              { title: "SimpleChatEngine", value: "simple" },
+              { title: "ContextChatEngine", value: "context" },
+            ],
+            initial: 0,
+          },
+          handlers,
+        );
+        program.engine = engine;
+        preferences.engine = engine;
+      }
+    }
+  }
+
+  if (!program.openAIKey) {
+    const { key } = await prompts(
+      {
+        type: "text",
+        name: "key",
+        message: "Please provide your OpenAI API key (leave blank to skip):",
+      },
+      handlers,
+    );
+    program.openAIKey = key;
+    preferences.openAIKey = key;
+  }
+
+  if (
+    program.framework !== "fastapi" &&
+    !process.argv.includes("--eslint") &&
+    !process.argv.includes("--no-eslint")
+  ) {
+    if (ciInfo.isCI) {
+      program.eslint = getPrefOrDefault("eslint");
+    } else {
+      const styledEslint = blue("ESLint");
+      const { eslint } = await prompts({
+        onState: onPromptState,
+        type: "toggle",
+        name: "eslint",
+        message: `Would you like to use ${styledEslint}?`,
+        initial: getPrefOrDefault("eslint"),
+        active: "Yes",
+        inactive: "No",
+      });
+      program.eslint = Boolean(eslint);
+      preferences.eslint = Boolean(eslint);
+    }
+  }
+
+  await createApp({
+    template: program.template,
+    framework: program.framework,
+    engine: program.engine,
+    ui: program.ui,
+    appPath: resolvedProjectPath,
+    packageManager,
+    eslint: program.eslint,
+    frontend: program.frontend,
+    openAIKey: program.openAIKey,
+  });
+  conf.set("preferences", preferences);
+}
+
+const update = checkForUpdate(packageJson).catch(() => null);
+
+async function notifyUpdate(): Promise<void> {
+  try {
+    const res = await update;
+    if (res?.latest) {
+      const updateMessage =
+        packageManager === "yarn"
+          ? "yarn global add create-llama"
+          : packageManager === "pnpm"
+          ? "pnpm add -g create-llama"
+          : "npm i -g create-llama";
+
+      console.log(
+        yellow(bold("A new version of `create-llama` is available!")) +
+          "\n" +
+          "You can update by running: " +
+          cyan(updateMessage) +
+          "\n",
+      );
+    }
+    process.exit();
+  } catch {
+    // ignore error
+  }
+}
+
+run()
+  .then(notifyUpdate)
+  .catch(async (reason) => {
+    console.log();
+    console.log("Aborting installation.");
+    if (reason.command) {
+      console.log(`  ${cyan(reason.command)} has failed.`);
+    } else {
+      console.log(
+        red("Unexpected error. Please report it as a bug:") + "\n",
+        reason,
+      );
+    }
+    console.log();
+
+    await notifyUpdate();
+
+    process.exit(1);
+  });

packages/create-llama/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "create-llama",
+  "version": "0.0.1",
+  "keywords": [
+    "rag",
+    "llamaindex",
+    "next.js"
+  ],
+  "description": "Create LlamaIndex-powered apps with one command",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/run-llama/LlamaIndexTS",
+    "directory": "packages/create-llama"
+  },
+  "license": "MIT",
+  "bin": {
+    "create-llama": "./dist/index.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "dev": "ncc build ./index.ts -w -o dist/",
+    "build": "ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register",
+    "lint": "eslint . --ignore-pattern dist",
+    "prepublishOnly": "cd ../../ && turbo run build"
+  },
+  "devDependencies": {
+    "@types/async-retry": "1.4.2",
+    "@types/ci-info": "2.0.0",
+    "@types/cross-spawn": "6.0.0",
+    "@types/node": "^20.9.0",
+    "@types/prompts": "2.0.1",
+    "@types/tar": "6.1.5",
+    "@types/validate-npm-package-name": "3.0.0",
+    "@vercel/ncc": "0.34.0",
+    "async-retry": "1.3.1",
+    "async-sema": "3.0.1",
+    "ci-info": "github:watson/ci-info#f43f6a1cefff47fb361c88cf4b943fdbcaafe540",
+    "commander": "2.20.0",
+    "conf": "10.2.0",
+    "cross-spawn": "7.0.3",
+    "fast-glob": "3.3.1",
+    "got": "10.7.0",
+    "picocolors": "1.0.0",
+    "prompts": "2.1.0",
+    "tar": "6.1.15",
+    "terminal-link": "^3.0.0",
+    "update-check": "1.5.4",
+    "validate-npm-package-name": "3.0.0"
+  },
+  "engines": {
+    "node": ">=16.14.0"
+  }
+}

packages/create-llama/templates/.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+poetry.lock
+storage

packages/create-llama/templates/README-fullstack.md

@@ -0,0 +1,18 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, startup the backend as described in the [backend README](./backend/README.md).
+
+Second, run the development server of the frontend as described in the [frontend README](./frontend/README.md).
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

packages/create-llama/templates/components/engines/context/constants.mjs

@@ -0,0 +1,4 @@
+export const STORAGE_DIR = "./data";
+export const STORAGE_CACHE_DIR = "./cache";
+export const CHUNK_SIZE = 512;
+export const CHUNK_OVERLAP = 20;

packages/create-llama/templates/components/engines/context/generate.mjs

@@ -0,0 +1,48 @@
+import {
+  serviceContextFromDefaults,
+  SimpleDirectoryReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+
+import {
+  CHUNK_OVERLAP,
+  CHUNK_SIZE,
+  STORAGE_CACHE_DIR,
+  STORAGE_DIR,
+} from "./constants.mjs";
+
+async function getRuntime(func) {
+  const start = Date.now();
+  await func();
+  const end = Date.now();
+  return end - start;
+}
+
+async function generateDatasource(serviceContext) {
+  console.log(`Generating storage context...`);
+  // Split documents, create embeddings and store them in the storage context
+  const ms = await getRuntime(async () => {
+    const storageContext = await storageContextFromDefaults({
+      persistDir: STORAGE_CACHE_DIR,
+    });
+    const documents = await new SimpleDirectoryReader().loadData({
+      directoryPath: STORAGE_DIR,
+    });
+    await VectorStoreIndex.fromDocuments(documents, {
+      storageContext,
+      serviceContext,
+    });
+  });
+  console.log(`Storage context successfully generated in ${ms / 1000}s.`);
+}
+
+(async () => {
+  const serviceContext = serviceContextFromDefaults({
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
+
+  await generateDatasource(serviceContext);
+  console.log("Finished generating storage.");
+})();

packages/create-llama/templates/components/engines/context/index.ts

@@ -0,0 +1,44 @@
+import {
+  ContextChatEngine,
+  LLM,
+  serviceContextFromDefaults,
+  SimpleDocumentStore,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./constants.mjs";
+
+async function getDataSource(llm: LLM) {
+  const serviceContext = serviceContextFromDefaults({
+    llm,
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
+  let storageContext = await storageContextFromDefaults({
+    persistDir: `${STORAGE_CACHE_DIR}`,
+  });
+
+  const numberOfDocs = Object.keys(
+    (storageContext.docStore as SimpleDocumentStore).toDict(),
+  ).length;
+  if (numberOfDocs === 0) {
+    throw new Error(
+      `StorageContext is empty - call 'npm run generate' to generate the storage first`,
+    );
+  }
+  return await VectorStoreIndex.init({
+    storageContext,
+    serviceContext,
+  });
+}
+
+export async function createChatEngine(llm: LLM) {
+  const index = await getDataSource(llm);
+  const retriever = index.asRetriever();
+  retriever.similarityTopK = 5;
+
+  return new ContextChatEngine({
+    chatModel: llm,
+    retriever,
+  });
+}

packages/create-llama/templates/components/ui/shadcn/README-template.md

@@ -0,0 +1 @@
+Using the chat component from https://github.com/marcusschiesser/ui (based on https://ui.shadcn.com/)

packages/create-llama/templates/components/ui/shadcn/button.tsx

@@ -0,0 +1,56 @@
+import { Slot } from "@radix-ui/react-slot";
+import { cva, type VariantProps } from "class-variance-authority";
+import * as React from "react";
+
+import { cn } from "./lib/utils";
+
+const buttonVariants = cva(
+  "inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50",
+  {
+    variants: {
+      variant: {
+        default: "bg-primary text-primary-foreground hover:bg-primary/90",
+        destructive:
+          "bg-destructive text-destructive-foreground hover:bg-destructive/90",
+        outline:
+          "border border-input bg-background hover:bg-accent hover:text-accent-foreground",
+        secondary:
+          "bg-secondary text-secondary-foreground hover:bg-secondary/80",
+        ghost: "hover:bg-accent hover:text-accent-foreground",
+        link: "text-primary underline-offset-4 hover:underline",
+      },
+      size: {
+        default: "h-10 px-4 py-2",
+        sm: "h-9 rounded-md px-3",
+        lg: "h-11 rounded-md px-8",
+        icon: "h-10 w-10",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+      size: "default",
+    },
+  },
+);
+
+export interface ButtonProps
+  extends React.ButtonHTMLAttributes<HTMLButtonElement>,
+    VariantProps<typeof buttonVariants> {
+  asChild?: boolean;
+}
+
+const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
+  ({ className, variant, size, asChild = false, ...props }, ref) => {
+    const Comp = asChild ? Slot : "button";
+    return (
+      <Comp
+        className={cn(buttonVariants({ variant, size, className }))}
+        ref={ref}
+        {...props}
+      />
+    );
+  },
+);
+Button.displayName = "Button";
+
+export { Button, buttonVariants };

packages/create-llama/templates/components/ui/shadcn/chat/chat-actions.tsx

@@ -0,0 +1,28 @@
+import { PauseCircle, RefreshCw } from "lucide-react";
+
+import { Button } from "../button";
+import { ChatHandler } from "./chat.interface";
+
+export default function ChatActions(
+  props: Pick<ChatHandler, "stop" | "reload"> & {
+    showReload?: boolean;
+    showStop?: boolean;
+  },
+) {
+  return (
+    <div className="space-x-4">
+      {props.showStop && (
+        <Button variant="outline" size="sm" onClick={props.stop}>
+          <PauseCircle className="mr-2 h-4 w-4" />
+          Stop generating
+        </Button>
+      )}
+      {props.showReload && (
+        <Button variant="outline" size="sm" onClick={props.reload}>
+          <RefreshCw className="mr-2 h-4 w-4" />
+          Regenerate
+        </Button>
+      )}
+    </div>
+  );
+}

packages/create-llama/templates/components/ui/shadcn/chat/chat-avatar.tsx

@@ -0,0 +1,25 @@
+import { User2 } from "lucide-react";
+import Image from "next/image";
+
+export default function ChatAvatar({ role }: { role: string }) {
+  if (role === "user") {
+    return (
+      <div className="flex h-8 w-8 shrink-0 select-none items-center justify-center rounded-md border bg-background shadow">
+        <User2 className="h-4 w-4" />
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex h-8 w-8 shrink-0 select-none items-center justify-center rounded-md border bg-black text-white shadow">
+      <Image
+        className="rounded-md"
+        src="/llama.png"
+        alt="Llama Logo"
+        width={24}
+        height={24}
+        priority
+      />
+    </div>
+  );
+}

packages/create-llama/templates/components/ui/shadcn/chat/chat-input.tsx

@@ -0,0 +1,29 @@
+import { Button } from "../button";
+import { Input } from "../input";
+import { ChatHandler } from "./chat.interface";
+
+export default function ChatInput(
+  props: Pick<
+    ChatHandler,
+    "isLoading" | "handleSubmit" | "handleInputChange" | "input"
+  >,
+) {
+  return (
+    <form
+      onSubmit={props.handleSubmit}
+      className="flex w-full items-start justify-between gap-4 rounded-xl bg-white p-4 shadow-xl"
+    >
+      <Input
+        autoFocus
+        name="message"
+        placeholder="Type a message"
+        className="flex-1"
+        value={props.input}
+        onChange={props.handleInputChange}
+      />
+      <Button type="submit" disabled={props.isLoading}>
+        Send message
+      </Button>
+    </form>
+  );
+}

packages/create-llama/templates/components/ui/shadcn/chat/chat-message.tsx

@@ -0,0 +1,33 @@
+import { Check, Copy } from "lucide-react";
+
+import { Button } from "../button";
+import ChatAvatar from "./chat-avatar";
+import { Message } from "./chat.interface";
+import Markdown from "./markdown";
+import { useCopyToClipboard } from "./use-copy-to-clipboard";
+
+export default function ChatMessage(chatMessage: Message) {
+  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 2000 });
+  return (
+    <div className="flex items-start gap-4 pr-5 pt-5">
+      <ChatAvatar role={chatMessage.role} />
+      <div className="group flex flex-1 justify-between gap-2">
+        <div className="flex-1">
+          <Markdown content={chatMessage.content} />
+        </div>
+        <Button
+          onClick={() => copyToClipboard(chatMessage.content)}
+          size="icon"
+          variant="ghost"
+          className="h-8 w-8 opacity-0 group-hover:opacity-100"
+        >
+          {isCopied ? (
+            <Check className="h-4 w-4" />
+          ) : (
+            <Copy className="h-4 w-4" />
+          )}
+        </Button>
+      </div>
+    </div>
+  );
+}

packages/create-llama/templates/components/ui/shadcn/chat/chat-messages.tsx

@@ -0,0 +1,51 @@
+import { useEffect, useRef } from "react";
+
+import ChatActions from "./chat-actions";
+import ChatMessage from "./chat-message";
+import { ChatHandler } from "./chat.interface";
+
+export default function ChatMessages(
+  props: Pick<ChatHandler, "messages" | "isLoading" | "reload" | "stop">,
+) {
+  const scrollableChatContainerRef = useRef<HTMLDivElement>(null);
+  const messageLength = props.messages.length;
+  const lastMessage = props.messages[messageLength - 1];
+
+  const scrollToBottom = () => {
+    if (scrollableChatContainerRef.current) {
+      scrollableChatContainerRef.current.scrollTop =
+        scrollableChatContainerRef.current.scrollHeight;
+    }
+  };
+
+  const isLastMessageFromAssistant =
+    messageLength > 0 && lastMessage?.role !== "user";
+  const showReload =
+    props.reload && !props.isLoading && isLastMessageFromAssistant;
+  const showStop = props.stop && props.isLoading;
+
+  useEffect(() => {
+    scrollToBottom();
+  }, [messageLength, lastMessage]);
+
+  return (
+    <div className="w-full rounded-xl bg-white p-4 shadow-xl pb-0">
+      <div
+        className="flex h-[50vh] flex-col gap-5 divide-y overflow-y-auto pb-4"
+        ref={scrollableChatContainerRef}
+      >
+        {props.messages.map((m) => (
+          <ChatMessage key={m.id} {...m} />
+        ))}
+      </div>
+      <div className="flex justify-end py-4">
+        <ChatActions
+          reload={props.reload}
+          stop={props.stop}
+          showReload={showReload}
+          showStop={showStop}
+        />
+      </div>
+    </div>
+  );
+}

packages/create-llama/templates/components/ui/shadcn/chat/chat.interface.ts

@@ -0,0 +1,15 @@
+export interface Message {
+  id: string;
+  content: string;
+  role: string;
+}
+
+export interface ChatHandler {
+  messages: Message[];
+  input: string;
+  isLoading: boolean;
+  handleSubmit: (e: React.FormEvent<HTMLFormElement>) => void;
+  handleInputChange: (e: React.ChangeEvent<HTMLInputElement>) => void;
+  reload?: () => void;
+  stop?: () => void;
+}

packages/create-llama/templates/components/ui/shadcn/chat/codeblock.tsx

@@ -0,0 +1,139 @@
+"use client"
+
+import React, { FC, memo } from "react"
+import { Check, Copy, Download } from "lucide-react"
+import { Prism, SyntaxHighlighterProps } from "react-syntax-highlighter"
+import { coldarkDark } from "react-syntax-highlighter/dist/cjs/styles/prism"
+
+import { Button } from "../button"
+import { useCopyToClipboard } from "./use-copy-to-clipboard"
+
+// TODO: Remove this when @type/react-syntax-highlighter is updated
+const SyntaxHighlighter = Prism as unknown as FC<SyntaxHighlighterProps>
+
+interface Props {
+  language: string
+  value: string
+}
+
+interface languageMap {
+  [key: string]: string | undefined
+}
+
+export const programmingLanguages: languageMap = {
+  javascript: ".js",
+  python: ".py",
+  java: ".java",
+  c: ".c",
+  cpp: ".cpp",
+  "c++": ".cpp",
+  "c#": ".cs",
+  ruby: ".rb",
+  php: ".php",
+  swift: ".swift",
+  "objective-c": ".m",
+  kotlin: ".kt",
+  typescript: ".ts",
+  go: ".go",
+  perl: ".pl",
+  rust: ".rs",
+  scala: ".scala",
+  haskell: ".hs",
+  lua: ".lua",
+  shell: ".sh",
+  sql: ".sql",
+  html: ".html",
+  css: ".css",
+  // add more file extensions here, make sure the key is same as language prop in CodeBlock.tsx component
+}
+
+export const generateRandomString = (length: number, lowercase = false) => {
+  const chars = "ABCDEFGHJKLMNPQRSTUVWXY3456789" // excluding similar looking characters like Z, 2, I, 1, O, 0
+  let result = ""
+  for (let i = 0; i < length; i++) {
+    result += chars.charAt(Math.floor(Math.random() * chars.length))
+  }
+  return lowercase ? result.toLowerCase() : result
+}
+
+const CodeBlock: FC<Props> = memo(({ language, value }) => {
+  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 2000 })
+
+  const downloadAsFile = () => {
+    if (typeof window === "undefined") {
+      return
+    }
+    const fileExtension = programmingLanguages[language] || ".file"
+    const suggestedFileName = `file-${generateRandomString(
+      3,
+      true
+    )}${fileExtension}`
+    const fileName = window.prompt("Enter file name" || "", suggestedFileName)
+
+    if (!fileName) {
+      // User pressed cancel on prompt.
+      return
+    }
+
+    const blob = new Blob([value], { type: "text/plain" })
+    const url = URL.createObjectURL(blob)
+    const link = document.createElement("a")
+    link.download = fileName
+    link.href = url
+    link.style.display = "none"
+    document.body.appendChild(link)
+    link.click()
+    document.body.removeChild(link)
+    URL.revokeObjectURL(url)
+  }
+
+  const onCopy = () => {
+    if (isCopied) return
+    copyToClipboard(value)
+  }
+
+  return (
+    <div className="codeblock relative w-full bg-zinc-950 font-sans">
+      <div className="flex w-full items-center justify-between bg-zinc-800 px-6 py-2 pr-4 text-zinc-100">
+        <span className="text-xs lowercase">{language}</span>
+        <div className="flex items-center space-x-1">
+          <Button variant="ghost" onClick={downloadAsFile} size="icon">
+            <Download />
+            <span className="sr-only">Download</span>
+          </Button>
+          <Button variant="ghost" size="icon" onClick={onCopy}>
+            {isCopied ? (
+              <Check className="h-4 w-4" />
+            ) : (
+              <Copy className="h-4 w-4" />
+            )}
+            <span className="sr-only">Copy code</span>
+          </Button>
+        </div>
+      </div>
+      <SyntaxHighlighter
+        language={language}
+        style={coldarkDark}
+        PreTag="div"
+        showLineNumbers
+        customStyle={{
+          width: "100%",
+          background: "transparent",
+          padding: "1.5rem 1rem",
+          borderRadius: "0.5rem",
+        }}
+        codeTagProps={{
+          style: {
+            fontSize: "0.9rem",
+            fontFamily: "var(--font-mono)",
+          },
+        }}
+      >
+        {value}
+      </SyntaxHighlighter>
+    </div>
+  )
+})
+CodeBlock.displayName = "CodeBlock"
+
+export { CodeBlock }

packages/create-llama/templates/components/ui/shadcn/chat/index.ts

@@ -0,0 +1,5 @@
+import ChatInput from "./chat-input";
+import ChatMessages from "./chat-messages";
+
+export { type ChatHandler, type Message } from "./chat.interface";
+export { ChatMessages, ChatInput };

packages/create-llama/templates/components/ui/shadcn/chat/markdown.tsx

@@ -0,0 +1,59 @@
+import { FC, memo } from "react"
+import ReactMarkdown, { Options } from "react-markdown"
+import remarkGfm from "remark-gfm"
+import remarkMath from "remark-math"
+
+import { CodeBlock } from "./codeblock"
+
+const MemoizedReactMarkdown: FC<Options> = memo(
+  ReactMarkdown,
+  (prevProps, nextProps) =>
+    prevProps.children === nextProps.children &&
+    prevProps.className === nextProps.className
+)
+
+export default function Markdown({ content }: { content: string }) {
+  return (
+    <MemoizedReactMarkdown
+      className="prose dark:prose-invert prose-p:leading-relaxed prose-pre:p-0 break-words"
+      remarkPlugins={[remarkGfm, remarkMath]}
+      components={{
+        p({ children }) {
+          return <p className="mb-2 last:mb-0">{children}</p>
+        },
+        code({ node, inline, className, children, ...props }) {
+          if (children.length) {
+            if (children[0] == "▍") {
+              return (
+                <span className="mt-1 animate-pulse cursor-default">▍</span>
+              )
+            }
+
+            children[0] = (children[0] as string).replace("`▍`", "▍")
+          }
+
+          const match = /language-(\w+)/.exec(className || "")
+
+          if (inline) {
+            return (
+              <code className={className} {...props}>
+                {children}
+              </code>
+            )
+          }
+
+          return (
+            <CodeBlock
+              key={Math.random()}
+              language={(match && match[1]) || ""}
+              value={String(children).replace(/\n$/, "")}
+              {...props}
+            />
+          )
+        },
+      }}
+    >
+      {content}
+    </MemoizedReactMarkdown>
+  )
+}

packages/create-llama/templates/components/ui/shadcn/chat/use-copy-to-clipboard.tsx

@@ -0,0 +1,33 @@
+'use client'
+
+import * as React from 'react'
+
+export interface useCopyToClipboardProps {
+  timeout?: number
+}
+
+export function useCopyToClipboard({
+  timeout = 2000
+}: useCopyToClipboardProps) {
+  const [isCopied, setIsCopied] = React.useState<Boolean>(false)
+
+  const copyToClipboard = (value: string) => {
+    if (typeof window === 'undefined' || !navigator.clipboard?.writeText) {
+      return
+    }
+
+    if (!value) {
+      return
+    }
+
+    navigator.clipboard.writeText(value).then(() => {
+      setIsCopied(true)
+
+      setTimeout(() => {
+        setIsCopied(false)
+      }, timeout)
+    })
+  }
+
+  return { isCopied, copyToClipboard }
+}