changesets

feat: improved chat history summarizer

.changeset/big-hairs-eat.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Streaming improvements including Anthropic (thanks @kkang2097)

.changeset/breezy-days-admire.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Portkey integration (Thank you @noble-varghese)

.changeset/honest-students-care.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add export for PromptHelper (thanks @zigamall)

.changeset/odd-mangos-judge.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Chat History summarization (thanks @marcusschlesser)

.changeset/shaggy-cooks-push.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Notion database support (thanks @TomPenguin)

.changeset/six-hats-compare.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Publish ESM module again

.changeset/sixty-vans-punch.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-KeywordIndex (thanks @swk777)

.changeset/twelve-otters-wash.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Pinecone demo (thanks @Einsenhorn)

apps/docs/docs/end_to_end.md

@@ -14,7 +14,7 @@ Read a file and chat about it with the LLM.
 
 Create a vector index and query it. The vector index will use embeddings to fetch the top k most relevant nodes. By default, the top k is 2.
 
-## [Summary Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/summarIndex.ts)
+## [Summary Index](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/summaryIndex.ts)
 
 Create a list index and query it. This example also use the `LLMRetriever`, which will use the LLM to select the best nodes to use when generating answer.
 

apps/docs/docs/introduction.md

@@ -19,7 +19,7 @@ That's where **LlamaIndex.TS** comes in.
 
 LlamaIndex.TS provides the following tools:
 
-- **Data loading** ingest your existing `txt` and `pdf` data directly
+- **Data loading** ingest your existing `.txt`, `.pdf`, `.csv`, `.md` and `.docx` data directly
 - **Data indexes** structure your data in intermediate representations that are easy and performant for LLMs to consume.
 - **Engines** provide natural language access to your data. For example:
   - Query engines are powerful retrieval interfaces for knowledge-augmented output.

apps/docs/docs/modules/high_level/data_loader.md

@@ -4,7 +4,7 @@ sidebar_position: 1
 
 # Reader / Loader
 
-LlamaIndex.TS supports easy loading of files from folders using the `SimpleDirectoryReader` class. Currently, `.txt` and `.pdf` files are supported, with more planned in the future!
+LlamaIndex.TS supports easy loading of files from folders using the `SimpleDirectoryReader` class. Currently, `.txt`, `.pdf`, `.csv`, `.md` and `.docx` files are supported, with more planned in the future!
 
 ```typescript
 import { SimpleDirectoryReader } from "llamaindex";

apps/docs/package.json

@@ -15,24 +15,24 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "2.4.1",
-    "@docusaurus/preset-classic": "2.4.1",
-    "@docusaurus/remark-plugin-npm2yarn": "^2.4.1",
+    "@docusaurus/core": "2.4.3",
+    "@docusaurus/preset-classic": "2.4.3",
+    "@docusaurus/remark-plugin-npm2yarn": "^2.4.3",
     "@mdx-js/react": "^1.6.22",
     "clsx": "^1.2.1",
-    "postcss": "^8.4.28",
+    "postcss": "^8.4.31",
     "prism-react-renderer": "^1.3.5",
     "raw-loader": "^4.0.2",
     "react": "^17.0.2",
     "react-dom": "^17.0.2"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "2.4.1",
-    "@docusaurus/types": "^2.4.1",
-    "@tsconfig/docusaurus": "^1.0.7",
+    "@docusaurus/module-type-aliases": "2.4.3",
+    "@docusaurus/types": "^2.4.3",
+    "@tsconfig/docusaurus": "^2.0.1",
     "docusaurus-plugin-typedoc": "^0.19.2",
     "typedoc": "^0.24.8",
-    "typedoc-plugin-markdown": "^3.15.4",
+    "typedoc-plugin-markdown": "^3.16.0",
     "typescript": "^4.9.5"
   },
   "browserslist": {

apps/simple/CHANGELOG.md

@@ -1,5 +1,32 @@
 # simple
 
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [a52143b]
+- Updated dependencies [1b7fd95]
+- Updated dependencies [0db3f41]
+  - llamaindex@0.0.29
+
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [96bb657]
+- Updated dependencies [96bb657]
+- Updated dependencies [837854d]
+  - llamaindex@0.0.28
+
+## 0.0.25
+
+### Patch Changes
+
+- Updated dependencies [4a5591b]
+- Updated dependencies [4a5591b]
+- Updated dependencies [4a5591b]
+  - llamaindex@0.0.27
+
 ## 0.0.24
 
 ### Patch Changes

apps/simple/llmStream.ts

@@ -0,0 +1,47 @@
+import { ChatMessage, OpenAI, SimpleChatEngine } from "llamaindex";
+import {Anthropic} from "../../packages/core/src/llm/LLM";
+import { stdin as input, stdout as output } from "node:process";
+import readline from "node:readline/promises";
+
+async function main() {
+  const query: string = `
+Where is Istanbul?
+  `;
+
+  // const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 });
+  const llm = new Anthropic();
+  const message: ChatMessage = { content: query, role: "user" };
+
+  //TODO: Add callbacks later
+
+  //Stream Complete
+  //Note: Setting streaming flag to true or false will auto-set your return type to
+  //either an AsyncGenerator or a Response.
+  // Omitting the streaming flag automatically sets streaming to false
+
+  const chatEngine: SimpleChatEngine = new SimpleChatEngine({
+    chatHistory: undefined,
+    llm: llm,
+  });
+
+  const rl = readline.createInterface({ input, output });
+  while (true) {
+    const query = await rl.question("Query: ");
+
+    if (!query) {
+      break;
+    }
+
+    //Case 1: .chat(query, undefined, true) => Stream
+    //Case 2: .chat(query, undefined, false) => Response object
+    //Case 3: .chat(query, undefined) => Response object
+    const chatStream = await chatEngine.chat(query, undefined, true);
+    var accumulated_result = "";
+    for await (const part of chatStream) {
+      accumulated_result += part;
+      process.stdout.write(part);
+    }
+  }
+}
+
+main();

apps/simple/mongo.ts

@@ -0,0 +1,68 @@
+import { MongoClient } from "mongodb";
+import { VectorStoreIndex } from "../../packages/core/src/indices";
+import { Document } from "../../packages/core/src/Node";
+import { SimpleMongoReader } from "../../packages/core/src/readers/SimpleMongoReader";
+
+import { stdin as input, stdout as output } from "node:process";
+import readline from "node:readline/promises";
+
+async function main() {
+  //Dummy test code
+  const query: object = { _id: "waldo" };
+  const options: object = {};
+  const projections: object = { embedding: 0 };
+  const limit: number = Infinity;
+  const uri: string = process.env.MONGODB_URI ?? "fake_uri";
+  const client: MongoClient = new MongoClient(uri);
+
+  //Where the real code starts
+  const MR = new SimpleMongoReader(client);
+  const documents: Document[] = await MR.loadData(
+    "data",
+    "posts",
+    1,
+    {},
+    options,
+    projections,
+  );
+
+  //
+  //If you need to look at low-level details of
+  // a queryEngine (for example, needing to check each individual node)
+  //
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  // var storageContext = await storageContextFromDefaults({});
+  // var serviceContext = serviceContextFromDefaults({});
+  // const docStore = storageContext.docStore;
+
+  // for (const doc of documents) {
+  //   docStore.setDocumentHash(doc.id_, doc.hash);
+  // }
+  // const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
+  // console.log(nodes);
+
+  //
+  //Making Vector Store from documents
+  //
+
+  const index = await VectorStoreIndex.fromDocuments(documents);
+  // Create query engine
+  const queryEngine = index.asQueryEngine();
+
+  const rl = readline.createInterface({ input, output });
+  while (true) {
+    const query = await rl.question("Query: ");
+
+    if (!query) {
+      break;
+    }
+
+    const response = await queryEngine.query(query);
+
+    // Output response
+    console.log(response.toString());
+  }
+}
+
+main();

apps/simple/package.json

@@ -1,9 +1,10 @@
 {
-  "version": "0.0.24",
+  "version": "0.0.27",
   "private": true,
   "name": "simple",
   "dependencies": {
     "@notionhq/client": "^2.2.12",
+    "@pinecone-database/pinecone": "^1.0.1",
     "commander": "^11.0.0",
     "llamaindex": "workspace:*"
   },

apps/simple/portkey.ts

@@ -0,0 +1,23 @@
+import { Portkey } from "llamaindex";
+
+(async () => {
+  const llms = [{
+
+  }]
+  const portkey = new Portkey({
+    mode: "single",
+    llms: [{
+      provider:"anyscale",
+      virtual_key:"anyscale-3b3c04",
+      model: "meta-llama/Llama-2-13b-chat-hf",
+      max_tokens: 2000
+    }]
+  });
+  const result = portkey.stream_chat([
+    { role: "system", content: "You are a helpful assistant." },
+    { role: "user", content: "Tell me a joke." }
+  ]);
+  for await (const res of result) {
+    process.stdout.write(res)
+  }
+})();

apps/simple/vectorIndexFromVectorStore.ts

@@ -0,0 +1,197 @@
+import {
+  OpenAI,
+  ResponseSynthesizer,
+  RetrieverQueryEngine,
+  serviceContextFromDefaults,
+  TextNode,
+  TreeSummarize,
+  VectorIndexRetriever,
+  VectorStore,
+  VectorStoreIndex,
+  VectorStoreQuery,
+  VectorStoreQueryResult,
+} from "llamaindex";
+
+import { Index, Pinecone, RecordMetadata } from "@pinecone-database/pinecone";
+
+/**
+ * Please do not use this class in production; it's only for demonstration purposes.
+ */
+class PineconeVectorStore<T extends RecordMetadata = RecordMetadata>
+  implements VectorStore
+{
+  storesText = true;
+  isEmbeddingQuery = false;
+
+  indexName!: string;
+  pineconeClient!: Pinecone;
+  index!: Index<T>;
+
+  constructor({ indexName, client }: { indexName: string; client: Pinecone }) {
+    this.indexName = indexName;
+    this.pineconeClient = client;
+    this.index = client.index<T>(indexName);
+  }
+
+  client() {
+    return this.pineconeClient;
+  }
+
+  async query(
+    query: VectorStoreQuery,
+    kwargs?: any,
+  ): Promise<VectorStoreQueryResult> {
+    let queryEmbedding: number[] = [];
+    if (query.queryEmbedding) {
+      if (typeof query.alpha === "number") {
+        const alpha = query.alpha;
+        queryEmbedding = query.queryEmbedding.map((v) => v * alpha);
+      } else {
+        queryEmbedding = query.queryEmbedding;
+      }
+    }
+
+    // Current LlamaIndexTS implementation only support exact match filter, so we use kwargs instead.
+    const filter = kwargs?.filter || {};
+
+    const response = await this.index.query({
+      filter,
+      vector: queryEmbedding,
+      topK: query.similarityTopK,
+      includeValues: true,
+      includeMetadata: true,
+    });
+
+    console.log(
+      `Numbers of vectors returned by Pinecone after preFilters are applied: ${
+        response?.matches?.length || 0
+      }.`,
+    );
+
+    const topKIds: string[] = [];
+    const topKNodes: TextNode[] = [];
+    const topKScores: number[] = [];
+
+    const metadataToNode = (metadata?: T): Partial<TextNode> => {
+      if (!metadata) {
+        throw new Error("metadata is undefined.");
+      }
+
+      const nodeContent = metadata["_node_content"];
+      if (!nodeContent) {
+        throw new Error("nodeContent is undefined.");
+      }
+
+      if (typeof nodeContent !== "string") {
+        throw new Error("nodeContent is not a string.");
+      }
+
+      return JSON.parse(nodeContent);
+    };
+
+    if (response.matches) {
+      for (const match of response.matches) {
+        const node = new TextNode({
+          ...metadataToNode(match.metadata),
+          embedding: match.values,
+        });
+
+        topKIds.push(match.id);
+        topKNodes.push(node);
+        topKScores.push(match.score ?? 0);
+      }
+    }
+
+    const result = {
+      ids: topKIds,
+      nodes: topKNodes,
+      similarities: topKScores,
+    };
+
+    return result;
+  }
+
+  add(): Promise<string[]> {
+    return Promise.resolve([]);
+  }
+
+  delete(): Promise<void> {
+    throw new Error("Method `delete` not implemented.");
+  }
+
+  persist(): Promise<void> {
+    throw new Error("Method `persist` not implemented.");
+  }
+}
+
+/**
+ * The goal of this example is to show how to use Pinecone as a vector store
+ * for LlamaIndexTS with(out) preFilters.
+ *
+ * It should not be used in production like that,
+ * as you might want to find a proper PineconeVectorStore implementation.
+ */
+async function main() {
+  process.env.PINECONE_API_KEY = "Your Pinecone API Key.";
+  process.env.PINECONE_ENVIRONMENT = "Your Pinecone Environment.";
+  process.env.PINECONE_PROJECT_ID = "Your Pinecone Project ID.";
+  process.env.PINECONE_INDEX_NAME = "Your Pinecone Index Name.";
+  process.env.OPENAI_API_KEY = "Your OpenAI API Key.";
+  process.env.OPENAI_API_ORGANIZATION = "Your OpenAI API Organization.";
+
+  const getPineconeVectorStore = async () => {
+    return new PineconeVectorStore({
+      indexName: process.env.PINECONE_INDEX_NAME || "index-name",
+      client: new Pinecone(),
+    });
+  };
+
+  const getServiceContext = () => {
+    const openAI = new OpenAI({
+      model: "gpt-4",
+      apiKey: process.env.OPENAI_API_KEY,
+    });
+
+    return serviceContextFromDefaults({
+      llm: openAI,
+    });
+  };
+
+  const getQueryEngine = async (filter: unknown) => {
+    const vectorStore = await getPineconeVectorStore();
+    const serviceContext = getServiceContext();
+
+    const vectorStoreIndex = await VectorStoreIndex.fromVectorStore(
+      vectorStore,
+      serviceContext,
+    );
+
+    const retriever = new VectorIndexRetriever({
+      index: vectorStoreIndex,
+      similarityTopK: 500,
+    });
+
+    const responseSynthesizer = new ResponseSynthesizer({
+      serviceContext,
+      responseBuilder: new TreeSummarize(serviceContext),
+    });
+
+    return new RetrieverQueryEngine(retriever, responseSynthesizer, {
+      filter,
+    });
+  };
+
+  // whatever is a key from your metadata
+  const queryEngine = await getQueryEngine({
+    whatever: {
+      $gte: 1,
+      $lte: 100,
+    },
+  });
+
+  const response = await queryEngine.query("How many results do you have?");
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);

examples/portkey.ts

@@ -0,0 +1,23 @@
+import { Portkey } from "llamaindex";
+
+(async () => {
+  const llms = [{
+
+  }]
+  const portkey = new Portkey({
+    mode: "single",
+    llms: [{
+      provider:"anyscale",
+      virtual_key:"anyscale-3b3c04",
+      model: "meta-llama/Llama-2-13b-chat-hf",
+      max_tokens: 2000
+    }]
+  });
+  const result = portkey.stream_chat([
+    { role: "system", content: "You are a helpful assistant." },
+    { role: "user", content: "Tell me a joke." }
+  ]);
+  for await (const res of result) {
+    process.stdout.write(res)
+  }
+})();

package.json

@@ -11,16 +11,16 @@
     "publish-snapshot": "turbo run build lint test && changeset version --snapshot && changeset publish"
   },
   "devDependencies": {
-    "@turbo/gen": "^1.10.13",
-    "@types/jest": "^29.5.4",
+    "@turbo/gen": "^1.10.15",
+    "@types/jest": "^29.5.5",
     "eslint": "^7.32.0",
     "eslint-config-custom": "workspace:*",
     "husky": "^8.0.3",
-    "jest": "^29.6.4",
+    "jest": "^29.7.0",
     "prettier": "^3.0.3",
     "prettier-plugin-organize-imports": "^3.2.3",
     "ts-jest": "^29.1.1",
-    "turbo": "^1.10.13"
+    "turbo": "^1.10.15"
   },
   "packageManager": "pnpm@7.15.0",
   "dependencies": {

packages/core/CHANGELOG.md

@@ -1,5 +1,29 @@
 # llamaindex
 
+## 0.0.29
+
+### Patch Changes
+
+- a52143b: Added DocxReader for Word documents (thanks @jayantasamaddar)
+- 1b7fd95: Updated OpenAI streaming (thanks @kkang2097)
+- 0db3f41: Migrated to Tiktoken lite, which hopefully fixes the Windows issue
+
+## 0.0.28
+
+### Patch Changes
+
+- 96bb657: Typesafe metadata (thanks @TomPenguin)
+- 96bb657: MongoReader (thanks @kkang2097)
+- 837854d: Make OutputParser less strict and add tests (Thanks @kkang2097)
+
+## 0.0.27
+
+### Patch Changes
+
+- 4a5591b: Chat History summarization (thanks @marcusschiesser)
+- 4a5591b: Notion database support (thanks @TomPenguin)
+- 4a5591b: KeywordIndex (thanks @swk777)
+
 ## 0.0.26
 
 ### Patch Changes

packages/core/package.json

@@ -1,34 +1,40 @@
 {
   "name": "llamaindex",
-  "version": "0.0.26",
+  "version": "0.0.29",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.6.2",
-    "@notionhq/client": "^2.2.12",
+    "@notionhq/client": "^2.2.13",
     "lodash": "^4.17.21",
+    "mammoth": "^1.6.0",
+    "md-utils-ts": "^2.0.0",
+    "mongodb": "^6.1.0",
     "notion-md-crawler": "^0.0.2",
-    "openai": "^4.3.1",
+    "openai": "^4.11.1",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
+    "portkey-ai": "^0.1.11",
     "rake-modified": "^1.0.8",
-    "replicate": "^0.16.1",
-    "tiktoken-node": "^0.0.6",
-    "uuid": "^9.0.0",
+    "replicate": "^0.20.0",
+    "tiktoken": "^1.0.10",
+    "uuid": "^9.0.1",
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
-    "@types/lodash": "^4.14.197",
-    "@types/node": "^18.17.12",
-    "@types/papaparse": "^5.3.8",
-    "@types/pdf-parse": "^1.1.1",
-    "@types/uuid": "^9.0.3",
+    "@types/lodash": "^4.14.199",
+    "@types/node": "^18.18.4",
+    "@types/papaparse": "^5.3.9",
+    "@types/pdf-parse": "^1.1.2",
+    "@types/uuid": "^9.0.5",
     "node-stdlib-browser": "^1.2.0",
-    "tsup": "^7.2.0"
+    "tsup": "^7.2.0",
+    "typescript": "^4.9.5"
   },
   "engines": {
     "node": ">=18.0.0"
   },
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
   "scripts": {
     "lint": "eslint .",
     "test": "jest",

packages/core/src/ChatEngine.ts

@@ -23,8 +23,16 @@ export interface ChatEngine {
    * Send message along with the class's current chat history to the LLM.
    * @param message
    * @param chatHistory optional chat history if you want to customize the chat history
+   * @param streaming optional streaming flag, which auto-sets the return value if True.
    */
-  chat(message: string, chatHistory?: ChatMessage[]): Promise<Response>;
+  chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(
+    message: string,
+    chatHistory?: ChatMessage[],
+    streaming?: T,
+  ): Promise<R>;
 
   /**
    * Resets the chat history so that it's empty.
@@ -44,13 +52,45 @@ export class SimpleChatEngine implements ChatEngine {
     this.llm = init?.llm ?? new OpenAI();
   }
 
-  async chat(message: string, chatHistory?: ChatMessage[]): Promise<Response> {
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(message: string, chatHistory?: ChatMessage[], streaming?: T): Promise<R> {
+    //Streaming option
+    if (streaming) {
+      return this.streamChat(message, chatHistory) as R;
+    }
+
+    //Non-streaming option
     chatHistory = chatHistory ?? this.chatHistory;
     chatHistory.push({ content: message, role: "user" });
-    const response = await this.llm.chat(chatHistory);
+    const response = await this.llm.chat(chatHistory, undefined);
     chatHistory.push(response.message);
     this.chatHistory = chatHistory;
-    return new Response(response.message.content);
+    return new Response(response.message.content) as R;
+  }
+
+  protected async *streamChat(
+    message: string,
+    chatHistory?: ChatMessage[],
+  ): AsyncGenerator<string, void, unknown> {
+    chatHistory = chatHistory ?? this.chatHistory;
+    chatHistory.push({ content: message, role: "user" });
+    const response_generator = await this.llm.chat(
+      chatHistory,
+      undefined,
+      true,
+    );
+
+    var accumulator: string = "";
+    for await (const part of response_generator) {
+      accumulator += part;
+      yield part;
+    }
+
+    chatHistory.push({ content: accumulator, role: "assistant" });
+    this.chatHistory = chatHistory;
+    return;
   }
 
   reset() {
@@ -99,10 +139,14 @@ export class CondenseQuestionChatEngine implements ChatEngine {
     );
   }
 
-  async chat(
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(
     message: string,
     chatHistory?: ChatMessage[] | undefined,
-  ): Promise<Response> {
+    streaming?: T,
+  ): Promise<R> {
     chatHistory = chatHistory ?? this.chatHistory;
 
     const condensedQuestion = (
@@ -114,7 +158,7 @@ export class CondenseQuestionChatEngine implements ChatEngine {
     chatHistory.push({ content: message, role: "user" });
     chatHistory.push({ content: response.response, role: "assistant" });
 
-    return response;
+    return response as R;
   }
 
   reset() {
@@ -129,13 +173,13 @@ export class CondenseQuestionChatEngine implements ChatEngine {
  */
 export class ContextChatEngine implements ChatEngine {
   retriever: BaseRetriever;
-  chatModel: OpenAI;
+  chatModel: LLM;
   chatHistory: ChatMessage[];
   contextSystemPrompt: ContextSystemPrompt;
 
   constructor(init: {
     retriever: BaseRetriever;
-    chatModel?: OpenAI;
+    chatModel?: LLM;
     chatHistory?: ChatMessage[];
     contextSystemPrompt?: ContextSystemPrompt;
   }) {
@@ -147,9 +191,21 @@ export class ContextChatEngine implements ChatEngine {
       init?.contextSystemPrompt ?? defaultContextSystemPrompt;
   }
 
-  async chat(message: string, chatHistory?: ChatMessage[] | undefined) {
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(
+    message: string,
+    chatHistory?: ChatMessage[] | undefined,
+    streaming?: T,
+  ): Promise<R> {
     chatHistory = chatHistory ?? this.chatHistory;
 
+    //Streaming option
+    if (streaming) {
+      return this.streamChat(message, chatHistory) as R;
+    }
+
     const parentEvent: Event = {
       id: uuidv4(),
       type: "wrapper",
@@ -182,7 +238,52 @@ export class ContextChatEngine implements ChatEngine {
     return new Response(
       response.message.content,
       sourceNodesWithScore.map((r) => r.node),
+    ) as R;
+  }
+
+  protected async *streamChat(
+    message: string,
+    chatHistory?: ChatMessage[] | undefined,
+  ): AsyncGenerator<string, void, unknown> {
+    chatHistory = chatHistory ?? this.chatHistory;
+
+    const parentEvent: Event = {
+      id: uuidv4(),
+      type: "wrapper",
+      tags: ["final"],
+    };
+    const sourceNodesWithScore = await this.retriever.retrieve(
+      message,
+      parentEvent,
     );
+
+    const systemMessage: ChatMessage = {
+      content: this.contextSystemPrompt({
+        context: sourceNodesWithScore
+          .map((r) => (r.node as TextNode).text)
+          .join("\n\n"),
+      }),
+      role: "system",
+    };
+
+    chatHistory.push({ content: message, role: "user" });
+
+    const response_stream = await this.chatModel.chat(
+      [systemMessage, ...chatHistory],
+      parentEvent,
+      true,
+    );
+    var accumulator: string = "";
+    for await (const part of response_stream) {
+      accumulator += part;
+      yield part;
+    }
+
+    chatHistory.push({ content: accumulator, role: "system" });
+
+    this.chatHistory = chatHistory;
+
+    return;
   }
 
   reset() {
@@ -203,11 +304,42 @@ export class HistoryChatEngine implements ChatEngine {
     this.llm = init?.llm ?? new OpenAI();
   }
 
-  async chat(message: string): Promise<Response> {
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
+  >(
+    message: string,
+    chatHistory?: ChatMessage[] | undefined,
+    streaming?: T,
+  ): Promise<R> {
+    //Streaming option
+    if (streaming) {
+      return this.streamChat(message, chatHistory) as R;
+    }
     this.chatHistory.addMessage({ content: message, role: "user" });
-    const response = await this.llm.chat(this.chatHistory.messages);
+    const response = await this.llm.chat(this.chatHistory.requestMessages);
     this.chatHistory.addMessage(response.message);
-    return new Response(response.message.content);
+    return new Response(response.message.content) as R;
+  }
+
+  protected async *streamChat(
+    message: string,
+    chatHistory?: ChatMessage[] | undefined,
+  ): AsyncGenerator<string, void, unknown> {
+    this.chatHistory.addMessage({ content: message, role: "user" });
+    const response_stream = await this.llm.chat(
+      this.chatHistory.requestMessages,
+      undefined,
+      true,
+    );
+
+    var accumulator = "";
+    for await (const part of response_stream) {
+      accumulator += part;
+      yield part;
+    }
+    this.chatHistory.addMessage({ content: accumulator, role: "user" });
+    return;
   }
 
   reset() {

packages/core/src/ChatHistory.ts

@@ -16,6 +16,11 @@ export interface ChatHistory {
    */
   addMessage(message: ChatMessage): Promise<void>;
 
+  /**
+   * Returns the messages that should be used as input to the LLM.
+   */
+  requestMessages: ChatMessage[];
+
   /**
    * Resets the chat history so that it's empty.
    */
@@ -28,45 +33,86 @@ export class SimpleChatHistory implements ChatHistory {
   constructor(init?: Partial<SimpleChatHistory>) {
     this.messages = init?.messages ?? [];
   }
-
   async addMessage(message: ChatMessage) {
     this.messages.push(message);
   }
 
+  get requestMessages() {
+    return this.messages;
+  }
+
   reset() {
     this.messages = [];
   }
 }
 
 export class SummaryChatHistory implements ChatHistory {
+  messagesToSummarize: number;
   messages: ChatMessage[];
   summaryPrompt: SummaryPrompt;
   llm: LLM;
 
   constructor(init?: Partial<SummaryChatHistory>) {
+    this.messagesToSummarize = init?.messagesToSummarize ?? 5;
     this.messages = init?.messages ?? [];
     this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
     this.llm = init?.llm ?? new OpenAI();
   }
 
   private async summarize() {
-    const chatHistoryStr = messagesToHistoryStr(this.messages);
+    // get all messages after the last summary message (including)
+    const chatHistoryStr = messagesToHistoryStr(
+      this.messages.slice(this.getLastSummaryIndex()),
+    );
 
     const response = await this.llm.complete(
       this.summaryPrompt({ context: chatHistoryStr }),
     );
 
-    this.messages = [{ content: response.message.content, role: "system" }];
+    this.messages.push({ content: response.message.content, role: "memory" });
   }
 
   async addMessage(message: ChatMessage) {
-    // TODO: check if summarization is necessary
-    // TBD what are good conditions, e.g. depending on the context length of the LLM?
-    // for now we just have a dummy implementation at always summarizes the messages
-    await this.summarize();
+    const lastSummaryIndex = this.getLastSummaryIndex();
+    // if there are more than or equal `messagesToSummarize` messages since the last summary, call summarize
+    if (
+      lastSummaryIndex !== -1 &&
+      this.messages.length - lastSummaryIndex - 1 >= this.messagesToSummarize
+    ) {
+      // TODO: define what are better conditions, e.g. depending on the context length of the LLM?
+      // for now we just summarize each `messagesToSummarize` messages
+      await this.summarize();
+    }
     this.messages.push(message);
   }
 
+  // Find last summary message
+  private getLastSummaryIndex() {
+    return this.messages
+      .slice()
+      .reverse()
+      .findIndex((message) => message.role === "memory");
+  }
+
+  get requestMessages() {
+    const lastSummaryIndex = this.getLastSummaryIndex();
+    // get array of all system messages
+    const systemMessages = this.messages.filter(
+      (message) => message.role === "system",
+    );
+    // convert summary message so it can be send to the LLM
+    const summaryMessage: ChatMessage = {
+      content: `This is a summary of conversation so far: ${this.messages[lastSummaryIndex].content}`,
+      role: "system",
+    };
+    // return system messages, last summary and all messages after the last summary message
+    return [
+      ...systemMessages,
+      summaryMessage,
+      ...this.messages.slice(lastSummaryIndex + 1),
+    ];
+  }
+
   reset() {
     this.messages = [];
   }

packages/core/src/GlobalsHelper.ts

@@ -1,3 +1,6 @@
+import cl100k_base from "tiktoken/encoders/cl100k_base.json";
+import { Tiktoken } from "tiktoken/lite";
+
 import { v4 as uuidv4 } from "uuid";
 import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
 
@@ -6,14 +9,30 @@ import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
  */
 class GlobalsHelper {
   defaultTokenizer: {
-    encode: (text: string) => number[];
-    decode: (tokens: number[]) => string;
+    encode: (text: string) => Uint32Array;
+    decode: (tokens: Uint32Array) => string;
   } | null = null;
 
+  private initDefaultTokenizer() {
+    const encoding = new Tiktoken(
+      cl100k_base.bpe_ranks,
+      cl100k_base.special_tokens,
+      cl100k_base.pat_str,
+    );
+
+    this.defaultTokenizer = {
+      encode: (text: string) => {
+        return encoding.encode(text);
+      },
+      decode: (tokens: Uint32Array) => {
+        return new TextDecoder().decode(encoding.decode(tokens));
+      },
+    };
+  }
+
   tokenizer() {
     if (!this.defaultTokenizer) {
-      const tiktoken = require("tiktoken-node");
-      this.defaultTokenizer = tiktoken.getEncoding("gpt2");
+      this.initDefaultTokenizer();
     }
 
     return this.defaultTokenizer!.encode.bind(this.defaultTokenizer);
@@ -21,8 +40,7 @@ class GlobalsHelper {
 
   tokenizerDecoder() {
     if (!this.defaultTokenizer) {
-      const tiktoken = require("tiktoken-node");
-      this.defaultTokenizer = tiktoken.getEncoding("gpt2");
+      this.initDefaultTokenizer();
     }
 
     return this.defaultTokenizer!.decode.bind(this.defaultTokenizer);

packages/core/src/Node.ts

@@ -23,19 +23,23 @@ export enum MetadataMode {
   NONE = "NONE",
 }
 
-export interface RelatedNodeInfo {
+export type Metadata = Record<string, any>;
+
+export interface RelatedNodeInfo<T extends Metadata = Metadata> {
   nodeId: string;
   nodeType?: ObjectType;
-  metadata: Record<string, any>;
+  metadata: T;
   hash?: string;
 }
 
-export type RelatedNodeType = RelatedNodeInfo | RelatedNodeInfo[];
+export type RelatedNodeType<T extends Metadata = Metadata> =
+  | RelatedNodeInfo<T>
+  | RelatedNodeInfo<T>[];
 
 /**
  * Generic abstract class for retrievable nodes
  */
-export abstract class BaseNode {
+export abstract class BaseNode<T extends Metadata = Metadata> {
   /**
    * The unique ID of the Node/Document. The trailing underscore is here
    * to avoid collisions with the id keyword in Python.
@@ -46,13 +50,13 @@ export abstract class BaseNode {
   embedding?: number[];
 
   // Metadata fields
-  metadata: Record<string, any> = {};
+  metadata: T = {} as T;
   excludedEmbedMetadataKeys: string[] = [];
   excludedLlmMetadataKeys: string[] = [];
-  relationships: Partial<Record<NodeRelationship, RelatedNodeType>> = {};
+  relationships: Partial<Record<NodeRelationship, RelatedNodeType<T>>> = {};
   hash: string = "";
 
-  constructor(init?: Partial<BaseNode>) {
+  constructor(init?: Partial<BaseNode<T>>) {
     Object.assign(this, init);
   }
 
@@ -62,7 +66,7 @@ export abstract class BaseNode {
   abstract getMetadataStr(metadataMode: MetadataMode): string;
   abstract setContent(value: any): void;
 
-  get sourceNode(): RelatedNodeInfo | undefined {
+  get sourceNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.SOURCE];
 
     if (Array.isArray(relationship)) {
@@ -72,7 +76,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get prevNode(): RelatedNodeInfo | undefined {
+  get prevNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.PREVIOUS];
 
     if (Array.isArray(relationship)) {
@@ -84,7 +88,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get nextNode(): RelatedNodeInfo | undefined {
+  get nextNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.NEXT];
 
     if (Array.isArray(relationship)) {
@@ -94,7 +98,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get parentNode(): RelatedNodeInfo | undefined {
+  get parentNode(): RelatedNodeInfo<T> | undefined {
     const relationship = this.relationships[NodeRelationship.PARENT];
 
     if (Array.isArray(relationship)) {
@@ -104,7 +108,7 @@ export abstract class BaseNode {
     return relationship;
   }
 
-  get childNodes(): RelatedNodeInfo[] | undefined {
+  get childNodes(): RelatedNodeInfo<T>[] | undefined {
     const relationship = this.relationships[NodeRelationship.CHILD];
 
     if (!Array.isArray(relationship)) {
@@ -126,7 +130,7 @@ export abstract class BaseNode {
     return this.embedding;
   }
 
-  asRelatedNodeInfo(): RelatedNodeInfo {
+  asRelatedNodeInfo(): RelatedNodeInfo<T> {
     return {
       nodeId: this.id_,
       metadata: this.metadata,
@@ -146,7 +150,7 @@ export abstract class BaseNode {
 /**
  * TextNode is the default node type for text. Most common node type in LlamaIndex.TS
  */
-export class TextNode extends BaseNode {
+export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
   text: string = "";
   startCharIdx?: number;
   endCharIdx?: number;
@@ -154,7 +158,7 @@ export class TextNode extends BaseNode {
   // metadataTemplate: NOTE write your own formatter if needed
   metadataSeparator: string = "\n";
 
-  constructor(init?: Partial<TextNode>) {
+  constructor(init?: Partial<TextNode<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -233,10 +237,10 @@ export class TextNode extends BaseNode {
 //   }
 // }
 
-export class IndexNode extends TextNode {
+export class IndexNode<T extends Metadata = Metadata> extends TextNode<T> {
   indexId: string = "";
 
-  constructor(init?: Partial<IndexNode>) {
+  constructor(init?: Partial<IndexNode<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -253,8 +257,8 @@ export class IndexNode extends TextNode {
 /**
  * A document is just a special text node with a docId.
  */
-export class Document extends TextNode {
-  constructor(init?: Partial<Document>) {
+export class Document<T extends Metadata = Metadata> extends TextNode<T> {
+  constructor(init?: Partial<Document<T>>) {
     super(init);
     Object.assign(this, init);
 
@@ -292,7 +296,7 @@ export function jsonToNode(json: any) {
 /**
  * A node with a similarity score
  */
-export interface NodeWithScore {
-  node: BaseNode;
+export interface NodeWithScore<T extends Metadata = Metadata> {
+  node: BaseNode<T>;
   score?: number;
 }

packages/core/src/OutputParser.ts

@@ -53,30 +53,31 @@ class OutputParserError extends Error {
  * @param text A markdown block with JSON
  * @returns parsed JSON object
  */
-function parseJsonMarkdown(text: string) {
+export function parseJsonMarkdown(text: string) {
   text = text.trim();
 
-  const beginDelimiter = "```json";
-  const endDelimiter = "```";
+  const left_square = text.indexOf("[");
+  const left_brace = text.indexOf("{");
 
-  const beginIndex = text.indexOf(beginDelimiter);
-  const endIndex = text.indexOf(
-    endDelimiter,
-    beginIndex + beginDelimiter.length,
-  );
-  if (beginIndex === -1 || endIndex === -1) {
-    throw new OutputParserError("Not a json markdown", { output: text });
+  var left: number;
+  var right: number;
+  if (left_square < left_brace && left_square != -1) {
+    left = left_square;
+    right = text.lastIndexOf("]");
+  } else {
+    left = left_brace;
+    right = text.lastIndexOf("}");
   }
-
-  const jsonText = text.substring(beginIndex + beginDelimiter.length, endIndex);
-
+  const jsonText = text.substring(left, right + 1);
   try {
+    //Single JSON object case
+    if (left_square === -1) {
+      return [JSON.parse(jsonText)];
+    }
+    //Multiple JSON object case.
     return JSON.parse(jsonText);
   } catch (e) {
-    throw new OutputParserError("Not a valid json", {
-      cause: e as Error,
-      output: text,
-    });
+    throw new OutputParserError("Not a json markdown", { output: text });
   }
 }
 

packages/core/src/PromptHelper.ts

@@ -34,7 +34,7 @@ export class PromptHelper {
   numOutput = DEFAULT_NUM_OUTPUTS;
   chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO;
   chunkSizeLimit?: number;
-  tokenizer: (text: string) => number[];
+  tokenizer: (text: string) => Uint32Array;
   separator = " ";
 
   constructor(
@@ -42,7 +42,7 @@ export class PromptHelper {
     numOutput = DEFAULT_NUM_OUTPUTS,
     chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO,
     chunkSizeLimit?: number,
-    tokenizer?: (text: string) => number[],
+    tokenizer?: (text: string) => Uint32Array,
     separator = " ",
   ) {
     this.contextWindow = contextWindow;

packages/core/src/QueryEngine.ts

@@ -1,4 +1,5 @@
 import { v4 as uuidv4 } from "uuid";
+import { Event } from "./callbacks/CallbackManager";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   BaseQuestionGenerator,
@@ -10,7 +11,6 @@ import { CompactAndRefine, ResponseSynthesizer } from "./ResponseSynthesizer";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { QueryEngineTool, ToolMetadata } from "./Tool";
-import { Event } from "./callbacks/CallbackManager";
 
 /**
  * A query engine is a question answerer that can use one or more steps.
@@ -30,16 +30,19 @@ export interface BaseQueryEngine {
 export class RetrieverQueryEngine implements BaseQueryEngine {
   retriever: BaseRetriever;
   responseSynthesizer: ResponseSynthesizer;
+  preFilters?: unknown;
 
   constructor(
     retriever: BaseRetriever,
     responseSynthesizer?: ResponseSynthesizer,
+    preFilters?: unknown,
   ) {
     this.retriever = retriever;
     const serviceContext: ServiceContext | undefined =
       this.retriever.getServiceContext();
     this.responseSynthesizer =
       responseSynthesizer || new ResponseSynthesizer({ serviceContext });
+    this.preFilters = preFilters;
   }
 
   async query(query: string, parentEvent?: Event) {
@@ -48,7 +51,11 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
       type: "wrapper",
       tags: ["final"],
     };
-    const nodes = await this.retriever.retrieve(query, _parentEvent);
+    const nodes = await this.retriever.retrieve(
+      query,
+      _parentEvent,
+      this.preFilters,
+    );
     return this.responseSynthesizer.synthesize(query, nodes, _parentEvent);
   }
 }

packages/core/src/ResponseSynthesizer.ts

@@ -1,18 +1,18 @@
+import { Event } from "./callbacks/CallbackManager";
+import { LLM } from "./llm/LLM";
 import { MetadataMode, NodeWithScore } from "./Node";
 import {
+  defaultRefinePrompt,
+  defaultTextQaPrompt,
+  defaultTreeSummarizePrompt,
   RefinePrompt,
   SimplePrompt,
   TextQaPrompt,
   TreeSummarizePrompt,
-  defaultRefinePrompt,
-  defaultTextQaPrompt,
-  defaultTreeSummarizePrompt,
 } from "./Prompt";
 import { getBiggestPrompt } from "./PromptHelper";
 import { Response } from "./Response";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { Event } from "./callbacks/CallbackManager";
-import { LLM } from "./llm/LLM";
 
 /**
  * Response modes of the response synthesizer
@@ -231,6 +231,7 @@ export class TreeSummarize implements BaseResponseBuilder {
       throw new Error("Must have at least one text chunk");
     }
 
+    // Should we send the query here too?
     const packedTextChunks = this.serviceContext.promptHelper.repack(
       this.summaryTemplate,
       textChunks,
@@ -241,6 +242,7 @@ export class TreeSummarize implements BaseResponseBuilder {
         await this.serviceContext.llm.complete(
           this.summaryTemplate({
             context: packedTextChunks[0],
+            query,
           }),
           parentEvent,
         )
@@ -251,6 +253,7 @@ export class TreeSummarize implements BaseResponseBuilder {
           this.serviceContext.llm.complete(
             this.summaryTemplate({
               context: chunk,
+              query,
             }),
             parentEvent,
           ),

packages/core/src/Retriever.ts

@@ -1,11 +1,15 @@
+import { Event } from "./callbacks/CallbackManager";
 import { NodeWithScore } from "./Node";
 import { ServiceContext } from "./ServiceContext";
-import { Event } from "./callbacks/CallbackManager";
 
 /**
  * Retrievers retrieve the nodes that most closely match our query in similarity.
  */
 export interface BaseRetriever {
-  retrieve(query: string, parentEvent?: Event): Promise<NodeWithScore[]>;
+  retrieve(
+    query: string,
+    parentEvent?: Event,
+    preFilters?: unknown,
+  ): Promise<NodeWithScore[]>;
   getServiceContext(): ServiceContext;
 }

packages/core/src/callbacks/CallbackManager.ts

@@ -20,7 +20,8 @@ interface BaseCallbackResponse {
   event: Event;
 }
 
-export interface StreamToken {
+//Specify StreamToken per mainstream LLM
+export interface DefaultStreamToken {
   id: string;
   object: string;
   created: number;
@@ -35,10 +36,28 @@ export interface StreamToken {
   }[];
 }
 
+//OpenAI stream token schema is the default.
+//Note: Anthropic and Replicate also use similar token schemas.
+export type OpenAIStreamToken = DefaultStreamToken;
+export type AnthropicStreamToken = {
+  completion: string;
+  model: string;
+  stop_reason: string | undefined;
+  stop?: boolean | undefined;
+  log_id?: string;
+};
+
+//
+//Callback Responses
+//
+//TODO: Write Embedding Callbacks
+
+//StreamCallbackResponse should let practitioners implement callbacks out of the box...
+//When custom streaming LLMs are involved, people are expected to write their own StreamCallbackResponses
 export interface StreamCallbackResponse extends BaseCallbackResponse {
   index: number;
   isDone?: boolean;
-  token?: StreamToken;
+  token?: DefaultStreamToken;
 }
 
 export interface RetrievalCallbackResponse extends BaseCallbackResponse {

packages/core/src/callbacks/utility/handleOpenAIStream.ts

@@ -1,45 +0,0 @@
-import { ChatCompletionChunk } from "openai/resources/chat";
-import { Stream } from "openai/streaming";
-import { globalsHelper } from "../../GlobalsHelper";
-import { MessageType } from "../../llm/LLM";
-import { Event, StreamCallbackResponse } from "../CallbackManager";
-
-/**
- * Handles the OpenAI streaming interface and pipes it to the callback function
- * @param response - The response from the OpenAI API.
- * @param onLLMStream - A callback function to handle the LLM stream.
- * @param parentEvent - An optional parent event.
- * @returns A promise that resolves to an object with a message and a role.
- */
-export async function handleOpenAIStream({
-  response,
-  onLLMStream,
-  parentEvent,
-}: {
-  response: Stream<ChatCompletionChunk>;
-  onLLMStream: (data: StreamCallbackResponse) => void;
-  parentEvent?: Event;
-}): Promise<{ message: string; role: MessageType }> {
-  const event = globalsHelper.createEvent({
-    parentEvent,
-    type: "llmPredict",
-  });
-  let index = 0;
-  let cumulativeText = "";
-  let messageRole: MessageType = "assistant";
-  for await (const part of response) {
-    const { content = "", role = "assistant" } = part.choices[0].delta;
-
-    // ignore the first token
-    if (!content && role === "assistant" && index === 0) {
-      continue;
-    }
-
-    cumulativeText += content;
-    messageRole = role;
-    onLLMStream?.({ event, index, token: part });
-    index++;
-  }
-  onLLMStream?.({ event, index, isDone: true });
-  return { message: cumulativeText, role: messageRole };
-}

packages/core/src/index.ts

@@ -1,30 +1,27 @@
+export * from "./callbacks/CallbackManager";
 export * from "./ChatEngine";
+export * from "./constants";
 export * from "./Embedding";
 export * from "./GlobalsHelper";
+export * from "./indices";
+export * from "./llm/LLM";
 export * from "./Node";
 export * from "./NodeParser";
 export * from "./OutputParser";
 export * from "./Prompt";
+export * from "./PromptHelper";
 export * from "./QueryEngine";
 export * from "./QuestionGenerator";
-export * from "./Response";
-export * from "./ResponseSynthesizer";
-export * from "./Retriever";
-export * from "./ServiceContext";
-export * from "./TextSplitter";
-export * from "./Tool";
-export * from "./constants";
-export * from "./llm/LLM";
-
-export * from "./indices";
-
-export * from "./callbacks/CallbackManager";
-
+export * from "./readers/base";
 export * from "./readers/CSVReader";
 export * from "./readers/MarkdownReader";
 export * from "./readers/NotionReader";
 export * from "./readers/PDFReader";
 export * from "./readers/SimpleDirectoryReader";
-export * from "./readers/base";
-
+export * from "./Response";
+export * from "./ResponseSynthesizer";
+export * from "./Retriever";
+export * from "./ServiceContext";
 export * from "./storage";
+export * from "./TextSplitter";
+export * from "./Tool";

packages/core/src/indices/vectorStore/VectorIndexRetriever.ts

@@ -1,9 +1,9 @@
+import { Event } from "../../callbacks/CallbackManager";
+import { DEFAULT_SIMILARITY_TOP_K } from "../../constants";
 import { globalsHelper } from "../../GlobalsHelper";
 import { NodeWithScore } from "../../Node";
 import { BaseRetriever } from "../../Retriever";
 import { ServiceContext } from "../../ServiceContext";
-import { Event } from "../../callbacks/CallbackManager";
-import { DEFAULT_SIMILARITY_TOP_K } from "../../constants";
 import {
   VectorStoreQuery,
   VectorStoreQueryMode,
@@ -32,7 +32,7 @@ export class VectorIndexRetriever implements BaseRetriever {
     this.similarityTopK = similarityTopK ?? DEFAULT_SIMILARITY_TOP_K;
   }
 
-  async retrieve(query: string, parentEvent?: Event): Promise<NodeWithScore[]> {
+  async retrieve(query: string, parentEvent?: Event, preFilters?: unknown): Promise<NodeWithScore[]> {
     const queryEmbedding =
       await this.serviceContext.embedModel.getQueryEmbedding(query);
 
@@ -41,10 +41,15 @@ export class VectorIndexRetriever implements BaseRetriever {
       mode: VectorStoreQueryMode.DEFAULT,
       similarityTopK: this.similarityTopK,
     };
-    const result = await this.index.vectorStore.query(q);
+    const result = await this.index.vectorStore.query(q, preFilters);
 
     let nodesWithScores: NodeWithScore[] = [];
     for (let i = 0; i < result.ids.length; i++) {
+      const nodeFromResult = result.nodes?.[i];
+      if (!this.index.indexStruct.nodesDict[result.ids[i]] && nodeFromResult) {
+        this.index.indexStruct.nodesDict[result.ids[i]] = nodeFromResult;
+      }
+
       const node = this.index.indexStruct.nodesDict[result.ids[i]];
       nodesWithScores.push({
         node: node,

packages/core/src/indices/vectorStore/VectorStoreIndex.ts

@@ -219,6 +219,27 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
     return index;
   }
 
+  static async fromVectorStore(
+    vectorStore: VectorStore,
+    serviceContext: ServiceContext,
+  ) {
+    if (!vectorStore.storesText) {
+      throw new Error(
+        "Cannot initialize from a vector store that does not store text",
+      );
+    }
+
+    const storageContext = await storageContextFromDefaults({ vectorStore });
+
+    const index = await VectorStoreIndex.init({
+      nodes: [],
+      storageContext,
+      serviceContext,
+    });
+
+    return index;
+  }
+
   asRetriever(options?: any): VectorIndexRetriever {
     return new VectorIndexRetriever({ index: this, ...options });
   }

packages/core/src/llm/LLM.ts

@@ -1,10 +1,18 @@
 import OpenAILLM, { ClientOptions as OpenAIClientOptions } from "openai";
-import { CallbackManager, Event } from "../callbacks/CallbackManager";
-import { handleOpenAIStream } from "../callbacks/utility/handleOpenAIStream";
 import {
+  AnthropicStreamToken,
+  CallbackManager,
+  Event,
+  EventType,
+  OpenAIStreamToken,
+  StreamCallbackResponse,
+} from "../callbacks/CallbackManager";
+
+import { LLMOptions } from "portkey-ai";
+import {
+  AnthropicSession,
   ANTHROPIC_AI_PROMPT,
   ANTHROPIC_HUMAN_PROMPT,
-  AnthropicSession,
   getAnthropicSession,
 } from "./anthropic";
 import {
@@ -14,7 +22,8 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure";
-import { OpenAISession, getOpenAISession } from "./openai";
+import { getOpenAISession, OpenAISession } from "./openai";
+import { getPortkeySession, PortkeySession } from "./portkey";
 import { ReplicateSession } from "./replicate";
 
 export type MessageType =
@@ -22,7 +31,8 @@ export type MessageType =
   | "assistant"
   | "system"
   | "generic"
-  | "function";
+  | "function"
+  | "memory";
 
 export interface ChatMessage {
   content: string;
@@ -42,17 +52,35 @@ export type CompletionResponse = ChatResponse;
  * Unified language model interface
  */
 export interface LLM {
+  // Whether a LLM has streaming support
+  hasStreaming: boolean;
   /**
    * Get a chat response from the LLM
    * @param messages
+   *
+   * The return type of chat() and complete() are set by the "streaming" parameter being set to True.
    */
-  chat(messages: ChatMessage[], parentEvent?: Event): Promise<ChatResponse>;
+  chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
+    messages: ChatMessage[],
+    parentEvent?: Event,
+    streaming?: T,
+  ): Promise<R>;
 
   /**
    * Get a prompt completion from the LLM
    * @param prompt the prompt to complete
    */
-  complete(prompt: string, parentEvent?: Event): Promise<CompletionResponse>;
+  complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
+    prompt: string,
+    parentEvent?: Event,
+    streaming?: T,
+  ): Promise<R>;
 }
 
 export const GPT4_MODELS = {
@@ -77,13 +105,15 @@ export const ALL_AVAILABLE_OPENAI_MODELS = {
  * OpenAI LLM implementation
  */
 export class OpenAI implements LLM {
+  hasStreaming: boolean = true;
+
   // Per completion OpenAI params
   model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS;
   temperature: number;
   topP: number;
   maxTokens?: number;
   additionalChatOptions?: Omit<
-    Partial<OpenAILLM.Chat.CompletionCreateParams>,
+    Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
     "max_tokens" | "messages" | "model" | "temperature" | "top_p" | "streaming"
   >;
 
@@ -170,11 +200,59 @@ export class OpenAI implements LLM {
     }
   }
 
-  async chat(
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(messages: ChatMessage[], parentEvent?: Event, streaming?: T): Promise<R> {
+    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
+      model: this.model,
+      temperature: this.temperature,
+      max_tokens: this.maxTokens,
+      messages: messages.map((message) => ({
+        role: this.mapMessageType(message.role),
+        content: message.content,
+      })),
+      top_p: this.topP,
+      ...this.additionalChatOptions,
+    };
+    // Streaming
+    if (streaming) {
+      if (!this.hasStreaming) {
+        throw Error("No streaming support for this LLM.");
+      }
+      return this.streamChat(messages, parentEvent) as R;
+    }
+    // Non-streaming
+    const response = await this.session.openai.chat.completions.create({
+      ...baseRequestParams,
+      stream: false,
+    });
+
+    const content = response.choices[0].message?.content ?? "";
+    return {
+      message: { content, role: response.choices[0].message.role },
+    } as R;
+  }
+
+  async complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
+    return this.chat(
+      [{ content: prompt, role: "user" }],
+      parentEvent,
+      streaming,
+    );
+  }
+
+  //We can wrap a stream in a generator to add some additional logging behavior
+  //For future edits: syntax for generator type is <typeof Yield, typeof Return, typeof Accept>
+  //"typeof Accept" refers to what types you'll accept when you manually call generator.next(<AcceptType>)
+  protected async *streamChat(
     messages: ChatMessage[],
     parentEvent?: Event,
-  ): Promise<ChatResponse> {
-    const baseRequestParams: OpenAILLM.Chat.CompletionCreateParams = {
+  ): AsyncGenerator<string, void, unknown> {
+    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
       model: this.model,
       temperature: this.temperature,
       max_tokens: this.maxTokens,
@@ -186,36 +264,54 @@ export class OpenAI implements LLM {
       ...this.additionalChatOptions,
     };
 
-    if (this.callbackManager?.onLLMStream) {
-      // Streaming
-      const response = await this.session.openai.chat.completions.create({
+    //Now let's wrap our stream in a callback
+    const onLLMStream = this.callbackManager?.onLLMStream
+      ? this.callbackManager.onLLMStream
+      : () => {};
+
+    const chunk_stream: AsyncIterable<OpenAIStreamToken> =
+      await this.session.openai.chat.completions.create({
         ...baseRequestParams,
         stream: true,
       });
 
-      const { message, role } = await handleOpenAIStream({
-        response,
-        onLLMStream: this.callbackManager.onLLMStream,
-        parentEvent,
-      });
-      return { message: { content: message, role } };
-    } else {
-      // Non-streaming
-      const response = await this.session.openai.chat.completions.create({
-        ...baseRequestParams,
-        stream: false,
-      });
+    const event: Event = parentEvent
+      ? parentEvent
+      : {
+          id: "unspecified",
+          type: "llmPredict" as EventType,
+        };
 
-      const content = response.choices[0].message?.content ?? "";
-      return { message: { content, role: response.choices[0].message.role } };
+    //Indices
+    var idx_counter: number = 0;
+    for await (const part of chunk_stream) {
+      //Increment
+      part.choices[0].index = idx_counter;
+      const is_done: boolean =
+        part.choices[0].finish_reason === "stop" ? true : false;
+      //onLLMStream Callback
+
+      const stream_callback: StreamCallbackResponse = {
+        event: event,
+        index: idx_counter,
+        isDone: is_done,
+        token: part,
+      };
+      onLLMStream(stream_callback);
+
+      idx_counter++;
+
+      yield part.choices[0].delta.content ? part.choices[0].delta.content : "";
     }
+    return;
   }
 
-  async complete(
-    prompt: string,
+  //streamComplete doesn't need to be async because it's child function is already async
+  protected streamComplete(
+    query: string,
     parentEvent?: Event,
-  ): Promise<CompletionResponse> {
-    return this.chat([{ content: prompt, role: "user" }], parentEvent);
+  ): AsyncGenerator<string, void, unknown> {
+    return this.streamChat([{ content: query, role: "user" }], parentEvent);
   }
 }
 
@@ -279,6 +375,7 @@ export class LlamaDeuce implements LLM {
   topP: number;
   maxTokens?: number;
   replicateSession: ReplicateSession;
+  hasStreaming: boolean;
 
   constructor(init?: Partial<LlamaDeuce>) {
     this.model = init?.model ?? "Llama-2-70b-chat-4bit";
@@ -293,6 +390,7 @@ export class LlamaDeuce implements LLM {
       init?.maxTokens ??
       ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model].contextWindow; // For Replicate, the default is 500 tokens which is too low.
     this.replicateSession = init?.replicateSession ?? new ReplicateSession();
+    this.hasStreaming = init?.hasStreaming ?? false;
   }
 
   mapMessagesToPrompt(messages: ChatMessage[]) {
@@ -399,10 +497,10 @@ If a question does not make any sense, or is not factually coherent, explain why
     };
   }
 
-  async chat(
-    messages: ChatMessage[],
-    _parentEvent?: Event,
-  ): Promise<ChatResponse> {
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(messages: ChatMessage[], _parentEvent?: Event, streaming?: T): Promise<R> {
     const api = ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model]
       .replicateApi as `${string}/${string}:${string}`;
 
@@ -423,6 +521,9 @@ If a question does not make any sense, or is not factually coherent, explain why
       replicateOptions.input.max_length = this.maxTokens;
     }
 
+    //TODO: Add streaming for this
+
+    //Non-streaming
     const response = await this.replicateSession.replicate.run(
       api,
       replicateOptions,
@@ -433,13 +534,13 @@ If a question does not make any sense, or is not factually coherent, explain why
         //^ We need to do this because Replicate returns a list of strings (for streaming functionality which is not exposed by the run function)
         role: "assistant",
       },
-    };
+    } as R;
   }
 
-  async complete(
-    prompt: string,
-    parentEvent?: Event,
-  ): Promise<CompletionResponse> {
+  async complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
     return this.chat([{ content: prompt, role: "user" }], parentEvent);
   }
 }
@@ -449,6 +550,8 @@ If a question does not make any sense, or is not factually coherent, explain why
  */
 
 export class Anthropic implements LLM {
+  hasStreaming: boolean = true;
+
   // Per completion Anthropic params
   model: string;
   temperature: number;
@@ -498,10 +601,22 @@ export class Anthropic implements LLM {
     );
   }
 
-  async chat(
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
     messages: ChatMessage[],
     parentEvent?: Event | undefined,
-  ): Promise<ChatResponse> {
+    streaming?: T,
+  ): Promise<R> {
+    //Streaming
+    if (streaming) {
+      if (!this.hasStreaming) {
+        throw Error("No streaming support for this LLM.");
+      }
+      return this.streamChat(messages, parentEvent) as R;
+    }
+    //Non-streaming
     const response = await this.session.anthropic.completions.create({
       model: this.model,
       prompt: this.mapMessagesToPrompt(messages),
@@ -514,12 +629,174 @@ export class Anthropic implements LLM {
       message: { content: response.completion.trimStart(), role: "assistant" },
       //^ We're trimming the start because Anthropic often starts with a space in the response
       // That space will be re-added when we generate the next prompt.
-    };
+    } as R;
   }
-  async complete(
+
+  protected async *streamChat(
+    messages: ChatMessage[],
+    parentEvent?: Event | undefined,
+  ): AsyncGenerator<string, void, unknown> {
+    // AsyncIterable<AnthropicStreamToken>
+    const stream: AsyncIterable<AnthropicStreamToken> =
+      await this.session.anthropic.completions.create({
+        model: this.model,
+        prompt: this.mapMessagesToPrompt(messages),
+        max_tokens_to_sample: this.maxTokens ?? 100000,
+        temperature: this.temperature,
+        top_p: this.topP,
+        stream: true,
+      });
+
+    var idx_counter: number = 0;
+    for await (const part of stream) {
+      //TODO: LLM Stream Callback, pending re-work.
+
+      idx_counter++;
+      yield part.completion;
+    }
+    return;
+  }
+
+  async complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
     prompt: string,
     parentEvent?: Event | undefined,
-  ): Promise<CompletionResponse> {
-    return this.chat([{ content: prompt, role: "user" }], parentEvent);
+    streaming?: T,
+  ): Promise<R> {
+    if (streaming) {
+      return this.streamComplete(prompt, parentEvent) as R;
+    }
+    return this.chat(
+      [{ content: prompt, role: "user" }],
+      parentEvent,
+      streaming,
+    ) as R;
+  }
+
+  protected streamComplete(
+    prompt: string,
+    parentEvent?: Event | undefined,
+  ): AsyncGenerator<string, void, unknown> {
+    return this.streamChat([{ content: prompt, role: "user" }], parentEvent);
+  }
+}
+
+export class Portkey implements LLM {
+  hasStreaming: boolean = true;
+
+  apiKey?: string = undefined;
+  baseURL?: string = undefined;
+  mode?: string = undefined;
+  llms?: [LLMOptions] | null = undefined;
+  session: PortkeySession;
+  callbackManager?: CallbackManager;
+
+  constructor(init?: Partial<Portkey>) {
+    this.apiKey = init?.apiKey;
+    this.baseURL = init?.baseURL;
+    this.mode = init?.mode;
+    this.llms = init?.llms;
+    this.session = getPortkeySession({
+      apiKey: this.apiKey,
+      baseURL: this.baseURL,
+      llms: this.llms,
+      mode: this.mode,
+    });
+    this.callbackManager = init?.callbackManager;
+  }
+
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
+    messages: ChatMessage[],
+    parentEvent?: Event | undefined,
+    streaming?: T,
+    params?: Record<string, any>,
+  ): Promise<R> {
+    if (streaming) {
+      return this.streamChat(messages, parentEvent, params) as R;
+    } else {
+      const resolvedParams = params || {};
+      const response = await this.session.portkey.chatCompletions.create({
+        messages,
+        ...resolvedParams,
+      });
+
+      const content = response.choices[0].message?.content ?? "";
+      const role = response.choices[0].message?.role || "assistant";
+      return { message: { content, role: role as MessageType } } as R;
+    }
+  }
+
+  async complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
+    prompt: string,
+    parentEvent?: Event | undefined,
+    streaming?: T,
+  ): Promise<R> {
+    return this.chat(
+      [{ content: prompt, role: "user" }],
+      parentEvent,
+      streaming,
+    );
+  }
+
+  async *streamChat(
+    messages: ChatMessage[],
+    parentEvent?: Event,
+    params?: Record<string, any>,
+  ): AsyncGenerator<string, void, unknown> {
+    // Wrapping the stream in a callback.
+    const onLLMStream = this.callbackManager?.onLLMStream
+      ? this.callbackManager.onLLMStream
+      : () => {};
+
+    const chunkStream = await this.session.portkey.chatCompletions.create({
+      messages,
+      ...params,
+      stream: true,
+    });
+
+    const event: Event = parentEvent
+      ? parentEvent
+      : {
+          id: "unspecified",
+          type: "llmPredict" as EventType,
+        };
+
+    //Indices
+    var idx_counter: number = 0;
+    for await (const part of chunkStream) {
+      //Increment
+      part.choices[0].index = idx_counter;
+      const is_done: boolean =
+        part.choices[0].finish_reason === "stop" ? true : false;
+      //onLLMStream Callback
+
+      const stream_callback: StreamCallbackResponse = {
+        event: event,
+        index: idx_counter,
+        isDone: is_done,
+        // token: part,
+      };
+      onLLMStream(stream_callback);
+
+      idx_counter++;
+
+      yield part.choices[0].delta?.content ?? "";
+    }
+    return;
+  }
+
+  streamComplete(
+    query: string,
+    parentEvent?: Event,
+  ): AsyncGenerator<string, void, unknown> {
+    return this.streamChat([{ content: query, role: "user" }], parentEvent);
   }
 }

packages/core/src/llm/portkey.ts

@@ -0,0 +1,62 @@
+import _ from "lodash";
+import { LLMOptions, Portkey } from "portkey-ai";
+
+export const readEnv = (env: string, default_val?: string): string | undefined => {
+  if (typeof process !== 'undefined') {
+      return process.env?.[env] ?? default_val;
+  }
+  return default_val;
+};
+
+interface PortkeyOptions {
+  apiKey?: string;
+  baseURL?: string;
+  mode?: string;
+  llms?: [LLMOptions] | null
+}
+
+export class PortkeySession {
+  portkey: Portkey;
+
+  constructor(options:PortkeyOptions = {}) {
+    if (!options.apiKey) {
+      options.apiKey = readEnv('PORTKEY_API_KEY')
+    }
+
+    if (!options.baseURL) {
+      options.baseURL = readEnv('PORTKEY_BASE_URL', "https://api.portkey.ai")
+    }
+
+    this.portkey = new Portkey({});
+    this.portkey.llms = [{}]
+    if (!options.apiKey) {
+      throw new Error("Set Portkey ApiKey in PORTKEY_API_KEY env variable");
+    }
+
+    this.portkey = new Portkey(options);
+  }
+}
+
+let defaultPortkeySession: {
+  session: PortkeySession;
+  options: PortkeyOptions;
+}[] = [];
+
+/**
+ * Get a session for the Portkey API. If one already exists with the same options,
+ * it will be returned. Otherwise, a new session will be created.
+ * @param options
+ * @returns
+ */
+export function getPortkeySession(options: PortkeyOptions = {}) {
+  let session = defaultPortkeySession.find((session) => {
+    return _.isEqual(session.options, options);
+  })?.session;
+
+  if (!session) {
+    session = new PortkeySession(options);
+    defaultPortkeySession.push({ session, options });
+  }
+  return session;
+}
+

packages/core/src/readers/DocxReader.ts

@@ -0,0 +1,17 @@
+import mammoth from "mammoth";
+import { Document } from "../Node";
+import { DEFAULT_FS } from "../storage/constants";
+import { GenericFileSystem } from "../storage/FileSystem";
+import { BaseReader } from "./base";
+
+export class DocxReader implements BaseReader {
+  /** DocxParser */
+  async loadData(
+    file: string,
+    fs: GenericFileSystem = DEFAULT_FS,
+  ): Promise<Document[]> {
+    const dataBuffer = (await fs.readFile(file)) as any;
+    const { value } = await mammoth.extractRawText({ buffer: dataBuffer });
+    return [new Document({ text: value, id_: file })];
+  }
+}

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -1,11 +1,12 @@
 import _ from "lodash";
 import { Document } from "../Node";
-import { CompleteFileSystem, walk } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
+import { CompleteFileSystem, walk } from "../storage/FileSystem";
+import { BaseReader } from "./base";
 import { PapaCSVReader } from "./CSVReader";
+import { DocxReader } from "./DocxReader";
 import { MarkdownReader } from "./MarkdownReader";
 import { PDFReader } from "./PDFReader";
-import { BaseReader } from "./base";
 
 /**
  * Read a .txt file
@@ -25,6 +26,7 @@ const FILE_EXT_TO_READER: Record<string, BaseReader> = {
   pdf: new PDFReader(),
   csv: new PapaCSVReader(),
   md: new MarkdownReader(),
+  docx: new DocxReader(),
 };
 
 export type SimpleDirectoryReaderLoadDataProps = {

packages/core/src/readers/SimpleMongoReader.ts

@@ -0,0 +1,51 @@
+import { MongoClient } from "mongodb";
+import { Document } from "../Node";
+import { BaseReader } from "./base";
+
+/**
+ * Read in from MongoDB
+ */
+export class SimpleMongoReader implements BaseReader {
+  private client: MongoClient;
+
+  constructor(client: MongoClient) {
+    this.client = client;
+  }
+
+  /**
+   * Loads data from MongoDB collection
+   * @param {string} db_name - The name of the database to load.
+   * @param {string} collection_name - The name of the collection to load.
+   * @param {Number} [max_docs = 0] - Maximum number of documents to return. 0 means no limit.
+   * @param {Record<string, any>} [query_dict={}] - Specific query, as specified by MongoDB NodeJS documentation.
+   * @param {Record<string, any>} [query_options={}] - Specific query options, as specified by MongoDB NodeJS documentation.
+   * @param {Record<string, any>} [projection = {}] - Projection options, as specified by MongoDB NodeJS documentation.
+   * @returns {Promise<Document[]>}
+   */
+  async loadData(
+    db_name: string,
+    collection_name: string,
+    max_docs = 0,
+    //For later: Think about whether we want to pass generic objects in...
+    query_dict: Record<string, any> = {},
+    query_options: Record<string, any> = {},
+    projection: Record<string, any> = {},
+  ): Promise<Document[]> {
+    //Get items from collection using built-in functions
+    const cursor: Partial<Document>[] = await this.client
+      .db(db_name)
+      .collection(collection_name)
+      .find(query_dict, query_options)
+      .limit(max_docs)
+      .project(projection)
+      .toArray();
+
+    //Aggregate results and return
+    const documents: Document[] = [];
+    cursor.forEach((element: Partial<Document>) => {
+      //For later: Metadata filtering
+      documents.push(new Document({ text: JSON.stringify(element) }));
+    });
+    return documents;
+  }
+}

packages/core/src/storage/vectorStore/types.ts

@@ -63,6 +63,9 @@ export interface VectorStore {
   client(): any;
   add(embeddingResults: BaseNode[]): Promise<string[]>;
   delete(refDocId: string, deleteKwargs?: any): Promise<void>;
-  query(query: VectorStoreQuery, kwargs?: any): Promise<VectorStoreQueryResult>;
+  query(
+    query: VectorStoreQuery,
+    options?: any,
+  ): Promise<VectorStoreQueryResult>;
   persist(persistPath: string, fs?: GenericFileSystem): Promise<void>;
 }

packages/core/src/tests/OutputParser.test.ts

@@ -0,0 +1,90 @@
+import { SubQuestionOutputParser } from "../OutputParser";
+
+//This parser is really important, so make sure to add tests
+// as the parser sees through more iterations.
+describe("SubQuestionOutputParser", () => {
+  test("parses expected", () => {
+    const parser = new SubQuestionOutputParser();
+
+    const data = [
+      {
+        name: "uber_10k",
+        description: "Provides information about Uber financials for year 2021",
+      },
+      {
+        name: "lyft_10k",
+        description: "Provides information about Lyft financials for year 2021",
+      },
+    ];
+
+    const data_str: string = JSON.stringify(data);
+    const full_string = `\`\`\`json
+${data_str}
+\`\`\``;
+
+    const real_answer = { parsedOutput: data, rawOutput: full_string };
+
+    expect(parser.parse(full_string)).toEqual(real_answer);
+  });
+
+  //This is in case our LLM outputs a list response, but without ```json.
+  test("parses without ```json", () => {
+    const parser = new SubQuestionOutputParser();
+
+    const data = [
+      {
+        name: "uber_10k",
+        description: "Provides information about Uber financials for year 2021",
+      },
+      {
+        name: "lyft_10k",
+        description: "Provides information about Lyft financials for year 2021",
+      },
+    ];
+
+    const data_str: string = JSON.stringify(data);
+    const full_string = `${data_str}`;
+
+    const real_answer = { parsedOutput: data, rawOutput: full_string };
+
+    expect(parser.parse(JSON.stringify(data))).toEqual(real_answer);
+  });
+
+  test("parses null single response", () => {
+    const parser = new SubQuestionOutputParser();
+    const data_str =
+      "[\n" +
+      "    {\n" +
+      `        "subQuestion": "Sorry, I don't have any relevant information to answer your question",\n` +
+      '        "toolName": ""\n' +
+      "    }\n" +
+      "]";
+    const data = [
+      {
+        subQuestion:
+          "Sorry, I don't have any relevant information to answer your question",
+        toolName: "",
+      },
+    ];
+    const real_answer = { parsedOutput: data, rawOutput: data_str };
+    expect(parser.parse(data_str)).toEqual(real_answer);
+  });
+
+  test("Single JSON object case", () => {
+    const parser = new SubQuestionOutputParser();
+    const data_str =
+      "    {\n" +
+      `        "subQuestion": "Sorry, I don't have any relevant information to answer your question",\n` +
+      '        "toolName": ""\n' +
+      "    }\n";
+    const data = [
+      {
+        subQuestion:
+          "Sorry, I don't have any relevant information to answer your question",
+        toolName: "",
+      },
+    ];
+    const real_answer = { parsedOutput: data, rawOutput: data_str };
+    expect(parser.parse(data_str)).toEqual(real_answer);
+  });
+});

packages/core/tsconfig.json

@@ -6,10 +6,10 @@
     "moduleResolution": "node",
     "preserveWatchOutput": true,
     "skipLibCheck": true,
-    "noEmit": true,
     "strict": true,
     "lib": ["es2015", "dom"],
-    "target": "ES2015"
+    "target": "ES2015",
+    "resolveJsonModule": true
   },
   "exclude": ["node_modules"]
 }

packages/eslint-config-custom/index.js

@@ -18,6 +18,12 @@ module.exports = {
           "OPENAI_API_BASE",
           "OPENAI_API_VERSION",
           "OPENAI_API_TYPE",
+          "OPENAI_API_ORGANIZATION",
+
+          "PINECONE_API_KEY",
+          "PINECONE_ENVIRONMENT",
+          "PINECONE_PROJECT_ID",
+          "PINECONE_INDEX_NAME",
 
           "AZURE_OPENAI_API_KEY",
           "AZURE_OPENAI_API_INSTANCE_NAME",
@@ -28,6 +34,7 @@ module.exports = {
           "NO_PROXY",
 
           "NOTION_TOKEN",
+          "MONGODB_URI",
         ],
       },
     ],

sweep.yaml

@@ -7,5 +7,17 @@ branch: "main"
 gha_enabled: False
 # This is the description of your project. It will be used by sweep when creating PRs. You can tell Sweep what's unique about your project, what frameworks you use, or anything else you want.
 # Here's an example: sweepai/sweep is a python project. The main api endpoints are in sweepai/api.py. Write code that adheres to PEP8.
-description: ""
+description: "LlamaIndexTS is a data framework in TypeScript for your LLM applications"
+
+sandbox:
+  install:
+    - npm install -g pnpm
+    - pnpm i
+    - pnpm add --save-dev prettier -w
+  check:
+    - pnpx prettier --write {file_path}
+    - pnpm eslint --fix {file_path}
+    - pnpx ts-node --type-check {file_path}
+    - pnpm test
+
 # Default Values: https://github.com/sweepai/sweep/blob/main/sweep.yaml