RELEASING: Releasing 2 package(s)

Releases:
  llamaindex@0.0.48
  create-llama@0.0.16

[skip ci]

.eslintrc.js

@@ -7,4 +7,8 @@ module.exports = {
       rootDir: ["apps/*/"],
     },
   },
+  rules: {
+    "max-params": ["error", 4],
+  },
+  ignorePatterns: ["dist/"],
 };

.github/workflows/e2e.yml

@@ -8,6 +8,9 @@ on:
       - ".github/workflows/e2e.yml"
     branches: [main]
 
+env:
+  POETRY_VERSION: "1.6.1"
+
 jobs:
   e2e:
     name: create-llama
@@ -16,10 +19,19 @@ jobs:
       fail-fast: true
       matrix:
         node-version: [18, 20]
+        python-version: ["3.11"]
         os: [macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
       - uses: pnpm/action-setup@v2
       - name: Setup Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4

.gitignore

@@ -39,9 +39,6 @@ yarn-error.log*
 dist/
 lib/
 
-# vs code
-.vscode/launch.json
-
 .cache
 test-results/
 playwright-report/

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Example",
+      "skipFiles": ["<node_internals>/**"],
+      "runtimeExecutable": "pnpm",
+      "cwd": "${workspaceFolder}/examples",
+      "runtimeArgs": ["ts-node", "${fileBasename}"]
+    }
+  ]
+}

README.md

@@ -106,7 +106,6 @@ const nextConfig = {
       ...config.resolve.alias,
       sharp$: false,
       "onnxruntime-node$": false,
-      mongodb$: false,
     };
     return config;
   },

apps/docs/docs/modules/high_level/chat_engine.md

@@ -11,7 +11,16 @@ const retriever = index.asRetriever();
 const chatEngine = new ContextChatEngine({ retriever });
 
 // start chatting
-const response = await chatEngine.chat(query);
+const response = await chatEngine.chat({ message: query });
+```
+
+The `chat` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await chatEngine.chat({ message: query, stream: true });
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
 ```
 
 ## Api References

apps/docs/docs/modules/high_level/query_engine.md

@@ -8,7 +8,16 @@ A query engine wraps a `Retriever` and a `ResponseSynthesizer` into a pipeline,
 
 ```typescript
 const queryEngine = index.asQueryEngine();
-const response = await queryEngine.query("query string");
+const response = await queryEngine.query({ query: "query string" });
+```
+
+The `query` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await queryEngine.query({ query: "query string", stream: true });
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
 ```
 
 ## Sub Question Query Engine

apps/docs/docs/modules/low_level/response_synthesizer.md

@@ -35,13 +35,26 @@ const nodesWithScore: NodeWithScore[] = [
   },
 ];
 
-const response = await responseSynthesizer.synthesize(
-  "What age am I?",
+const response = await responseSynthesizer.synthesize({
+  query: "What age am I?",
   nodesWithScore,
-);
+});
 console.log(response.response);
 ```
 
+The `synthesize` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await responseSynthesizer.synthesize({
+  query: "What age am I?",
+  nodesWithScore,
+  stream: true,
+});
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
+```
+
 ## API Reference
 
 - [ResponseSynthesizer](../../api/classes/ResponseSynthesizer.md)

apps/docs/docs/modules/observability/_category_.yml

@@ -0,0 +1 @@
+label: Observability

apps/docs/docs/modules/observability/index.md

@@ -0,0 +1,35 @@
+# Observability
+
+LlamaIndex provides **one-click observability** 🔭 to allow you to build principled LLM applications in a production setting.
+
+A key requirement for principled development of LLM applications over your data (RAG systems, agents) is being able to observe, debug, and evaluate
+your system - both as a whole and for each component.
+
+This feature allows you to seamlessly integrate the LlamaIndex library with powerful observability/evaluation tools offered by our partners.
+Configure a variable once, and you'll be able to do things like the following:
+
+- View LLM/prompt inputs/outputs
+- Ensure that the outputs of any component (LLMs, embeddings) are performing as expected
+- View call traces for both indexing and querying
+
+Each provider has similarities and differences. Take a look below for the full set of guides for each one!
+
+## OpenLLMetry
+
+[OpenLLMetry](https://github.com/traceloop/openllmetry-js) is an open-source project based on OpenTelemetry for tracing and monitoring
+LLM applications. It connects to [all major observability platforms](https://www.traceloop.com/docs/openllmetry/integrations/introduction) and installs in minutes.
+
+### Usage Pattern
+
+```bash
+npm install @traceloop/node-server-sdk
+```
+
+```js
+import * as traceloop from "@traceloop/node-server-sdk";
+
+traceloop.initialize({
+  apiKey: process.env.TRACELOOP_API_KEY,
+  disableBatch: true,
+});
+```

examples/anthropic.ts

@@ -2,13 +2,15 @@ import { Anthropic } from "llamaindex";
 
 (async () => {
   const anthropic = new Anthropic();
-  const result = await anthropic.chat([
-    { content: "You want to talk in rhymes.", role: "system" },
-    {
-      content:
-        "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
-      role: "user",
-    },
-  ]);
+  const result = await anthropic.chat({
+    messages: [
+      { content: "You want to talk in rhymes.", role: "system" },
+      {
+        content:
+          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
+        role: "user",
+      },
+    ],
+  });
   console.log(result);
 })();

examples/astradb/query.ts

@@ -18,7 +18,9 @@ async function main() {
 
     const queryEngine = await index.asQueryEngine({ retriever });
 
-    const results = await queryEngine.query("What is the best reviewed movie?");
+    const results = await queryEngine.query({
+      query: "What is the best reviewed movie?",
+    });
 
     console.log(results.response);
   } catch (e) {

examples/chatEngine.ts

@@ -25,8 +25,11 @@ async function main() {
 
   while (true) {
     const query = await rl.question("Query: ");
-    const response = await chatEngine.chat(query);
-    console.log(response.toString());
+    const stream = await chatEngine.chat({ message: query, stream: true });
+    console.log();
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.response);
+    }
   }
 }
 

examples/chatHistory.ts

@@ -0,0 +1,34 @@
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+
+import { OpenAI, SimpleChatEngine, SummaryChatHistory } from "llamaindex";
+
+async function main() {
+  // Set maxTokens to 75% of the context window size of 4096
+  // This will trigger the summarizer once the chat history reaches 25% of the context window size (1024 tokens)
+  const llm = new OpenAI({ model: "gpt-3.5-turbo", maxTokens: 4096 * 0.75 });
+  const chatHistory = new SummaryChatHistory({ llm });
+  const chatEngine = new SimpleChatEngine({ llm });
+  const rl = readline.createInterface({ input, output });
+
+  while (true) {
+    const query = await rl.question("Query: ");
+    const stream = await chatEngine.chat({
+      message: query,
+      chatHistory,
+      stream: true,
+    });
+    if (chatHistory.getLastSummary()) {
+      // Print the summary of the conversation so far that is produced by the SummaryChatHistory
+      console.log(`Summary: ${chatHistory.getLastSummary()?.content}`);
+    }
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.response);
+    }
+    console.log();
+  }
+}
+
+main().catch(console.error);

examples/chromadb/test.ts

@@ -28,9 +28,9 @@ async function main() {
 
     console.log("Querying index");
     const queryEngine = index.asQueryEngine();
-    const response = await queryEngine.query(
-      "Tell me about Godfrey Cheshire's rating of La Sapienza.",
-    );
+    const response = await queryEngine.query({
+      query: "Tell me about Godfrey Cheshire's rating of La Sapienza.",
+    });
     console.log(response.toString());
   } catch (e) {
     console.error(e);

examples/gptllama.ts

@@ -25,12 +25,12 @@ import { ChatMessage, LlamaDeuce, OpenAI } from "llamaindex";
 
   while (true) {
     const next = history.length % 2 === 1 ? gpt4 : l2;
-    const r = await next.chat(
-      history.map(({ content, role }) => ({
+    const r = await next.chat({
+      messages: history.map(({ content, role }) => ({
         content,
         role: next === l2 ? role : role === "user" ? "assistant" : "user",
       })),
-    );
+    });
     history.push({
       content: r.message.content,
       role: next === l2 ? "assistant" : "user",

examples/huggingface.ts

@@ -32,12 +32,15 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const stream = await queryEngine.query({
+    query: "What did the author do in college?",
+    stream: true,
+  });
 
   // Output response
-  console.log(response.toString());
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.response);
+  }
 }
 
 main().catch(console.error);

examples/jsonExtract.ts

@@ -21,18 +21,20 @@ async function main() {
     action_items: ["action item 1", "action item 2"],
   };
 
-  const response = await llm.chat([
-    {
-      role: "system",
-      content: `You are an expert assistant for summarizing and extracting insights from sales call transcripts.\n\nGenerate a valid JSON in the following format:\n\n${JSON.stringify(
-        example,
-      )}`,
-    },
-    {
-      role: "user",
-      content: `Here is the transcript: \n------\n${transcript}\n------`,
-    },
-  ]);
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "system",
+        content: `You are an expert assistant for summarizing and extracting insights from sales call transcripts.\n\nGenerate a valid JSON in the following format:\n\n${JSON.stringify(
+          example,
+        )}`,
+      },
+      {
+        role: "user",
+        content: `Here is the transcript: \n------\n${transcript}\n------`,
+      },
+    ],
+  });
 
   const json = JSON.parse(response.message.content);
 

examples/keywordIndex.ts

@@ -20,9 +20,9 @@ async function main() {
         mode,
       }),
     });
-    const response = await queryEngine.query(
-      "What did the author do growing up?",
-    );
+    const response = await queryEngine.query({
+      query: "What did the author do growing up?",
+    });
     console.log(response.toString());
   });
 }

examples/llamadeuce.ts

@@ -2,6 +2,8 @@ import { DeuceChatStrategy, LlamaDeuce } from "llamaindex";
 
 (async () => {
   const deuce = new LlamaDeuce({ chatStrategy: DeuceChatStrategy.META });
-  const result = await deuce.chat([{ content: "Hello, world!", role: "user" }]);
+  const result = await deuce.chat({
+    messages: [{ content: "Hello, world!", role: "user" }],
+  });
   console.log(result);
 })();

examples/lowlevel.ts

@@ -27,9 +27,12 @@ import {
     },
   ];
 
-  const response = await responseSynthesizer.synthesize(
-    "What age am I?",
+  const stream = await responseSynthesizer.synthesize({
+    query: "What age am I?",
     nodesWithScore,
-  );
-  console.log(response.response);
+    stream: true,
+  });
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.response);
+  }
 })();

examples/markdown.ts

@@ -11,7 +11,9 @@ async function main() {
   // Query the index
   const queryEngine = index.asQueryEngine();
 
-  const response = await queryEngine.query("What does the example code do?");
+  const response = await queryEngine.query({
+    query: "What does the example code do?",
+  });
 
   // Output response
   console.log(response.toString());

examples/mistral.ts

@@ -27,7 +27,7 @@ async function rag(llm: LLM, embedModel: BaseEmbedding, query: string) {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(query);
+  const response = await queryEngine.query({ query });
   return response.response;
 }
 
@@ -43,19 +43,20 @@ async function rag(llm: LLM, embedModel: BaseEmbedding, query: string) {
 
   // chat api (non-streaming)
   const llm = new MistralAI({ model: "mistral-tiny" });
-  const response = await llm.chat([
-    { content: "What is the best French cheese?", role: "user" },
-  ]);
+  const response = await llm.chat({
+    messages: [{ content: "What is the best French cheese?", role: "user" }],
+  });
   console.log(response.message.content);
 
   // chat api (streaming)
-  const stream = await llm.chat(
-    [{ content: "Who is the most renowned French painter?", role: "user" }],
-    undefined,
-    true,
-  );
+  const stream = await llm.chat({
+    messages: [
+      { content: "Who is the most renowned French painter?", role: "user" },
+    ],
+    stream: true,
+  });
   for await (const chunk of stream) {
-    process.stdout.write(chunk);
+    process.stdout.write(chunk.delta);
   }
 
   // rag

examples/mongo.ts

@@ -54,7 +54,7 @@ async function main() {
       break;
     }
 
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
 
     // Output response
     console.log(response.toString());

examples/mongodb/3_query.ts

@@ -24,9 +24,9 @@ async function query() {
 
   const retriever = index.asRetriever({ similarityTopK: 20 });
   const queryEngine = index.asQueryEngine({ retriever });
-  const result = await queryEngine.query(
-    "What does the author think of web frameworks?",
-  );
+  const result = await queryEngine.query({
+    query: "What does the author think of web frameworks?",
+  });
   console.log(result.response);
   await client.close();
 }

examples/multimodal/rag.ts

@@ -46,9 +46,9 @@ async function main() {
     responseSynthesizer: new MultiModalResponseSynthesizer({ serviceContext }),
     retriever: index.asRetriever({ similarityTopK: 3, imageSimilarityTopK: 1 }),
   });
-  const result = await queryEngine.query(
-    "Tell me more about Vincent van Gogh's famous paintings",
-  );
+  const result = await queryEngine.query({
+    query: "Tell me more about Vincent van Gogh's famous paintings",
+  });
   console.log(result.response, "\n");
   images.forEach((image) =>
     console.log(`Image retrieved and used in inference: ${image.toString()}`),

examples/ollama.ts

@@ -3,32 +3,34 @@ import { Ollama } from "llamaindex";
 (async () => {
   const llm = new Ollama({ model: "llama2", temperature: 0.75 });
   {
-    const response = await llm.chat([
-      { content: "Tell me a joke.", role: "user" },
-    ]);
+    const response = await llm.chat({
+      messages: [{ content: "Tell me a joke.", role: "user" }],
+    });
     console.log("Response 1:", response.message.content);
   }
   {
-    const response = await llm.complete("How are you?");
-    console.log("Response 2:", response.message.content);
+    const response = await llm.complete({ prompt: "How are you?" });
+    console.log("Response 2:", response.text);
   }
   {
-    const response = await llm.chat(
-      [{ content: "Tell me a joke.", role: "user" }],
-      undefined,
-      true,
-    );
+    const response = await llm.chat({
+      messages: [{ content: "Tell me a joke.", role: "user" }],
+      stream: true,
+    });
     console.log("Response 3:");
     for await (const message of response) {
-      process.stdout.write(message); // no newline
+      process.stdout.write(message.delta); // no newline
     }
     console.log(); // newline
   }
   {
-    const response = await llm.complete("How are you?", undefined, true);
+    const response = await llm.complete({
+      prompt: "How are you?",
+      stream: true,
+    });
     console.log("Response 4:");
     for await (const message of response) {
-      process.stdout.write(message); // no newline
+      process.stdout.write(message.text); // no newline
     }
     console.log(); // newline
   }

examples/openai.ts

@@ -4,12 +4,12 @@ import { OpenAI } from "llamaindex";
   const llm = new OpenAI({ model: "gpt-4-1106-preview", temperature: 0.1 });
 
   // complete api
-  const response1 = await llm.complete("How are you?");
-  console.log(response1.message.content);
+  const response1 = await llm.complete({ prompt: "How are you?" });
+  console.log(response1.text);
 
   // chat api
-  const response2 = await llm.chat([
-    { content: "Tell me a joke.", role: "user" },
-  ]);
+  const response2 = await llm.chat({
+    messages: [{ content: "Tell me a joke.", role: "user" }],
+  });
   console.log(response2.message.content);
 })();

examples/pg-vector-store/query.ts

@@ -31,7 +31,7 @@ async function main() {
       }
 
       try {
-        const answer = await queryEngine.query(question);
+        const answer = await queryEngine.query({ query: question });
         console.log(answer.response);
       } catch (error) {
         console.error("Error:", error);

examples/pinecone-vector-store/query.ts

@@ -29,7 +29,7 @@ async function main() {
       }
 
       try {
-        const answer = await queryEngine.query(question);
+        const answer = await queryEngine.query({ query: question });
         console.log(answer.response);
       } catch (error) {
         console.error("Error:", error);

examples/portkey.ts

@@ -12,11 +12,14 @@ import { Portkey } from "llamaindex";
       },
     ],
   });
-  const result = portkey.streamChat([
-    { role: "system", content: "You are a helpful assistant." },
-    { role: "user", content: "Tell me a joke." },
-  ]);
+  const result = await portkey.chat({
+    messages: [
+      { role: "system", content: "You are a helpful assistant." },
+      { role: "user", content: "Tell me a joke." },
+    ],
+    stream: true,
+  });
   for await (const res of result) {
-    process.stdout.write(res);
+    process.stdout.write(res.delta);
   }
 })();

examples/readers/load-assemblyai.ts

@@ -48,7 +48,7 @@ program
         break;
       }
 
-      const response = await queryEngine.query(query);
+      const response = await queryEngine.query({ query });
 
       console.log(response.toString());
     }

examples/readers/load-csv.ts

@@ -38,9 +38,9 @@ Given the CSV file, generate me Typescript code to answer the question: ${query}
   const queryEngine = index.asQueryEngine({ responseSynthesizer });
 
   // Query the index
-  const response = await queryEngine.query(
-    "What is the correlation between survival and age?",
-  );
+  const response = await queryEngine.query({
+    query: "What is the correlation between survival and age?",
+  });
 
   // Output response
   console.log(response.toString());

examples/readers/load-docx.ts

@@ -15,7 +15,7 @@ async function main() {
 
   // Test query
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(SAMPLE_QUERY);
+  const response = await queryEngine.query({ query: SAMPLE_QUERY });
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 

examples/readers/load-html.ts

@@ -10,9 +10,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What were the notable changes in 18.1?",
-  );
+  const response = await queryEngine.query({
+    query: "What were the notable changes in 18.1?",
+  });
 
   // Output response
   console.log(response.toString());

examples/readers/load-md.ts

@@ -15,7 +15,7 @@ async function main() {
 
   // Test query
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(SAMPLE_QUERY);
+  const response = await queryEngine.query({ query: SAMPLE_QUERY });
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 

examples/readers/load-notion.ts

@@ -79,7 +79,7 @@ program
         break;
       }
 
-      const response = await queryEngine.query(query);
+      const response = await queryEngine.query({ query });
 
       // Output response
       console.log(response.toString());

examples/readers/load-pdf.ts

@@ -10,7 +10,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query("What mistakes did they make?");
+  const response = await queryEngine.query({
+    query: "What mistakes did they make?",
+  });
 
   // Output response
   console.log(response.toString());

examples/sentenceWindow.ts

@@ -31,9 +31,9 @@ async function main() {
   const queryEngine = index.asQueryEngine({
     nodePostprocessors: [new MetadataReplacementPostProcessor("window")],
   });
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/storageContext.ts

@@ -20,9 +20,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());
@@ -35,9 +35,9 @@ async function main() {
     storageContext: secondStorageContext,
   });
   const loadedQueryEngine = loadedIndex.asQueryEngine();
-  const loadedResponse = await loadedQueryEngine.query(
-    "What did the author do growing up?",
-  );
+  const loadedResponse = await loadedQueryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(loadedResponse.toString());
 }
 

examples/subquestion.ts

@@ -18,9 +18,9 @@ import essay from "./essay";
     ],
   });
 
-  const response = await queryEngine.query(
-    "How was Paul Grahams life different before and after YC?",
-  );
+  const response = await queryEngine.query({
+    query: "How was Paul Grahams life different before and after YC?",
+  });
 
   console.log(response.toString());
 })();

examples/summaryIndex.ts

@@ -20,9 +20,9 @@ async function main() {
   const queryEngine = index.asQueryEngine({
     retriever: index.asRetriever({ mode: SummaryRetrieverMode.LLM }),
   });
-  const response = await queryEngine.query(
-    "What did the author do growing up?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(response.toString());
 }
 

examples/together-ai.ts

@@ -6,8 +6,8 @@ const together = new TogetherLLM({
 });
 
 (async () => {
-  const generator = await together.chat(
-    [
+  const generator = await together.chat({
+    messages: [
       {
         role: "system",
         content: "You are an AI assistant",
@@ -17,12 +17,11 @@ const together = new TogetherLLM({
         content: "Tell me about San Francisco",
       },
     ],
-    undefined,
-    true,
-  );
+    stream: true,
+  });
   console.log("Chatting with Together AI...");
   for await (const message of generator) {
-    process.stdout.write(message);
+    process.stdout.write(message.delta);
   }
   const embedding = new TogetherEmbedding();
   const vector = await embedding.getTextEmbedding("Hello world!");

examples/together-ai/vector-index.ts

@@ -0,0 +1,39 @@
+import fs from "node:fs/promises";
+
+import {
+  Document,
+  TogetherEmbedding,
+  TogetherLLM,
+  VectorStoreIndex,
+  serviceContextFromDefaults,
+} from "llamaindex";
+
+async function main() {
+  const apiKey = process.env.TOGETHER_API_KEY;
+  if (!apiKey) {
+    throw new Error("Missing TOGETHER_API_KEY");
+  }
+  const path = require.resolve("llamaindex/examples/abramov.txt");
+  const essay = await fs.readFile(path, "utf-8");
+
+  const document = new Document({ text: essay, id_: path });
+
+  const serviceContext = serviceContextFromDefaults({
+    llm: new TogetherLLM({ model: "mistralai/Mixtral-8x7B-Instruct-v0.1" }),
+    embedModel: new TogetherEmbedding(),
+  });
+
+  const index = await VectorStoreIndex.fromDocuments([document], {
+    serviceContext,
+  });
+
+  const queryEngine = index.asQueryEngine();
+
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);

examples/vectorIndex.ts

@@ -16,9 +16,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vectorIndexAnthropic.ts

@@ -35,9 +35,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine({ responseSynthesizer });
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vectorIndexCustomize.ts

@@ -33,9 +33,9 @@ async function main() {
     [nodePostprocessor],
   );
 
-  const response = await queryEngine.query(
-    "What did the author do growing up?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(response.response);
 }
 

examples/vectorIndexFromVectorStore.ts

@@ -189,7 +189,9 @@ async function main() {
     },
   });
 
-  const response = await queryEngine.query("How many results do you have?");
+  const response = await queryEngine.query({
+    query: "How many results do you have?",
+  });
 
   console.log(response.toString());
 }

examples/vectorIndexGPT4.ts

@@ -25,9 +25,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vision.ts

@@ -4,12 +4,12 @@ import { OpenAI } from "llamaindex";
   const llm = new OpenAI({ model: "gpt-4-vision-preview", temperature: 0.1 });
 
   // complete api
-  const response1 = await llm.complete("How are you?");
-  console.log(response1.message.content);
+  const response1 = await llm.complete({ prompt: "How are you?" });
+  console.log(response1.text);
 
   // chat api
-  const response2 = await llm.chat([
-    { content: "Tell me a joke!", role: "user" },
-  ]);
+  const response2 = await llm.chat({
+    messages: [{ content: "Tell me a joke!", role: "user" }],
+  });
   console.log(response2.message.content);
 })();

packages/core/CHANGELOG.md

@@ -1,5 +1,26 @@
 # llamaindex
 
+## 0.0.48
+
+### Patch Changes
+
+- 34a26e5: Remove HistoryChatEngine and use ChatHistory for all chat engines
+
+## 0.0.47
+
+### Patch Changes
+
+- 844029d: Add streaming support for QueryEngine (and unify streaming interface with ChatEngine)
+- 844029d: Breaking: Use parameter object for query and chat methods of ChatEngine and QueryEngine
+
+## 0.0.46
+
+### Patch Changes
+
+- 977f284: fixing import statement
+- 5d3bb66: fix: class SimpleKVStore might throw error in ES module
+- f18c9f6: refactor: Updated low-level streaming interface
+
 ## 0.0.45
 
 ### Patch Changes

packages/core/README.md

@@ -106,7 +106,6 @@ const nextConfig = {
       ...config.resolve.alias,
       sharp$: false,
       "onnxruntime-node$": false,
-      mongodb$: false,
     };
     return config;
   },

packages/core/jest.config.cjs

@@ -2,4 +2,5 @@
 module.exports = {
   preset: "ts-jest",
   testEnvironment: "node",
+  testPathIgnorePatterns: ["/lib/"],
 };

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.0.45",
+  "version": "0.0.48",
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.9.1",
@@ -49,7 +49,8 @@
       "types": "./dist/index.d.mts",
       "import": "./dist/index.mjs",
       "require": "./dist/index.js"
-    }
+    },
+    "./examples/*": "./examples/*"
   },
   "files": [
     "dist",

packages/core/src/ChatEngine.ts

@@ -1,452 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { ChatHistory } from "./ChatHistory";
-import { NodeWithScore, TextNode } from "./Node";
-import {
-  CondenseQuestionPrompt,
-  ContextSystemPrompt,
-  defaultCondenseQuestionPrompt,
-  defaultContextSystemPrompt,
-  messagesToHistoryStr,
-} from "./Prompt";
-import { BaseQueryEngine } from "./QueryEngine";
-import { Response } from "./Response";
-import { BaseRetriever } from "./Retriever";
-import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { Event } from "./callbacks/CallbackManager";
-import { ChatMessage, LLM, OpenAI } from "./llm";
-import { BaseNodePostprocessor } from "./postprocessors";
-
-/**
- * A ChatEngine is used to handle back and forth chats between the application and the LLM.
- */
-export interface ChatEngine {
-  /**
-   * Send message along with the class's current chat history to the LLM.
-   * @param message
-   * @param chatHistory optional chat history if you want to customize the chat history
-   * @param streaming optional streaming flag, which auto-sets the return value if True.
-   */
-  chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-    streaming?: T,
-  ): Promise<R>;
-
-  /**
-   * Resets the chat history so that it's empty.
-   */
-  reset(): void;
-}
-
-/**
- * SimpleChatEngine is the simplest possible chat engine. Useful for using your own custom prompts.
- */
-export class SimpleChatEngine implements ChatEngine {
-  chatHistory: ChatMessage[];
-  llm: LLM;
-
-  constructor(init?: Partial<SimpleChatEngine>) {
-    this.chatHistory = init?.chatHistory ?? [];
-    this.llm = init?.llm ?? new OpenAI();
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-    streaming?: T,
-  ): Promise<R> {
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-
-    //Non-streaming option
-    chatHistory = chatHistory ?? this.chatHistory;
-    chatHistory.push({ content: message, role: "user" });
-    const response = await this.llm.chat(chatHistory, undefined);
-    chatHistory.push(response.message);
-    this.chatHistory = chatHistory;
-    return new Response(response.message.content) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-  ): AsyncGenerator<string, void, unknown> {
-    chatHistory = chatHistory ?? this.chatHistory;
-    chatHistory.push({ content: message, role: "user" });
-    const response_generator = await this.llm.chat(
-      chatHistory,
-      undefined,
-      true,
-    );
-
-    var accumulator: string = "";
-    for await (const part of response_generator) {
-      accumulator += part;
-      yield part;
-    }
-
-    chatHistory.push({ content: accumulator, role: "assistant" });
-    this.chatHistory = chatHistory;
-    return;
-  }
-
-  reset() {
-    this.chatHistory = [];
-  }
-}
-
-/**
- * CondenseQuestionChatEngine is used in conjunction with a Index (for example VectorStoreIndex).
- * It does two steps on taking a user's chat message: first, it condenses the chat message
- * with the previous chat history into a question with more context.
- * Then, it queries the underlying Index using the new question with context and returns
- * the response.
- * CondenseQuestionChatEngine performs well when the input is primarily questions about the
- * underlying data. It performs less well when the chat messages are not questions about the
- * data, or are very referential to previous context.
- */
-export class CondenseQuestionChatEngine implements ChatEngine {
-  queryEngine: BaseQueryEngine;
-  chatHistory: ChatMessage[];
-  serviceContext: ServiceContext;
-  condenseMessagePrompt: CondenseQuestionPrompt;
-
-  constructor(init: {
-    queryEngine: BaseQueryEngine;
-    chatHistory: ChatMessage[];
-    serviceContext?: ServiceContext;
-    condenseMessagePrompt?: CondenseQuestionPrompt;
-  }) {
-    this.queryEngine = init.queryEngine;
-    this.chatHistory = init?.chatHistory ?? [];
-    this.serviceContext =
-      init?.serviceContext ?? serviceContextFromDefaults({});
-    this.condenseMessagePrompt =
-      init?.condenseMessagePrompt ?? defaultCondenseQuestionPrompt;
-  }
-
-  private async condenseQuestion(chatHistory: ChatMessage[], question: string) {
-    const chatHistoryStr = messagesToHistoryStr(chatHistory);
-
-    return this.serviceContext.llm.complete(
-      defaultCondenseQuestionPrompt({
-        question: question,
-        chatHistory: chatHistoryStr,
-      }),
-    );
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    const condensedQuestion = (
-      await this.condenseQuestion(chatHistory, extractText(message))
-    ).message.content;
-
-    const response = await this.queryEngine.query(condensedQuestion);
-
-    chatHistory.push({ content: message, role: "user" });
-    chatHistory.push({ content: response.response, role: "assistant" });
-
-    return response as R;
-  }
-
-  reset() {
-    this.chatHistory = [];
-  }
-}
-
-export interface Context {
-  message: ChatMessage;
-  nodes: NodeWithScore[];
-}
-
-export interface ContextGenerator {
-  generate(message: string, parentEvent?: Event): Promise<Context>;
-}
-
-export class DefaultContextGenerator implements ContextGenerator {
-  retriever: BaseRetriever;
-  contextSystemPrompt: ContextSystemPrompt;
-  nodePostprocessors: BaseNodePostprocessor[];
-
-  constructor(init: {
-    retriever: BaseRetriever;
-    contextSystemPrompt?: ContextSystemPrompt;
-    nodePostprocessors?: BaseNodePostprocessor[];
-  }) {
-    this.retriever = init.retriever;
-    this.contextSystemPrompt =
-      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
-    this.nodePostprocessors = init.nodePostprocessors || [];
-  }
-
-  private applyNodePostprocessors(nodes: NodeWithScore[]) {
-    return this.nodePostprocessors.reduce(
-      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
-      nodes,
-    );
-  }
-
-  async generate(message: string, parentEvent?: Event): Promise<Context> {
-    if (!parentEvent) {
-      parentEvent = {
-        id: randomUUID(),
-        type: "wrapper",
-        tags: ["final"],
-      };
-    }
-    const sourceNodesWithScore = await this.retriever.retrieve(
-      message,
-      parentEvent,
-    );
-
-    const nodes = this.applyNodePostprocessors(sourceNodesWithScore);
-
-    return {
-      message: {
-        content: this.contextSystemPrompt({
-          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
-        }),
-        role: "system",
-      },
-      nodes,
-    };
-  }
-}
-
-/**
- * ContextChatEngine uses the Index to get the appropriate context for each query.
- * The context is stored in the system prompt, and the chat history is preserved,
- * ideally allowing the appropriate context to be surfaced for each query.
- */
-export class ContextChatEngine implements ChatEngine {
-  chatModel: LLM;
-  chatHistory: ChatMessage[];
-  contextGenerator: ContextGenerator;
-
-  constructor(init: {
-    retriever: BaseRetriever;
-    chatModel?: LLM;
-    chatHistory?: ChatMessage[];
-    contextSystemPrompt?: ContextSystemPrompt;
-    nodePostprocessors?: BaseNodePostprocessor[];
-  }) {
-    this.chatModel =
-      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
-    this.chatHistory = init?.chatHistory ?? [];
-    this.contextGenerator = new DefaultContextGenerator({
-      retriever: init.retriever,
-      contextSystemPrompt: init?.contextSystemPrompt,
-    });
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-
-    const parentEvent: Event = {
-      id: randomUUID(),
-      type: "wrapper",
-      tags: ["final"],
-    };
-    const context = await this.contextGenerator.generate(
-      extractText(message),
-      parentEvent,
-    );
-
-    chatHistory.push({ content: message, role: "user" });
-
-    const response = await this.chatModel.chat(
-      [context.message, ...chatHistory],
-      parentEvent,
-    );
-    chatHistory.push(response.message);
-
-    this.chatHistory = chatHistory;
-
-    return new Response(
-      response.message.content,
-      context.nodes.map((r) => r.node),
-    ) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-  ): AsyncGenerator<string, void, unknown> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    const parentEvent: Event = {
-      id: randomUUID(),
-      type: "wrapper",
-      tags: ["final"],
-    };
-    const context = await this.contextGenerator.generate(
-      extractText(message),
-      parentEvent,
-    );
-
-    chatHistory.push({ content: message, role: "user" });
-
-    const response_stream = await this.chatModel.chat(
-      [context.message, ...chatHistory],
-      parentEvent,
-      true,
-    );
-    var accumulator: string = "";
-    for await (const part of response_stream) {
-      accumulator += part;
-      yield part;
-    }
-
-    chatHistory.push({ content: accumulator, role: "assistant" });
-
-    this.chatHistory = chatHistory;
-
-    return;
-  }
-
-  reset() {
-    this.chatHistory = [];
-  }
-}
-
-export interface MessageContentDetail {
-  type: "text" | "image_url";
-  text?: string;
-  image_url?: { url: string };
-}
-
-/**
- * Extended type for the content of a message that allows for multi-modal messages.
- */
-export type MessageContent = string | MessageContentDetail[];
-
-/**
- * Extracts just the text from a multi-modal message or the message itself if it's just text.
- *
- * @param message The message to extract text from.
- * @returns The extracted text
- */
-function extractText(message: MessageContent): string {
-  if (Array.isArray(message)) {
-    // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
-    // so we can pass them to the context generator
-    return (message as MessageContentDetail[])
-      .filter((c) => c.type === "text")
-      .map((c) => c.text)
-      .join("\n\n");
-  }
-  return message;
-}
-
-/**
- * HistoryChatEngine is a ChatEngine that uses a `ChatHistory` object
- * to keeps track of chat's message history.
- * A `ChatHistory` object is passed as a parameter for each call to the `chat` method,
- * so the state of the chat engine is preserved between calls.
- * Optionally, a `ContextGenerator` can be used to generate an additional context for each call to `chat`.
- */
-export class HistoryChatEngine {
-  llm: LLM;
-  contextGenerator?: ContextGenerator;
-
-  constructor(init?: Partial<HistoryChatEngine>) {
-    this.llm = init?.llm ?? new OpenAI();
-    this.contextGenerator = init?.contextGenerator;
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-    streaming?: T,
-  ): Promise<R> {
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-    const requestMessages = await this.prepareRequestMessages(
-      message,
-      chatHistory,
-    );
-    const response = await this.llm.chat(requestMessages);
-    chatHistory.addMessage(response.message);
-    return new Response(response.message.content) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-  ): AsyncGenerator<string, void, unknown> {
-    const requestMessages = await this.prepareRequestMessages(
-      message,
-      chatHistory,
-    );
-    const response_stream = await this.llm.chat(
-      requestMessages,
-      undefined,
-      true,
-    );
-
-    var accumulator = "";
-    for await (const part of response_stream) {
-      accumulator += part;
-      yield part;
-    }
-    chatHistory.addMessage({
-      content: accumulator,
-      role: "assistant",
-    });
-    return;
-  }
-
-  private async prepareRequestMessages(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-  ) {
-    chatHistory.addMessage({
-      content: message,
-      role: "user",
-    });
-    let requestMessages;
-    let context;
-    if (this.contextGenerator) {
-      const textOnly = extractText(message);
-      context = await this.contextGenerator.generate(textOnly);
-    }
-    requestMessages = await chatHistory.requestMessages(
-      context ? [context.message] : undefined,
-    );
-    return requestMessages;
-  }
-}

packages/core/src/ChatHistory.ts

@@ -1,4 +1,5 @@
-import { ChatMessage, LLM, MessageType, OpenAI } from "./llm/LLM";
+import { OpenAI } from "./llm/LLM";
+import { ChatMessage, LLM, MessageType } from "./llm/types";
 import {
   defaultSummaryPrompt,
   messagesToHistoryStr,
@@ -8,35 +9,38 @@ import {
 /**
  * A ChatHistory is used to keep the state of back and forth chat messages
  */
-export interface ChatHistory {
-  messages: ChatMessage[];
+export abstract class ChatHistory {
+  abstract get messages(): ChatMessage[];
   /**
    * Adds a message to the chat history.
    * @param message
    */
-  addMessage(message: ChatMessage): void;
+  abstract addMessage(message: ChatMessage): void;
 
   /**
    * Returns the messages that should be used as input to the LLM.
    */
-  requestMessages(transientMessages?: ChatMessage[]): Promise<ChatMessage[]>;
+  abstract requestMessages(
+    transientMessages?: ChatMessage[],
+  ): Promise<ChatMessage[]>;
 
   /**
    * Resets the chat history so that it's empty.
    */
-  reset(): void;
+  abstract reset(): void;
 
   /**
    * Returns the new messages since the last call to this function (or since calling the constructor)
    */
-  newMessages(): ChatMessage[];
+  abstract newMessages(): ChatMessage[];
 }
 
-export class SimpleChatHistory implements ChatHistory {
+export class SimpleChatHistory extends ChatHistory {
   messages: ChatMessage[];
   private messagesBefore: number;
 
   constructor(init?: Partial<SimpleChatHistory>) {
+    super();
     this.messages = init?.messages ?? [];
     this.messagesBefore = this.messages.length;
   }
@@ -60,7 +64,7 @@ export class SimpleChatHistory implements ChatHistory {
   }
 }
 
-export class SummaryChatHistory implements ChatHistory {
+export class SummaryChatHistory extends ChatHistory {
   tokensToSummarize: number;
   messages: ChatMessage[];
   summaryPrompt: SummaryPrompt;
@@ -68,6 +72,7 @@ export class SummaryChatHistory implements ChatHistory {
   private messagesBefore: number;
 
   constructor(init?: Partial<SummaryChatHistory>) {
+    super();
     this.messages = init?.messages ?? [];
     this.messagesBefore = this.messages.length;
     this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
@@ -79,6 +84,11 @@ export class SummaryChatHistory implements ChatHistory {
     }
     this.tokensToSummarize =
       this.llm.metadata.contextWindow - this.llm.metadata.maxTokens;
+    if (this.tokensToSummarize < this.llm.metadata.contextWindow * 0.25) {
+      throw new Error(
+        "The number of tokens that trigger the summarize process are less than 25% of the context window. Try lowering maxTokens or use a model with a larger context window.",
+      );
+    }
   }
 
   private async summarize(): Promise<ChatMessage> {
@@ -99,7 +109,7 @@ export class SummaryChatHistory implements ChatHistory {
       messagesToSummarize.shift();
     } while (this.llm.tokens(promptMessages) > this.tokensToSummarize);
 
-    const response = await this.llm.chat(promptMessages);
+    const response = await this.llm.chat({ messages: promptMessages });
     return { content: response.message.content, role: "memory" };
   }
 
@@ -119,6 +129,11 @@ export class SummaryChatHistory implements ChatHistory {
     return this.messages.length - 1 - index;
   }
 
+  public getLastSummary(): ChatMessage | null {
+    const lastSummaryIndex = this.getLastSummaryIndex();
+    return lastSummaryIndex ? this.messages[lastSummaryIndex] : null;
+  }
+
   private get systemMessages() {
     // get array of all system messages
     return this.messages.filter((message) => message.role === "system");
@@ -198,3 +213,12 @@ export class SummaryChatHistory implements ChatHistory {
     return newMessages;
   }
 }
+
+export function getHistory(
+  chatHistory?: ChatMessage[] | ChatHistory,
+): ChatHistory {
+  if (chatHistory instanceof ChatHistory) {
+    return chatHistory;
+  }
+  return new SimpleChatHistory({ messages: chatHistory });
+}

packages/core/src/Prompt.ts

@@ -1,4 +1,4 @@
-import { ChatMessage } from "./llm/LLM";
+import { ChatMessage } from "./llm/types";
 import { SubQuestion } from "./QuestionGenerator";
 import { ToolMetadata } from "./Tool";
 

packages/core/src/PromptHelper.ts

@@ -37,6 +37,7 @@ export class PromptHelper {
   tokenizer: (text: string) => Uint32Array;
   separator = " ";
 
+  // eslint-disable-next-line max-params
   constructor(
     contextWindow = DEFAULT_CONTEXT_WINDOW,
     numOutput = DEFAULT_NUM_OUTPUTS,

packages/core/src/QueryEngine.ts

@@ -17,16 +17,32 @@ import {
   ResponseSynthesizer,
 } from "./synthesizers";
 
+/**
+ * Parameters for sending a query.
+ */
+export interface QueryEngineParamsBase {
+  query: string;
+  parentEvent?: Event;
+}
+
+export interface QueryEngineParamsStreaming extends QueryEngineParamsBase {
+  stream: true;
+}
+
+export interface QueryEngineParamsNonStreaming extends QueryEngineParamsBase {
+  stream?: false | null;
+}
+
 /**
  * A query engine is a question answerer that can use one or more steps.
  */
 export interface BaseQueryEngine {
   /**
    * Query the query engine and get a response.
-   * @param query
-   * @param parentEvent
+   * @param params
    */
-  query(query: string, parentEvent?: Event): Promise<Response>;
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
 }
 
 /**
@@ -70,14 +86,31 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
     return this.applyNodePostprocessors(nodes);
   }
 
-  async query(query: string, parentEvent?: Event) {
-    const _parentEvent: Event = parentEvent || {
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { query, stream } = params;
+    const parentEvent: Event = params.parentEvent || {
       id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
     };
-    const nodes = await this.retrieve(query, _parentEvent);
-    return this.responseSynthesizer.synthesize(query, nodes, _parentEvent);
+    const nodesWithScore = await this.retrieve(query, parentEvent);
+    if (stream) {
+      return this.responseSynthesizer.synthesize({
+        query,
+        nodesWithScore,
+        parentEvent,
+        stream: true,
+      });
+    }
+    return this.responseSynthesizer.synthesize({
+      query,
+      nodesWithScore,
+      parentEvent,
+    });
   }
 }
 
@@ -131,11 +164,16 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
     });
   }
 
-  async query(query: string): Promise<Response> {
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { query, stream } = params;
     const subQuestions = await this.questionGen.generate(this.metadatas, query);
 
     // groups final retrieval+synthesis operation
-    const parentEvent: Event = {
+    const parentEvent: Event = params.parentEvent || {
       id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
@@ -153,10 +191,22 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
       subQuestions.map((subQ) => this.querySubQ(subQ, subQueryParentEvent)),
     );
 
-    const nodes = subQNodes
+    const nodesWithScore = subQNodes
       .filter((node) => node !== null)
       .map((node) => node as NodeWithScore);
-    return this.responseSynthesizer.synthesize(query, nodes, parentEvent);
+    if (stream) {
+      return this.responseSynthesizer.synthesize({
+        query,
+        nodesWithScore,
+        parentEvent,
+        stream: true,
+      });
+    }
+    return this.responseSynthesizer.synthesize({
+      query,
+      nodesWithScore,
+      parentEvent,
+    });
   }
 
   private async querySubQ(
@@ -167,7 +217,10 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
       const question = subQ.subQuestion;
       const queryEngine = this.queryEngines[subQ.toolName];
 
-      const response = await queryEngine.query(question, parentEvent);
+      const response = await queryEngine.query({
+        query: question,
+        parentEvent,
+      });
       const responseText = response.response;
       const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
       const node = new TextNode({ text: nodeText });

packages/core/src/QuestionGenerator.ts

@@ -9,7 +9,8 @@ import {
   defaultSubQuestionPrompt,
 } from "./Prompt";
 import { ToolMetadata } from "./Tool";
-import { LLM, OpenAI } from "./llm/LLM";
+import { OpenAI } from "./llm/LLM";
+import { LLM } from "./llm/types";
 
 export interface SubQuestion {
   subQuestion: string;
@@ -41,13 +42,13 @@ export class LLMQuestionGenerator implements BaseQuestionGenerator {
     const toolsStr = buildToolsText(tools);
     const queryStr = query;
     const prediction = (
-      await this.llm.complete(
-        this.prompt({
+      await this.llm.complete({
+        prompt: this.prompt({
           toolsStr,
           queryStr,
         }),
-      )
-    ).message.content;
+      })
+    ).text;
 
     const structuredOutput = this.outputParser.parse(prediction);
 

packages/core/src/Response.ts

@@ -1,7 +1,7 @@
 import { BaseNode } from "./Node";
 
 /**
- * Respone is the output of a LLM
+ * Response is the output of a LLM
  */
 export class Response {
   response: string;

packages/core/src/embeddings/utils.ts

@@ -66,6 +66,7 @@ export function similarity(
  * @param similarityCutoff minimum similarity score
  * @returns
  */
+// eslint-disable-next-line max-params
 export function getTopKEmbeddings(
   queryEmbedding: number[],
   embeddings: number[][],
@@ -108,6 +109,7 @@ export function getTopKEmbeddings(
   return [resultSimilarities, resultIds];
 }
 
+// eslint-disable-next-line max-params
 export function getTopKEmbeddingsLearner(
   queryEmbedding: number[],
   embeddings: number[][],
@@ -120,6 +122,7 @@ export function getTopKEmbeddingsLearner(
   // https://github.com/mljs/libsvm which itself hasn't been updated in a while
 }
 
+// eslint-disable-next-line max-params
 export function getTopKMMREmbeddings(
   queryEmbedding: number[],
   embeddings: number[][],

packages/core/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -0,0 +1,104 @@
+import { ChatHistory, getHistory } from "../../ChatHistory";
+import {
+  CondenseQuestionPrompt,
+  defaultCondenseQuestionPrompt,
+  messagesToHistoryStr,
+} from "../../Prompt";
+import { BaseQueryEngine } from "../../QueryEngine";
+import { Response } from "../../Response";
+import {
+  ServiceContext,
+  serviceContextFromDefaults,
+} from "../../ServiceContext";
+import { ChatMessage, LLM } from "../../llm";
+import { extractText, streamReducer } from "../../llm/utils";
+import {
+  ChatEngine,
+  ChatEngineParamsNonStreaming,
+  ChatEngineParamsStreaming,
+} from "./types";
+
+/**
+ * CondenseQuestionChatEngine is used in conjunction with a Index (for example VectorStoreIndex).
+ * It does two steps on taking a user's chat message: first, it condenses the chat message
+ * with the previous chat history into a question with more context.
+ * Then, it queries the underlying Index using the new question with context and returns
+ * the response.
+ * CondenseQuestionChatEngine performs well when the input is primarily questions about the
+ * underlying data. It performs less well when the chat messages are not questions about the
+ * data, or are very referential to previous context.
+ */
+
+export class CondenseQuestionChatEngine implements ChatEngine {
+  queryEngine: BaseQueryEngine;
+  chatHistory: ChatHistory;
+  llm: LLM;
+  condenseMessagePrompt: CondenseQuestionPrompt;
+
+  constructor(init: {
+    queryEngine: BaseQueryEngine;
+    chatHistory: ChatMessage[];
+    serviceContext?: ServiceContext;
+    condenseMessagePrompt?: CondenseQuestionPrompt;
+  }) {
+    this.queryEngine = init.queryEngine;
+    this.chatHistory = getHistory(init?.chatHistory);
+    this.llm = init?.serviceContext?.llm ?? serviceContextFromDefaults().llm;
+    this.condenseMessagePrompt =
+      init?.condenseMessagePrompt ?? defaultCondenseQuestionPrompt;
+  }
+
+  private async condenseQuestion(chatHistory: ChatHistory, question: string) {
+    const chatHistoryStr = messagesToHistoryStr(
+      await chatHistory.requestMessages(),
+    );
+
+    return this.llm.complete({
+      prompt: defaultCondenseQuestionPrompt({
+        question: question,
+        chatHistory: chatHistoryStr,
+      }),
+    });
+  }
+
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+    const chatHistory = params.chatHistory
+      ? getHistory(params.chatHistory)
+      : this.chatHistory;
+
+    const condensedQuestion = (
+      await this.condenseQuestion(chatHistory, extractText(message))
+    ).text;
+    chatHistory.addMessage({ content: message, role: "user" });
+
+    if (stream) {
+      const stream = await this.queryEngine.query({
+        query: condensedQuestion,
+        stream: true,
+      });
+      return streamReducer({
+        stream,
+        initialValue: "",
+        reducer: (accumulator, part) => (accumulator += part.response),
+        finished: (accumulator) => {
+          chatHistory.addMessage({ content: accumulator, role: "assistant" });
+        },
+      });
+    }
+    const response = await this.queryEngine.query({
+      query: condensedQuestion,
+    });
+    chatHistory.addMessage({ content: response.response, role: "assistant" });
+
+    return response;
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+}

packages/core/src/engines/chat/ContextChatEngine.ts

@@ -0,0 +1,113 @@
+import { randomUUID } from "node:crypto";
+import { ChatHistory, getHistory } from "../../ChatHistory";
+import { ContextSystemPrompt } from "../../Prompt";
+import { Response } from "../../Response";
+import { BaseRetriever } from "../../Retriever";
+import { Event } from "../../callbacks/CallbackManager";
+import { ChatMessage, ChatResponseChunk, LLM, OpenAI } from "../../llm";
+import { MessageContent } from "../../llm/types";
+import { extractText, streamConverter, streamReducer } from "../../llm/utils";
+import { BaseNodePostprocessor } from "../../postprocessors";
+import { DefaultContextGenerator } from "./DefaultContextGenerator";
+import {
+  ChatEngine,
+  ChatEngineParamsNonStreaming,
+  ChatEngineParamsStreaming,
+  ContextGenerator,
+} from "./types";
+
+/**
+ * ContextChatEngine uses the Index to get the appropriate context for each query.
+ * The context is stored in the system prompt, and the chat history is preserved,
+ * ideally allowing the appropriate context to be surfaced for each query.
+ */
+export class ContextChatEngine implements ChatEngine {
+  chatModel: LLM;
+  chatHistory: ChatHistory;
+  contextGenerator: ContextGenerator;
+
+  constructor(init: {
+    retriever: BaseRetriever;
+    chatModel?: LLM;
+    chatHistory?: ChatMessage[];
+    contextSystemPrompt?: ContextSystemPrompt;
+    nodePostprocessors?: BaseNodePostprocessor[];
+  }) {
+    this.chatModel =
+      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
+    this.chatHistory = getHistory(init?.chatHistory);
+    this.contextGenerator = new DefaultContextGenerator({
+      retriever: init.retriever,
+      contextSystemPrompt: init?.contextSystemPrompt,
+      nodePostprocessors: init?.nodePostprocessors,
+    });
+  }
+
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+    const chatHistory = params.chatHistory
+      ? getHistory(params.chatHistory)
+      : this.chatHistory;
+    const parentEvent: Event = {
+      id: randomUUID(),
+      type: "wrapper",
+      tags: ["final"],
+    };
+    const requestMessages = await this.prepareRequestMessages(
+      message,
+      chatHistory,
+      parentEvent,
+    );
+
+    if (stream) {
+      const stream = await this.chatModel.chat({
+        messages: requestMessages.messages,
+        parentEvent,
+        stream: true,
+      });
+      return streamConverter(
+        streamReducer({
+          stream,
+          initialValue: "",
+          reducer: (accumulator, part) => (accumulator += part.delta),
+          finished: (accumulator) => {
+            chatHistory.addMessage({ content: accumulator, role: "assistant" });
+          },
+        }),
+        (r: ChatResponseChunk) => new Response(r.delta, requestMessages.nodes),
+      );
+    }
+    const response = await this.chatModel.chat({
+      messages: requestMessages.messages,
+      parentEvent,
+    });
+    chatHistory.addMessage(response.message);
+    return new Response(response.message.content, requestMessages.nodes);
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+
+  private async prepareRequestMessages(
+    message: MessageContent,
+    chatHistory: ChatHistory,
+    parentEvent?: Event,
+  ) {
+    chatHistory.addMessage({
+      content: message,
+      role: "user",
+    });
+    const textOnly = extractText(message);
+    const context = await this.contextGenerator.generate(textOnly, parentEvent);
+    const nodes = context.nodes.map((r) => r.node);
+    const messages = await chatHistory.requestMessages(
+      context ? [context.message] : undefined,
+    );
+    return { nodes, messages };
+  }
+}

packages/core/src/engines/chat/DefaultContextGenerator.ts

@@ -0,0 +1,57 @@
+import { randomUUID } from "node:crypto";
+import { NodeWithScore, TextNode } from "../../Node";
+import { ContextSystemPrompt, defaultContextSystemPrompt } from "../../Prompt";
+import { BaseRetriever } from "../../Retriever";
+import { Event } from "../../callbacks/CallbackManager";
+import { BaseNodePostprocessor } from "../../postprocessors";
+import { Context, ContextGenerator } from "./types";
+
+export class DefaultContextGenerator implements ContextGenerator {
+  retriever: BaseRetriever;
+  contextSystemPrompt: ContextSystemPrompt;
+  nodePostprocessors: BaseNodePostprocessor[];
+
+  constructor(init: {
+    retriever: BaseRetriever;
+    contextSystemPrompt?: ContextSystemPrompt;
+    nodePostprocessors?: BaseNodePostprocessor[];
+  }) {
+    this.retriever = init.retriever;
+    this.contextSystemPrompt =
+      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
+    this.nodePostprocessors = init.nodePostprocessors || [];
+  }
+
+  private applyNodePostprocessors(nodes: NodeWithScore[]) {
+    return this.nodePostprocessors.reduce(
+      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
+      nodes,
+    );
+  }
+
+  async generate(message: string, parentEvent?: Event): Promise<Context> {
+    if (!parentEvent) {
+      parentEvent = {
+        id: randomUUID(),
+        type: "wrapper",
+        tags: ["final"],
+      };
+    }
+    const sourceNodesWithScore = await this.retriever.retrieve(
+      message,
+      parentEvent,
+    );
+
+    const nodes = this.applyNodePostprocessors(sourceNodesWithScore);
+
+    return {
+      message: {
+        content: this.contextSystemPrompt({
+          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
+        }),
+        role: "system",
+      },
+      nodes,
+    };
+  }
+}

packages/core/src/engines/chat/SimpleChatEngine.ts

@@ -0,0 +1,64 @@
+import { ChatHistory, getHistory } from "../../ChatHistory";
+import { Response } from "../../Response";
+import { ChatResponseChunk, LLM, OpenAI } from "../../llm";
+import { streamConverter, streamReducer } from "../../llm/utils";
+import {
+  ChatEngine,
+  ChatEngineParamsNonStreaming,
+  ChatEngineParamsStreaming,
+} from "./types";
+
+/**
+ * SimpleChatEngine is the simplest possible chat engine. Useful for using your own custom prompts.
+ */
+
+export class SimpleChatEngine implements ChatEngine {
+  chatHistory: ChatHistory;
+  llm: LLM;
+
+  constructor(init?: Partial<SimpleChatEngine>) {
+    this.chatHistory = getHistory(init?.chatHistory);
+    this.llm = init?.llm ?? new OpenAI();
+  }
+
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+
+    const chatHistory = params.chatHistory
+      ? getHistory(params.chatHistory)
+      : this.chatHistory;
+    chatHistory.addMessage({ content: message, role: "user" });
+
+    if (stream) {
+      const stream = await this.llm.chat({
+        messages: await chatHistory.requestMessages(),
+        stream: true,
+      });
+      return streamConverter(
+        streamReducer({
+          stream,
+          initialValue: "",
+          reducer: (accumulator, part) => (accumulator += part.delta),
+          finished: (accumulator) => {
+            chatHistory.addMessage({ content: accumulator, role: "assistant" });
+          },
+        }),
+        (r: ChatResponseChunk) => new Response(r.delta),
+      );
+    }
+
+    const response = await this.llm.chat({
+      messages: await chatHistory.requestMessages(),
+    });
+    chatHistory.addMessage(response.message);
+    return new Response(response.message.content);
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+}

packages/core/src/engines/chat/index.ts

@@ -0,0 +1,4 @@
+export { CondenseQuestionChatEngine } from "./CondenseQuestionChatEngine";
+export { ContextChatEngine } from "./ContextChatEngine";
+export { SimpleChatEngine } from "./SimpleChatEngine";
+export * from "./types";

packages/core/src/engines/chat/types.ts

@@ -0,0 +1,54 @@
+import { ChatHistory } from "../../ChatHistory";
+import { NodeWithScore } from "../../Node";
+import { Response } from "../../Response";
+import { Event } from "../../callbacks/CallbackManager";
+import { ChatMessage } from "../../llm";
+import { MessageContent } from "../../llm/types";
+
+/**
+ * Represents the base parameters for ChatEngine.
+ */
+export interface ChatEngineParamsBase {
+  message: MessageContent;
+  /**
+   * Optional chat history if you want to customize the chat history.
+   */
+  chatHistory?: ChatMessage[] | ChatHistory;
+}
+
+export interface ChatEngineParamsStreaming extends ChatEngineParamsBase {
+  stream: true;
+}
+
+export interface ChatEngineParamsNonStreaming extends ChatEngineParamsBase {
+  stream?: false | null;
+}
+
+/**
+ * A ChatEngine is used to handle back and forth chats between the application and the LLM.
+ */
+export interface ChatEngine {
+  /**
+   * Send message along with the class's current chat history to the LLM.
+   * @param params
+   */
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+
+  /**
+   * Resets the chat history so that it's empty.
+   */
+  reset(): void;
+}
+
+export interface Context {
+  message: ChatMessage;
+  nodes: NodeWithScore[];
+}
+
+/**
+ * A ContextGenerator is used to generate a context based on a message's text content
+ */
+export interface ContextGenerator {
+  generate(message: string, parentEvent?: Event): Promise<Context>;
+}

packages/core/src/index.ts

@@ -1,4 +1,3 @@
-export * from "./ChatEngine";
 export * from "./ChatHistory";
 export * from "./GlobalsHelper";
 export * from "./Node";
@@ -15,6 +14,7 @@ export * from "./Tool";
 export * from "./callbacks/CallbackManager";
 export * from "./constants";
 export * from "./embeddings";
+export * from "./engines/chat";
 export * from "./indices";
 export * from "./llm";
 export * from "./nodeParsers";

packages/core/src/indices/keyword/KeywordTableIndex.ts

@@ -149,12 +149,12 @@ export class KeywordTableIndex extends BaseIndex<KeywordTable> {
     text: string,
     serviceContext: ServiceContext,
   ): Promise<Set<string>> {
-    const response = await serviceContext.llm.complete(
-      defaultKeywordExtractPrompt({
+    const response = await serviceContext.llm.complete({
+      prompt: defaultKeywordExtractPrompt({
         context: text,
       }),
-    );
-    return extractKeywordsGivenResponse(response.message.content, "KEYWORDS:");
+    });
+    return extractKeywordsGivenResponse(response.text, "KEYWORDS:");
   }
 
   /**

packages/core/src/indices/keyword/KeywordTableIndexRetriever.ts

@@ -86,16 +86,13 @@ abstract class BaseKeywordTableRetriever implements BaseRetriever {
 // Extracts keywords using LLMs.
 export class KeywordTableLLMRetriever extends BaseKeywordTableRetriever {
   async getKeywords(query: string): Promise<string[]> {
-    const response = await this.serviceContext.llm.complete(
-      this.queryKeywordExtractTemplate({
+    const response = await this.serviceContext.llm.complete({
+      prompt: this.queryKeywordExtractTemplate({
         question: query,
         maxKeywords: this.maxKeywordsPerQuery,
       }),
-    );
-    const keywords = extractKeywordsGivenResponse(
-      response.message.content,
-      "KEYWORDS:",
-    );
+    });
+    const keywords = extractKeywordsGivenResponse(response.text, "KEYWORDS:");
     return [...keywords];
   }
 }

packages/core/src/indices/summary/SummaryIndexRetriever.ts

@@ -61,6 +61,7 @@ export class SummaryIndexLLMRetriever implements BaseRetriever {
   parseChoiceSelectAnswerFn: ChoiceSelectParserFunction;
   serviceContext: ServiceContext;
 
+  // eslint-disable-next-line max-params
   constructor(
     index: SummaryIndex,
     choiceSelectPrompt?: ChoiceSelectPrompt,
@@ -89,8 +90,10 @@ export class SummaryIndexLLMRetriever implements BaseRetriever {
       const fmtBatchStr = this.formatNodeBatchFn(nodesBatch);
       const input = { context: fmtBatchStr, query: query };
       const rawResponse = (
-        await this.serviceContext.llm.complete(this.choiceSelectPrompt(input))
-      ).message.content;
+        await this.serviceContext.llm.complete({
+          prompt: this.choiceSelectPrompt(input),
+        })
+      ).text;
 
       // parseResult is a map from doc number to relevance score
       const parseResult = this.parseChoiceSelectAnswerFn(

packages/core/src/llm/LLM.ts

@@ -10,8 +10,7 @@ import {
 
 import { ChatCompletionMessageParam } from "openai/resources";
 import { LLMOptions } from "portkey-ai";
-import { MessageContent } from "../ChatEngine";
-import { globalsHelper, Tokenizers } from "../GlobalsHelper";
+import { Tokenizers, globalsHelper } from "../GlobalsHelper";
 import {
   ANTHROPIC_AI_PROMPT,
   ANTHROPIC_HUMAN_PROMPT,
@@ -25,81 +24,19 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure";
-import { getOpenAISession, OpenAISession } from "./openai";
-import { getPortkeySession, PortkeySession } from "./portkey";
+import { BaseLLM } from "./base";
+import { OpenAISession, getOpenAISession } from "./openai";
+import { PortkeySession, getPortkeySession } from "./portkey";
 import { ReplicateSession } from "./replicate";
-
-export type MessageType =
-  | "user"
-  | "assistant"
-  | "system"
-  | "generic"
-  | "function"
-  | "memory";
-
-export interface ChatMessage {
-  content: any;
-  role: MessageType;
-}
-
-export interface ChatResponse {
-  message: ChatMessage;
-  raw?: Record<string, any>;
-  delta?: string;
-}
-
-// NOTE in case we need CompletionResponse to diverge from ChatResponse in the future
-export type CompletionResponse = ChatResponse;
-
-export interface LLMMetadata {
-  model: string;
-  temperature: number;
-  topP: number;
-  maxTokens?: number;
-  contextWindow: number;
-  tokenizer: Tokenizers | undefined;
-}
-
-/**
- * Unified language model interface
- */
-export interface LLM {
-  metadata: LLMMetadata;
-  // Whether a LLM has streaming support
-  hasStreaming: boolean;
-  /**
-   * Get a chat response from the LLM
-   * @param messages
-   *
-   * The return type of chat() and complete() are set by the "streaming" parameter being set to True.
-   */
-  chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-    streaming?: T,
-  ): Promise<R>;
-
-  /**
-   * Get a prompt completion from the LLM
-   * @param prompt the prompt to complete
-   */
-  complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: MessageContent,
-    parentEvent?: Event,
-    streaming?: T,
-  ): Promise<R>;
-
-  /**
-   * Calculates the number of tokens needed for the given chat messages
-   */
-  tokens(messages: ChatMessage[]): number;
-}
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMMetadata,
+  MessageType,
+} from "./types";
 
 export const GPT4_MODELS = {
   "gpt-4": { contextWindow: 8192 },
@@ -125,9 +62,7 @@ export const ALL_AVAILABLE_OPENAI_MODELS = {
 /**
  * OpenAI LLM implementation
  */
-export class OpenAI implements LLM {
-  hasStreaming: boolean = true;
-
+export class OpenAI extends BaseLLM {
   // Per completion OpenAI params
   model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
   temperature: number;
@@ -135,7 +70,7 @@ export class OpenAI implements LLM {
   maxTokens?: number;
   additionalChatOptions?: Omit<
     Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
-    "max_tokens" | "messages" | "model" | "temperature" | "top_p" | "streaming"
+    "max_tokens" | "messages" | "model" | "temperature" | "top_p" | "stream"
   >;
 
   // OpenAI session params
@@ -155,6 +90,7 @@ export class OpenAI implements LLM {
       azure?: AzureOpenAIConfig;
     },
   ) {
+    super();
     this.model = init?.model ?? "gpt-3.5-turbo";
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 1;
@@ -251,10 +187,14 @@ export class OpenAI implements LLM {
     }
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(messages: ChatMessage[], parentEvent?: Event, streaming?: T): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
       model: this.model,
       temperature: this.temperature,
@@ -270,11 +210,8 @@ export class OpenAI implements LLM {
       ...this.additionalChatOptions,
     };
     // Streaming
-    if (streaming) {
-      if (!this.hasStreaming) {
-        throw Error("No streaming support for this LLM.");
-      }
-      return this.streamChat(messages, parentEvent) as R;
+    if (stream) {
+      return this.streamChat(params);
     }
     // Non-streaming
     const response = await this.session.openai.chat.completions.create({
@@ -285,27 +222,13 @@ export class OpenAI implements LLM {
     const content = response.choices[0].message?.content ?? "";
     return {
       message: { content, role: response.choices[0].message.role },
-    } as R;
+    };
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    );
-  }
-
-  //We can wrap a stream in a generator to add some additional logging behavior
-  //For future edits: syntax for generator type is <typeof Yield, typeof Return, typeof Accept>
-  //"typeof Accept" refers to what types you'll accept when you manually call generator.next(<AcceptType>)
-  protected async *streamChat(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
+  protected async *streamChat({
+    messages,
+    parentEvent,
+  }: LLMChatParamsStreaming): AsyncIterable<ChatResponseChunk> {
     const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
       model: this.model,
       temperature: this.temperature,
@@ -339,6 +262,7 @@ export class OpenAI implements LLM {
           type: "llmPredict" as EventType,
         };
 
+    // TODO: add callback to streamConverter and use streamConverter here
     //Indices
     var idx_counter: number = 0;
     for await (const part of chunk_stream) {
@@ -360,18 +284,12 @@ export class OpenAI implements LLM {
 
       idx_counter++;
 
-      yield part.choices[0].delta.content ? part.choices[0].delta.content : "";
+      yield {
+        delta: part.choices[0].delta.content ?? "",
+      };
     }
     return;
   }
-
-  //streamComplete doesn't need to be async because it's child function is already async
-  protected streamComplete(
-    query: string,
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: query, role: "user" }], parentEvent);
-  }
 }
 
 export const ALL_AVAILABLE_LLAMADEUCE_MODELS = {
@@ -429,16 +347,16 @@ export enum DeuceChatStrategy {
 /**
  * Llama2 LLM implementation
  */
-export class LlamaDeuce implements LLM {
+export class LlamaDeuce extends BaseLLM {
   model: keyof typeof ALL_AVAILABLE_LLAMADEUCE_MODELS;
   chatStrategy: DeuceChatStrategy;
   temperature: number;
   topP: number;
   maxTokens?: number;
   replicateSession: ReplicateSession;
-  hasStreaming: boolean;
 
   constructor(init?: Partial<LlamaDeuce>) {
+    super();
     this.model = init?.model ?? "Llama-2-70b-chat-4bit";
     this.chatStrategy =
       init?.chatStrategy ??
@@ -451,7 +369,6 @@ export class LlamaDeuce implements LLM {
       init?.maxTokens ??
       ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model].contextWindow; // For Replicate, the default is 500 tokens which is too low.
     this.replicateSession = init?.replicateSession ?? new ReplicateSession();
-    this.hasStreaming = init?.hasStreaming ?? false;
   }
 
   tokens(messages: ChatMessage[]): number {
@@ -596,10 +513,14 @@ If a question does not make any sense, or is not factually coherent, explain why
     };
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(messages: ChatMessage[], _parentEvent?: Event, streaming?: T): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     const api = ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model]
       .replicateApi as `${string}/${string}:${string}`;
 
@@ -621,6 +542,9 @@ If a question does not make any sense, or is not factually coherent, explain why
     }
 
     //TODO: Add streaming for this
+    if (stream) {
+      throw new Error("Streaming not supported for LlamaDeuce");
+    }
 
     //Non-streaming
     const response = await this.replicateSession.replicate.run(
@@ -633,14 +557,7 @@ If a question does not make any sense, or is not factually coherent, explain why
         //^ We need to do this because Replicate returns a list of strings (for streaming functionality which is not exposed by the run function)
         role: "assistant",
       },
-    } as R;
-  }
-
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
-    return this.chat([{ content: prompt, role: "user" }], parentEvent);
+    };
   }
 }
 
@@ -654,9 +571,7 @@ export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
  * Anthropic LLM implementation
  */
 
-export class Anthropic implements LLM {
-  hasStreaming: boolean = true;
-
+export class Anthropic extends BaseLLM {
   // Per completion Anthropic params
   model: keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS;
   temperature: number;
@@ -672,6 +587,7 @@ export class Anthropic implements LLM {
   callbackManager?: CallbackManager;
 
   constructor(init?: Partial<Anthropic>) {
+    super();
     this.model = init?.model ?? "claude-2";
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 0.999; // Per Ben Mann
@@ -723,20 +639,17 @@ export class Anthropic implements LLM {
     );
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     //Streaming
-    if (streaming) {
-      if (!this.hasStreaming) {
-        throw Error("No streaming support for this LLM.");
-      }
-      return this.streamChat(messages, parentEvent) as R;
+    if (stream) {
+      return this.streamChat(messages, parentEvent);
     }
 
     //Non-streaming
@@ -752,13 +665,13 @@ export class Anthropic implements LLM {
       message: { content: response.completion.trimStart(), role: "assistant" },
       //^ We're trimming the start because Anthropic often starts with a space in the response
       // That space will be re-added when we generate the next prompt.
-    } as R;
+    };
   }
 
   protected async *streamChat(
     messages: ChatMessage[],
     parentEvent?: Event | undefined,
-  ): AsyncGenerator<string, void, unknown> {
+  ): AsyncIterable<ChatResponseChunk> {
     // AsyncIterable<AnthropicStreamToken>
     const stream: AsyncIterable<AnthropicStreamToken> =
       await this.session.anthropic.completions.create({
@@ -775,40 +688,13 @@ export class Anthropic implements LLM {
       //TODO: LLM Stream Callback, pending re-work.
 
       idx_counter++;
-      yield part.completion;
+      yield { delta: part.completion };
     }
     return;
   }
-
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: string,
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    if (streaming) {
-      return this.streamComplete(prompt, parentEvent) as R;
-    }
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    ) as R;
-  }
-
-  protected streamComplete(
-    prompt: string,
-    parentEvent?: Event | undefined,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: prompt, role: "user" }], parentEvent);
-  }
 }
 
-export class Portkey implements LLM {
-  hasStreaming: boolean = true;
-
+export class Portkey extends BaseLLM {
   apiKey?: string = undefined;
   baseURL?: string = undefined;
   mode?: string = undefined;
@@ -817,6 +703,7 @@ export class Portkey implements LLM {
   callbackManager?: CallbackManager;
 
   constructor(init?: Partial<Portkey>) {
+    super();
     this.apiKey = init?.apiKey;
     this.baseURL = init?.baseURL;
     this.mode = init?.mode;
@@ -838,50 +725,34 @@ export class Portkey implements LLM {
     throw new Error("metadata not implemented for Portkey");
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-    streaming?: T,
-    params?: Record<string, any>,
-  ): Promise<R> {
-    if (streaming) {
-      return this.streamChat(messages, parentEvent, params) as R;
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream, extraParams } = params;
+    if (stream) {
+      return this.streamChat(messages, parentEvent, extraParams);
     } else {
-      const resolvedParams = params || {};
+      const bodyParams = extraParams || {};
       const response = await this.session.portkey.chatCompletions.create({
         messages,
-        ...resolvedParams,
+        ...bodyParams,
       });
 
       const content = response.choices[0].message?.content ?? "";
       const role = response.choices[0].message?.role || "assistant";
-      return { message: { content, role: role as MessageType } } as R;
+      return { message: { content, role: role as MessageType } };
     }
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: string,
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    );
-  }
-
   async *streamChat(
     messages: ChatMessage[],
     parentEvent?: Event,
     params?: Record<string, any>,
-  ): AsyncGenerator<string, void, unknown> {
+  ): AsyncIterable<ChatResponseChunk> {
     // Wrapping the stream in a callback.
     const onLLMStream = this.callbackManager?.onLLMStream
       ? this.callbackManager.onLLMStream
@@ -919,15 +790,8 @@ export class Portkey implements LLM {
 
       idx_counter++;
 
-      yield part.choices[0].delta?.content ?? "";
+      yield { delta: part.choices[0].delta?.content ?? "" };
     }
     return;
   }
-
-  streamComplete(
-    query: string,
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: query, role: "user" }], parentEvent);
-  }
 }

packages/core/src/llm/base.ts

@@ -0,0 +1,53 @@
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+} from "./types";
+import { streamConverter } from "./utils";
+
+export abstract class BaseLLM implements LLM {
+  abstract metadata: LLMMetadata;
+
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const { prompt, parentEvent, stream } = params;
+    if (stream) {
+      const stream = await this.chat({
+        messages: [{ content: prompt, role: "user" }],
+        parentEvent,
+        stream: true,
+      });
+      return streamConverter(stream, (chunk) => {
+        return {
+          text: chunk.delta,
+        };
+      });
+    }
+    const chatResponse = await this.chat({
+      messages: [{ content: prompt, role: "user" }],
+      parentEvent,
+    });
+    return { text: chatResponse.message.content as string };
+  }
+
+  abstract chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  abstract chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+
+  abstract tokens(messages: ChatMessage[]): number;
+}

packages/core/src/llm/index.ts

@@ -2,3 +2,4 @@ export * from "./LLM";
 export * from "./mistral";
 export { Ollama } from "./ollama";
 export { TogetherLLM } from "./together";
+export * from "./types";

packages/core/src/llm/mistral.ts

@@ -4,7 +4,14 @@ import {
   EventType,
   StreamCallbackResponse,
 } from "../callbacks/CallbackManager";
-import { ChatMessage, ChatResponse, LLM } from "./LLM";
+import { BaseLLM } from "./base";
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+} from "./types";
 
 export const ALL_AVAILABLE_MISTRAL_MODELS = {
   "mistral-tiny": { contextWindow: 32000 },
@@ -41,9 +48,7 @@ export class MistralAISession {
 /**
  * MistralAI LLM implementation
  */
-export class MistralAI implements LLM {
-  hasStreaming: boolean = true;
-
+export class MistralAI extends BaseLLM {
   // Per completion MistralAI params
   model: keyof typeof ALL_AVAILABLE_MISTRAL_MODELS;
   temperature: number;
@@ -57,6 +62,7 @@ export class MistralAI implements LLM {
   private session: MistralAISession;
 
   constructor(init?: Partial<MistralAI>) {
+    super();
     this.model = init?.model ?? "mistral-small";
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 1;
@@ -94,16 +100,17 @@ export class MistralAI implements LLM {
     };
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(messages: ChatMessage[], parentEvent?: Event, streaming?: T): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, stream } = params;
     // Streaming
-    if (streaming) {
-      if (!this.hasStreaming) {
-        throw Error("No streaming support for this LLM.");
-      }
-      return this.streamChat(messages, parentEvent) as R;
+    if (stream) {
+      return this.streamChat(params);
     }
     // Non-streaming
     const client = await this.session.getClient();
@@ -111,24 +118,13 @@ export class MistralAI implements LLM {
     const message = response.choices[0].message;
     return {
       message,
-    } as R;
+    };
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    );
-  }
-
-  protected async *streamChat(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
+  protected async *streamChat({
+    messages,
+    parentEvent,
+  }: LLMChatParamsStreaming): AsyncIterable<ChatResponseChunk> {
     //Now let's wrap our stream in a callback
     const onLLMStream = this.callbackManager?.onLLMStream
       ? this.callbackManager.onLLMStream
@@ -163,16 +159,10 @@ export class MistralAI implements LLM {
 
       idx_counter++;
 
-      yield part.choices[0].delta.content ?? "";
+      yield {
+        delta: part.choices[0].delta.content ?? "",
+      };
     }
     return;
   }
-
-  //streamComplete doesn't need to be async because it's child function is already async
-  protected streamComplete(
-    query: string,
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: query, role: "user" }], parentEvent);
-  }
 }

packages/core/src/llm/ollama.ts

@@ -1,11 +1,27 @@
 import { ok } from "node:assert";
-import { MessageContent } from "../ChatEngine";
 import { CallbackManager, Event } from "../callbacks/CallbackManager";
 import { BaseEmbedding } from "../embeddings";
-import { ChatMessage, ChatResponse, LLM, LLMMetadata } from "./LLM";
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+} from "./types";
 
-const messageAccessor = (data: any) => data.message.content;
-const completionAccessor = (data: any) => data.response;
+const messageAccessor = (data: any): ChatResponseChunk => {
+  return {
+    delta: data.message.content,
+  };
+};
+const completionAccessor = (data: any): CompletionResponse => {
+  return { text: data.response };
+};
 
 // https://github.com/jmorganca/ollama
 export class Ollama extends BaseEmbedding implements LLM {
@@ -43,21 +59,21 @@ export class Ollama extends BaseEmbedding implements LLM {
     };
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     const payload = {
       model: this.model,
       messages: messages.map((message) => ({
         role: message.role,
         content: message.content,
       })),
-      stream: !!streaming,
+      stream: !!stream,
       options: {
         temperature: this.temperature,
         num_ctx: this.contextWindow,
@@ -73,7 +89,7 @@ export class Ollama extends BaseEmbedding implements LLM {
         "Content-Type": "application/json",
       },
     });
-    if (!streaming) {
+    if (!stream) {
       const raw = await response.json();
       const { message } = raw;
       return {
@@ -82,20 +98,20 @@ export class Ollama extends BaseEmbedding implements LLM {
           content: message.content,
         },
         raw,
-      } satisfies ChatResponse as R;
+      };
     } else {
       const stream = response.body;
       ok(stream, "stream is null");
       ok(stream instanceof ReadableStream, "stream is not readable");
-      return this.streamChat(stream, messageAccessor, parentEvent) as R;
+      return this.streamChat(stream, messageAccessor, parentEvent);
     }
   }
 
-  private async *streamChat(
+  private async *streamChat<T>(
     stream: ReadableStream<Uint8Array>,
-    accessor: (data: any) => string,
+    accessor: (data: any) => T,
     parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
+  ): AsyncIterable<T> {
     const reader = stream.getReader();
     while (true) {
       const { done, value } = await reader.read();
@@ -119,18 +135,20 @@ export class Ollama extends BaseEmbedding implements LLM {
     }
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: MessageContent,
-    parentEvent?: Event | undefined,
-    streaming?: T | undefined,
-  ): Promise<R> {
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const { prompt, parentEvent, stream } = params;
     const payload = {
       model: this.model,
       prompt: prompt,
-      stream: !!streaming,
+      stream: !!stream,
       options: {
         temperature: this.temperature,
         num_ctx: this.contextWindow,
@@ -146,20 +164,17 @@ export class Ollama extends BaseEmbedding implements LLM {
         "Content-Type": "application/json",
       },
     });
-    if (!streaming) {
+    if (!stream) {
       const raw = await response.json();
       return {
-        message: {
-          role: "assistant",
-          content: raw.response,
-        },
+        text: raw.response,
         raw,
-      } satisfies ChatResponse as R;
+      };
     } else {
       const stream = response.body;
       ok(stream, "stream is null");
       ok(stream instanceof ReadableStream, "stream is not readable");
-      return this.streamChat(stream, completionAccessor, parentEvent) as R;
+      return this.streamChat(stream, completionAccessor, parentEvent);
     }
   }
 

packages/core/src/llm/types.ts

@@ -0,0 +1,110 @@
+import { Tokenizers } from "../GlobalsHelper";
+import { Event } from "../callbacks/CallbackManager";
+
+/**
+ * Unified language model interface
+ */
+export interface LLM {
+  metadata: LLMMetadata;
+  /**
+   * Get a chat response from the LLM
+   *
+   * @param params
+   */
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+
+  /**
+   * Get a prompt completion from the LLM
+   * @param params
+   */
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+
+  /**
+   * Calculates the number of tokens needed for the given chat messages
+   */
+  tokens(messages: ChatMessage[]): number;
+}
+
+export type MessageType =
+  | "user"
+  | "assistant"
+  | "system"
+  | "generic"
+  | "function"
+  | "memory";
+
+export interface ChatMessage {
+  // TODO: use MessageContent
+  content: any;
+  role: MessageType;
+}
+
+export interface ChatResponse {
+  message: ChatMessage;
+  raw?: Record<string, any>;
+}
+
+export interface ChatResponseChunk {
+  delta: string;
+}
+
+export interface CompletionResponse {
+  text: string;
+  raw?: Record<string, any>;
+}
+
+export interface LLMMetadata {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  contextWindow: number;
+  tokenizer: Tokenizers | undefined;
+}
+
+export interface LLMChatParamsBase {
+  messages: ChatMessage[];
+  parentEvent?: Event;
+  extraParams?: Record<string, any>;
+}
+
+export interface LLMChatParamsStreaming extends LLMChatParamsBase {
+  stream: true;
+}
+
+export interface LLMChatParamsNonStreaming extends LLMChatParamsBase {
+  stream?: false | null;
+}
+
+export interface LLMCompletionParamsBase {
+  prompt: any;
+  parentEvent?: Event;
+}
+
+export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
+  stream: true;
+}
+
+export interface LLMCompletionParamsNonStreaming
+  extends LLMCompletionParamsBase {
+  stream?: false | null;
+}
+
+export interface MessageContentDetail {
+  type: "text" | "image_url";
+  text?: string;
+  image_url?: { url: string };
+}
+
+/**
+ * Extended type for the content of a message that allows for multi-modal messages.
+ */
+export type MessageContent = string | MessageContentDetail[];

packages/core/src/llm/utils.ts

@@ -0,0 +1,44 @@
+import { MessageContent, MessageContentDetail } from "./types";
+
+export async function* streamConverter<S, D>(
+  stream: AsyncIterable<S>,
+  converter: (s: S) => D,
+): AsyncIterable<D> {
+  for await (const data of stream) {
+    yield converter(data);
+  }
+}
+
+export async function* streamReducer<S, D>(params: {
+  stream: AsyncIterable<S>;
+  reducer: (previousValue: D, currentValue: S) => D;
+  initialValue: D;
+  finished?: (value: D | undefined) => void;
+}): AsyncIterable<S> {
+  let value = params.initialValue;
+  for await (const data of params.stream) {
+    value = params.reducer(value, data);
+    yield data;
+  }
+  if (params.finished) {
+    params.finished(value);
+  }
+}
+/**
+ * Extracts just the text from a multi-modal message or the message itself if it's just text.
+ *
+ * @param message The message to extract text from.
+ * @returns The extracted text
+ */
+
+export function extractText(message: MessageContent): string {
+  if (Array.isArray(message)) {
+    // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
+    // so we can pass them to the context generator
+    return (message as MessageContentDetail[])
+      .filter((c) => c.type === "text")
+      .map((c) => c.text)
+      .join("\n\n");
+  }
+  return message;
+}

packages/core/src/readers/SimpleMongoReader.ts

@@ -36,6 +36,7 @@ export class SimpleMongoReader implements BaseReader {
    * @returns {Promise<Document[]>}
    * @throws If a field specified in fieldNames or metadataNames is not found in a MongoDB document.
    */
+  // eslint-disable-next-line max-params
   public async loadData(
     dbName: string,
     collectionName: string,

packages/core/src/storage/StorageContext.ts

@@ -1,4 +1,4 @@
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "./FileSystem";
 import {
   DEFAULT_FS,

packages/core/src/storage/docStore/SimpleDocumentStore.ts

@@ -1,5 +1,5 @@
 import _ from "lodash";
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "../FileSystem";
 import {
   DEFAULT_DOC_STORE_PERSIST_FILENAME,

packages/core/src/storage/indexStore/SimpleIndexStore.ts

@@ -1,4 +1,4 @@
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "../FileSystem";
 import {
   DEFAULT_FS,

packages/core/src/storage/kvStore/SimpleKVStore.ts

@@ -1,5 +1,5 @@
-import * as _ from "lodash";
-import * as path from "path";
+import _ from "lodash";
+import path from "path";
 import { GenericFileSystem, exists } from "../FileSystem";
 import { DEFAULT_COLLECTION, DEFAULT_FS } from "../constants";
 import { BaseKVStore } from "./types";

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -1,5 +1,5 @@
 import _ from "lodash";
-import * as path from "path";
+import path from "path";
 import { BaseNode } from "../../Node";
 import {
   getTopKEmbeddings,

packages/core/src/synthesizers/MultiModalResponseSynthesizer.ts

@@ -1,16 +1,14 @@
-import { MessageContentDetail } from "../ChatEngine";
-import {
-  ImageNode,
-  MetadataMode,
-  NodeWithScore,
-  splitNodesByType,
-} from "../Node";
+import { ImageNode, MetadataMode, splitNodesByType } from "../Node";
 import { Response } from "../Response";
 import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext";
-import { Event } from "../callbacks/CallbackManager";
 import { imageToDataUrl } from "../embeddings";
+import { MessageContentDetail } from "../llm/types";
 import { TextQaPrompt, defaultTextQaPrompt } from "./../Prompt";
-import { BaseSynthesizer } from "./types";
+import {
+  BaseSynthesizer,
+  SynthesizeParamsNonStreaming,
+  SynthesizeParamsStreaming,
+} from "./types";
 
 export class MultiModalResponseSynthesizer implements BaseSynthesizer {
   serviceContext: ServiceContext;
@@ -27,11 +25,21 @@ export class MultiModalResponseSynthesizer implements BaseSynthesizer {
     this.textQATemplate = textQATemplate ?? defaultTextQaPrompt;
   }
 
-  async synthesize(
-    query: string,
-    nodesWithScore: NodeWithScore[],
-    parentEvent?: Event,
-  ): Promise<Response> {
+  synthesize(
+    params: SynthesizeParamsStreaming,
+  ): Promise<AsyncIterable<Response>>;
+  synthesize(params: SynthesizeParamsNonStreaming): Promise<Response>;
+  async synthesize({
+    query,
+    nodesWithScore,
+    parentEvent,
+    stream,
+  }: SynthesizeParamsStreaming | SynthesizeParamsNonStreaming): Promise<
+    AsyncIterable<Response> | Response
+  > {
+    if (stream) {
+      throw new Error("streaming not implemented");
+    }
     const nodes = nodesWithScore.map(({ node }) => node);
     const { imageNodes, textNodes } = splitNodesByType(nodes);
     const textChunks = textNodes.map((node) =>
@@ -54,7 +62,10 @@ export class MultiModalResponseSynthesizer implements BaseSynthesizer {
       { type: "text", text: textPrompt },
       ...images,
     ];
-    let response = await this.serviceContext.llm.complete(prompt, parentEvent);
-    return new Response(response.message.content, nodes);
+    let response = await this.serviceContext.llm.complete({
+      prompt,
+      parentEvent,
+    });
+    return new Response(response.text, nodes);
   }
 }

packages/core/src/synthesizers/ResponseSynthesizer.ts

@@ -1,15 +1,20 @@
-import { Event } from "../callbacks/CallbackManager";
-import { MetadataMode, NodeWithScore } from "../Node";
+import { MetadataMode } from "../Node";
 import { Response } from "../Response";
 import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext";
-import { BaseResponseBuilder, getResponseBuilder } from "./builders";
-import { BaseSynthesizer } from "./types";
+import { streamConverter } from "../llm/utils";
+import { getResponseBuilder } from "./builders";
+import {
+  BaseSynthesizer,
+  ResponseBuilder,
+  SynthesizeParamsNonStreaming,
+  SynthesizeParamsStreaming,
+} from "./types";
 
 /**
  * A ResponseSynthesizer is used to generate a response from a query and a list of nodes.
  */
 export class ResponseSynthesizer implements BaseSynthesizer {
-  responseBuilder: BaseResponseBuilder;
+  responseBuilder: ResponseBuilder;
   serviceContext: ServiceContext;
   metadataMode: MetadataMode;
 
@@ -18,7 +23,7 @@ export class ResponseSynthesizer implements BaseSynthesizer {
     serviceContext,
     metadataMode = MetadataMode.NONE,
   }: {
-    responseBuilder?: BaseResponseBuilder;
+    responseBuilder?: ResponseBuilder;
     serviceContext?: ServiceContext;
     metadataMode?: MetadataMode;
   } = {}) {
@@ -28,22 +33,36 @@ export class ResponseSynthesizer implements BaseSynthesizer {
     this.metadataMode = metadataMode;
   }
 
-  async synthesize(
-    query: string,
-    nodesWithScore: NodeWithScore[],
-    parentEvent?: Event,
-  ) {
-    let textChunks: string[] = nodesWithScore.map(({ node }) =>
+  synthesize(
+    params: SynthesizeParamsStreaming,
+  ): Promise<AsyncIterable<Response>>;
+  synthesize(params: SynthesizeParamsNonStreaming): Promise<Response>;
+  async synthesize({
+    query,
+    nodesWithScore,
+    parentEvent,
+    stream,
+  }: SynthesizeParamsStreaming | SynthesizeParamsNonStreaming): Promise<
+    AsyncIterable<Response> | Response
+  > {
+    const textChunks: string[] = nodesWithScore.map(({ node }) =>
       node.getContent(this.metadataMode),
     );
-    const response = await this.responseBuilder.getResponse(
+    const nodes = nodesWithScore.map(({ node }) => node);
+    if (stream) {
+      const response = await this.responseBuilder.getResponse({
+        query,
+        textChunks,
+        parentEvent,
+        stream,
+      });
+      return streamConverter(response, (chunk) => new Response(chunk, nodes));
+    }
+    const response = await this.responseBuilder.getResponse({
       query,
       textChunks,
       parentEvent,
-    );
-    return new Response(
-      response,
-      nodesWithScore.map(({ node }) => node),
-    );
+    });
+    return new Response(response, nodes);
   }
 }

packages/core/src/synthesizers/builders.ts

@@ -1,5 +1,6 @@
 import { Event } from "../callbacks/CallbackManager";
-import { LLM } from "../llm/LLM";
+import { LLM } from "../llm";
+import { streamConverter } from "../llm/utils";
 import {
   defaultRefinePrompt,
   defaultTextQaPrompt,
@@ -9,8 +10,13 @@ import {
   TextQaPrompt,
   TreeSummarizePrompt,
 } from "../Prompt";
-import { getBiggestPrompt } from "../PromptHelper";
+import { getBiggestPrompt, PromptHelper } from "../PromptHelper";
 import { ServiceContext } from "../ServiceContext";
+import {
+  ResponseBuilder,
+  ResponseBuilderParamsNonStreaming,
+  ResponseBuilderParamsStreaming,
+} from "./types";
 
 /**
  * Response modes of the response synthesizer
@@ -22,29 +28,10 @@ enum ResponseMode {
   SIMPLE = "simple",
 }
 
-/**
- * A ResponseBuilder is used in a response synthesizer to generate a response from multiple response chunks.
- */
-export interface BaseResponseBuilder {
-  /**
-   * Get the response from a query and a list of text chunks.
-   * @param query
-   * @param textChunks
-   * @param parentEvent
-   * @param prevResponse
-   */
-  getResponse(
-    query: string,
-    textChunks: string[],
-    parentEvent?: Event,
-    prevResponse?: string,
-  ): Promise<string>;
-}
-
 /**
  * A response builder that just concatenates responses.
  */
-export class SimpleResponseBuilder implements BaseResponseBuilder {
+export class SimpleResponseBuilder implements ResponseBuilder {
   llm: LLM;
   textQATemplate: SimplePrompt;
 
@@ -53,27 +40,42 @@ export class SimpleResponseBuilder implements BaseResponseBuilder {
     this.textQATemplate = defaultTextQaPrompt;
   }
 
-  async getResponse(
-    query: string,
-    textChunks: string[],
-    parentEvent?: Event,
-  ): Promise<string> {
+  getResponse(
+    params: ResponseBuilderParamsStreaming,
+  ): Promise<AsyncIterable<string>>;
+  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
+  async getResponse({
+    query,
+    textChunks,
+    parentEvent,
+    stream,
+  }:
+    | ResponseBuilderParamsStreaming
+    | ResponseBuilderParamsNonStreaming): Promise<
+    AsyncIterable<string> | string
+  > {
     const input = {
       query,
       context: textChunks.join("\n\n"),
     };
 
     const prompt = this.textQATemplate(input);
-    const response = await this.llm.complete(prompt, parentEvent);
-    return response.message.content;
+    if (stream) {
+      const response = await this.llm.complete({ prompt, parentEvent, stream });
+      return streamConverter(response, (chunk) => chunk.text);
+    } else {
+      const response = await this.llm.complete({ prompt, parentEvent, stream });
+      return response.text;
+    }
   }
 }
 
 /**
  * A response builder that uses the query to ask the LLM generate a better response using multiple text chunks.
  */
-export class Refine implements BaseResponseBuilder {
-  serviceContext: ServiceContext;
+export class Refine implements ResponseBuilder {
+  llm: LLM;
+  promptHelper: PromptHelper;
   textQATemplate: TextQaPrompt;
   refineTemplate: RefinePrompt;
 
@@ -82,31 +84,48 @@ export class Refine implements BaseResponseBuilder {
     textQATemplate?: TextQaPrompt,
     refineTemplate?: RefinePrompt,
   ) {
-    this.serviceContext = serviceContext;
+    this.llm = serviceContext.llm;
+    this.promptHelper = serviceContext.promptHelper;
     this.textQATemplate = textQATemplate ?? defaultTextQaPrompt;
     this.refineTemplate = refineTemplate ?? defaultRefinePrompt;
   }
 
-  async getResponse(
-    query: string,
-    textChunks: string[],
-    parentEvent?: Event,
-    prevResponse?: string,
-  ): Promise<string> {
-    let response: string | undefined = undefined;
+  getResponse(
+    params: ResponseBuilderParamsStreaming,
+  ): Promise<AsyncIterable<string>>;
+  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
+  async getResponse({
+    query,
+    textChunks,
+    parentEvent,
+    prevResponse,
+    stream,
+  }:
+    | ResponseBuilderParamsStreaming
+    | ResponseBuilderParamsNonStreaming): Promise<
+    AsyncIterable<string> | string
+  > {
+    let response: AsyncIterable<string> | string | undefined = prevResponse;
 
-    for (const chunk of textChunks) {
-      if (!prevResponse) {
-        response = await this.giveResponseSingle(query, chunk, parentEvent);
-      } else {
-        response = await this.refineResponseSingle(
-          prevResponse,
+    for (let i = 0; i < textChunks.length; i++) {
+      const chunk = textChunks[i];
+      const lastChunk = i === textChunks.length - 1;
+      if (!response) {
+        response = await this.giveResponseSingle(
           query,
           chunk,
+          !!stream && lastChunk,
+          parentEvent,
+        );
+      } else {
+        response = await this.refineResponseSingle(
+          response as string,
+          query,
+          chunk,
+          !!stream && lastChunk,
           parentEvent,
         );
       }
-      prevResponse = response;
     }
 
     return response ?? "Empty Response";
@@ -115,153 +134,205 @@ export class Refine implements BaseResponseBuilder {
   private async giveResponseSingle(
     queryStr: string,
     textChunk: string,
+    stream: boolean,
     parentEvent?: Event,
-  ): Promise<string> {
+  ) {
     const textQATemplate: SimplePrompt = (input) =>
       this.textQATemplate({ ...input, query: queryStr });
-    const textChunks = this.serviceContext.promptHelper.repack(textQATemplate, [
-      textChunk,
-    ]);
+    const textChunks = this.promptHelper.repack(textQATemplate, [textChunk]);
 
-    let response: string | undefined = undefined;
+    let response: AsyncIterable<string> | string | undefined = undefined;
 
-    for (const chunk of textChunks) {
+    for (let i = 0; i < textChunks.length; i++) {
+      const chunk = textChunks[i];
+      const lastChunk = i === textChunks.length - 1;
       if (!response) {
-        response = (
-          await this.serviceContext.llm.complete(
-            textQATemplate({
-              context: chunk,
-            }),
-            parentEvent,
-          )
-        ).message.content;
+        response = await this.complete({
+          prompt: textQATemplate({
+            context: chunk,
+          }),
+          parentEvent,
+          stream: stream && lastChunk,
+        });
       } else {
         response = await this.refineResponseSingle(
-          response,
+          response as string,
           queryStr,
           chunk,
+          stream && lastChunk,
           parentEvent,
         );
       }
     }
 
-    return response ?? "Empty Response";
+    return response;
   }
 
+  // eslint-disable-next-line max-params
   private async refineResponseSingle(
-    response: string,
+    initialReponse: string,
     queryStr: string,
     textChunk: string,
+    stream: boolean,
     parentEvent?: Event,
   ) {
     const refineTemplate: SimplePrompt = (input) =>
       this.refineTemplate({ ...input, query: queryStr });
 
-    const textChunks = this.serviceContext.promptHelper.repack(refineTemplate, [
-      textChunk,
-    ]);
+    const textChunks = this.promptHelper.repack(refineTemplate, [textChunk]);
 
-    for (const chunk of textChunks) {
-      response = (
-        await this.serviceContext.llm.complete(
-          refineTemplate({
-            context: chunk,
-            existingAnswer: response,
-          }),
-          parentEvent,
-        )
-      ).message.content;
+    let response: AsyncIterable<string> | string = initialReponse;
+
+    for (let i = 0; i < textChunks.length; i++) {
+      const chunk = textChunks[i];
+      const lastChunk = i === textChunks.length - 1;
+      response = await this.complete({
+        prompt: refineTemplate({
+          context: chunk,
+          existingAnswer: response as string,
+        }),
+        parentEvent,
+        stream: stream && lastChunk,
+      });
     }
     return response;
   }
+
+  async complete(params: {
+    prompt: string;
+    stream: boolean;
+    parentEvent?: Event;
+  }): Promise<AsyncIterable<string> | string> {
+    if (params.stream) {
+      const response = await this.llm.complete({ ...params, stream: true });
+      return streamConverter(response, (chunk) => chunk.text);
+    } else {
+      const response = await this.llm.complete({ ...params, stream: false });
+      return response.text;
+    }
+  }
 }
 
 /**
  * CompactAndRefine is a slight variation of Refine that first compacts the text chunks into the smallest possible number of chunks.
  */
 export class CompactAndRefine extends Refine {
-  async getResponse(
-    query: string,
-    textChunks: string[],
-    parentEvent?: Event,
-    prevResponse?: string,
-  ): Promise<string> {
+  getResponse(
+    params: ResponseBuilderParamsStreaming,
+  ): Promise<AsyncIterable<string>>;
+  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
+  async getResponse({
+    query,
+    textChunks,
+    parentEvent,
+    prevResponse,
+    stream,
+  }:
+    | ResponseBuilderParamsStreaming
+    | ResponseBuilderParamsNonStreaming): Promise<
+    AsyncIterable<string> | string
+  > {
     const textQATemplate: SimplePrompt = (input) =>
       this.textQATemplate({ ...input, query: query });
     const refineTemplate: SimplePrompt = (input) =>
       this.refineTemplate({ ...input, query: query });
 
     const maxPrompt = getBiggestPrompt([textQATemplate, refineTemplate]);
-    const newTexts = this.serviceContext.promptHelper.repack(
-      maxPrompt,
-      textChunks,
-    );
-    const response = super.getResponse(
+    const newTexts = this.promptHelper.repack(maxPrompt, textChunks);
+    const params = {
       query,
-      newTexts,
+      textChunks: newTexts,
       parentEvent,
       prevResponse,
-    );
-    return response;
+    };
+    if (stream) {
+      return super.getResponse({
+        ...params,
+        stream,
+      });
+    }
+    return super.getResponse(params);
   }
 }
+
 /**
  * TreeSummarize repacks the text chunks into the smallest possible number of chunks and then summarizes them, then recursively does so until there's one chunk left.
  */
-export class TreeSummarize implements BaseResponseBuilder {
-  serviceContext: ServiceContext;
+export class TreeSummarize implements ResponseBuilder {
+  llm: LLM;
+  promptHelper: PromptHelper;
   summaryTemplate: TreeSummarizePrompt;
 
   constructor(
     serviceContext: ServiceContext,
     summaryTemplate?: TreeSummarizePrompt,
   ) {
-    this.serviceContext = serviceContext;
+    this.llm = serviceContext.llm;
+    this.promptHelper = serviceContext.promptHelper;
     this.summaryTemplate = summaryTemplate ?? defaultTreeSummarizePrompt;
   }
 
-  async getResponse(
-    query: string,
-    textChunks: string[],
-    parentEvent?: Event,
-  ): Promise<string> {
+  getResponse(
+    params: ResponseBuilderParamsStreaming,
+  ): Promise<AsyncIterable<string>>;
+  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
+  async getResponse({
+    query,
+    textChunks,
+    parentEvent,
+    stream,
+  }:
+    | ResponseBuilderParamsStreaming
+    | ResponseBuilderParamsNonStreaming): Promise<
+    AsyncIterable<string> | string
+  > {
     if (!textChunks || textChunks.length === 0) {
       throw new Error("Must have at least one text chunk");
     }
 
     // Should we send the query here too?
-    const packedTextChunks = this.serviceContext.promptHelper.repack(
+    const packedTextChunks = this.promptHelper.repack(
       this.summaryTemplate,
       textChunks,
     );
 
     if (packedTextChunks.length === 1) {
-      return (
-        await this.serviceContext.llm.complete(
-          this.summaryTemplate({
-            context: packedTextChunks[0],
-            query,
-          }),
-          parentEvent,
-        )
-      ).message.content;
+      const params = {
+        prompt: this.summaryTemplate({
+          context: packedTextChunks[0],
+          query,
+        }),
+        parentEvent,
+      };
+      if (stream) {
+        const response = await this.llm.complete({ ...params, stream });
+        return streamConverter(response, (chunk) => chunk.text);
+      }
+      return (await this.llm.complete(params)).text;
     } else {
       const summaries = await Promise.all(
         packedTextChunks.map((chunk) =>
-          this.serviceContext.llm.complete(
-            this.summaryTemplate({
+          this.llm.complete({
+            prompt: this.summaryTemplate({
               context: chunk,
               query,
             }),
             parentEvent,
-          ),
+          }),
         ),
       );
 
-      return this.getResponse(
+      const params = {
         query,
-        summaries.map((s) => s.message.content),
-      );
+        textChunks: summaries.map((s) => s.text),
+      };
+      if (stream) {
+        return this.getResponse({
+          ...params,
+          stream,
+        });
+      }
+      return this.getResponse(params);
     }
   }
 }
@@ -269,7 +340,7 @@ export class TreeSummarize implements BaseResponseBuilder {
 export function getResponseBuilder(
   serviceContext: ServiceContext,
   responseMode?: ResponseMode,
-): BaseResponseBuilder {
+): ResponseBuilder {
   switch (responseMode) {
     case ResponseMode.SIMPLE:
       return new SimpleResponseBuilder(serviceContext);

packages/core/src/synthesizers/types.ts

@@ -2,14 +2,57 @@ import { Event } from "../callbacks/CallbackManager";
 import { NodeWithScore } from "../Node";
 import { Response } from "../Response";
 
+export interface SynthesizeParamsBase {
+  query: string;
+  nodesWithScore: NodeWithScore[];
+  parentEvent?: Event;
+}
+
+export interface SynthesizeParamsStreaming extends SynthesizeParamsBase {
+  stream: true;
+}
+
+export interface SynthesizeParamsNonStreaming extends SynthesizeParamsBase {
+  stream?: false | null;
+}
+
 /**
  * A BaseSynthesizer is used to generate a response from a query and a list of nodes.
- * TODO: convert response builders to implement this interface (similar to Python).
  */
 export interface BaseSynthesizer {
   synthesize(
-    query: string,
-    nodesWithScore: NodeWithScore[],
-    parentEvent?: Event,
-  ): Promise<Response>;
+    params: SynthesizeParamsStreaming,
+  ): Promise<AsyncIterable<Response>>;
+  synthesize(params: SynthesizeParamsNonStreaming): Promise<Response>;
+}
+
+export interface ResponseBuilderParamsBase {
+  query: string;
+  textChunks: string[];
+  parentEvent?: Event;
+  prevResponse?: string;
+}
+
+export interface ResponseBuilderParamsStreaming
+  extends ResponseBuilderParamsBase {
+  stream: true;
+}
+
+export interface ResponseBuilderParamsNonStreaming
+  extends ResponseBuilderParamsBase {
+  stream?: false | null;
+}
+
+/**
+ * A ResponseBuilder is used in a response synthesizer to generate a response from multiple response chunks.
+ */
+export interface ResponseBuilder {
+  /**
+   * Get the response from a query and a list of text chunks.
+   * @param params
+   */
+  getResponse(
+    params: ResponseBuilderParamsStreaming,
+  ): Promise<AsyncIterable<string>>;
+  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
 }

packages/core/src/tests/CallbackManager.test.ts

@@ -67,7 +67,7 @@ describe("CallbackManager: onLLMStream and onRetrieve", () => {
     });
     const queryEngine = vectorStoreIndex.asQueryEngine();
     const query = "What is the author's name?";
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
     expect(response.toString()).toBe("MOCK_TOKEN_1-MOCK_TOKEN_2");
     expect(streamCallbackData).toEqual([
       {
@@ -145,7 +145,7 @@ describe("CallbackManager: onLLMStream and onRetrieve", () => {
       responseSynthesizer,
     });
     const query = "What is the author's name?";
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
     expect(response.toString()).toBe("MOCK_TOKEN_1-MOCK_TOKEN_2");
     expect(streamCallbackData).toEqual([
       {

packages/core/src/tests/utility/mockOpenAI.ts

@@ -1,7 +1,8 @@
-import { CallbackManager, Event } from "../../callbacks/CallbackManager";
+import { CallbackManager } from "../../callbacks/CallbackManager";
 import { OpenAIEmbedding } from "../../embeddings";
 import { globalsHelper } from "../../GlobalsHelper";
-import { ChatMessage, OpenAI } from "../../llm/LLM";
+import { OpenAI } from "../../llm/LLM";
+import { LLMChatParamsBase } from "../../llm/types";
 
 export function mockLlmGeneration({
   languageModel,
@@ -13,7 +14,7 @@ export function mockLlmGeneration({
   jest
     .spyOn(languageModel, "chat")
     .mockImplementation(
-      async (messages: ChatMessage[], parentEvent?: Event) => {
+      async ({ messages, parentEvent }: LLMChatParamsBase) => {
         const text = "MOCK_TOKEN_1-MOCK_TOKEN_2";
         const event = globalsHelper.createEvent({
           parentEvent,

packages/create-llama/CHANGELOG.md

@@ -1,5 +1,19 @@
 # create-llama
 
+## 0.0.16
+
+### Patch Changes
+
+- a73942d: Fix: Bundle mongo dependency with NextJS
+- 9492cc6: Feat: Added option to automatically install dependencies (for Python and TS)
+- f74dea5: Feat: Show images in chat messages using GPT4 Vision (Express and NextJS only)
+
+## 0.0.15
+
+### Patch Changes
+
+- 8e124e5: feat: support showing image on chat message
+
 ## 0.0.14
 
 ### Patch Changes

packages/create-llama/create-app.ts

@@ -34,6 +34,7 @@ export async function createApp({
   communityProjectPath,
   vectorDb,
   externalPort,
+  installDependencies,
 }: InstallAppArgs): Promise<void> {
   const root = path.resolve(appPath);
 
@@ -75,6 +76,7 @@ export async function createApp({
     communityProjectPath,
     vectorDb,
     externalPort,
+    installDependencies,
   };
 
   if (frontend) {

packages/create-llama/e2e/utils.ts

@@ -7,6 +7,7 @@ import waitPort from "wait-port";
 export type AppType = "--frontend" | "--no-frontend" | "";
 const MODEL = "gpt-3.5-turbo";
 
+// eslint-disable-next-line max-params
 export async function runApp(
   cwd: string,
   name: string,
@@ -65,6 +66,7 @@ async function createProcess(command: string, cwd: string, port: number) {
   return cp;
 }
 
+// eslint-disable-next-line max-params
 export function runCreateLlama(
   cwd: string,
   templateType: string,
@@ -104,6 +106,7 @@ export function runCreateLlama(
     "--use-npm",
     "--external-port",
     externalPort,
+    "--install-dependencies",
   ].join(" ");
   console.log(`running command '${command}' in ${cwd}`);
   execSync(command, {

packages/create-llama/helpers/index.ts

@@ -7,6 +7,7 @@ import { cyan } from "picocolors";
 
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./constant";
 import { PackageManager } from "./get-pkg-manager";
+import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry";
 import { installPythonTemplate } from "./python";
 import { downloadAndExtractRepo } from "./repo";
 import {
@@ -69,6 +70,7 @@ const copyTestData = async (
   engine?: TemplateEngine,
   openAiKey?: string,
   vectorDb?: TemplateVectorDB,
+  // eslint-disable-next-line max-params
 ) => {
   if (engine === "context") {
     const srcPath = path.join(
@@ -89,18 +91,23 @@ const copyTestData = async (
   if (packageManager && engine === "context") {
     const runGenerate = `${cyan(
       framework === "fastapi"
-        ? "python app/engine/generate.py"
+        ? "poetry run python app/engine/generate.py"
         : `${packageManager} run generate`,
     )}`;
     const hasOpenAiKey = openAiKey || process.env["OPENAI_API_KEY"];
     const hasVectorDb = vectorDb && vectorDb !== "none";
-    const shouldRunGenerateAfterInstall =
-      hasOpenAiKey && framework !== "fastapi" && vectorDb === "none";
-    if (shouldRunGenerateAfterInstall) {
-      console.log(`\nRunning ${runGenerate} to generate the context data.\n`);
-      await callPackageManager(packageManager, true, ["run", "generate"]);
-      console.log();
-      return;
+    if (framework === "fastapi") {
+      if (hasOpenAiKey && vectorDb === "none" && isHavingPoetryLockFile()) {
+        console.log(`Running ${runGenerate} to generate the context data.`);
+        tryPoetryRun("python app/engine/generate.py");
+        return;
+      }
+    } else {
+      if (hasOpenAiKey && vectorDb === "none") {
+        console.log(`Running ${runGenerate} to generate the context data.`);
+        await callPackageManager(packageManager, true, ["run", "generate"]);
+        return;
+      }
     }
 
     const settings = [];
@@ -162,6 +169,10 @@ export const installTemplate = async (
       props.openAiKey,
       props.vectorDb,
     );
+  } else {
+    // this is a frontend for a full-stack app, create .env file with model information
+    const content = `MODEL=${props.model}\nNEXT_PUBLIC_MODEL=${props.model}\n`;
+    await fs.writeFile(path.join(props.root, ".env"), content);
   }
 };
 

packages/create-llama/helpers/poetry.ts

@@ -0,0 +1,34 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { execSync } from "child_process";
+import fs from "fs";
+
+export function isPoetryAvailable(): boolean {
+  try {
+    execSync("poetry --version", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function tryPoetryInstall(): boolean {
+  try {
+    execSync("poetry install", { stdio: "inherit" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function tryPoetryRun(command: string): boolean {
+  try {
+    execSync(`poetry run ${command}`, { stdio: "inherit" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function isHavingPoetryLockFile(): boolean {
+  try {
+    return fs.existsSync("poetry.lock");
+  } catch (_) {}
+  return false;
+}

packages/create-llama/helpers/python.ts

@@ -1,8 +1,10 @@
 import fs from "fs/promises";
 import path from "path";
-import { cyan } from "picocolors";
+import { cyan, yellow } from "picocolors";
 import { parse, stringify } from "smol-toml";
+import terminalLink from "terminal-link";
 import { copy } from "./copy";
+import { isPoetryAvailable, tryPoetryInstall } from "./poetry";
 import { InstallTemplateArgs, TemplateVectorDB } from "./types";
 
 interface Dependency {
@@ -96,9 +98,15 @@ export const installPythonTemplate = async ({
   framework,
   engine,
   vectorDb,
+  installDependencies,
 }: Pick<
   InstallTemplateArgs,
-  "root" | "framework" | "template" | "engine" | "vectorDb"
+  | "root"
+  | "framework"
+  | "template"
+  | "engine"
+  | "vectorDb"
+  | "installDependencies"
 >) => {
   console.log("\nInitializing Python project with template:", template, "\n");
   const templatePath = path.join(
@@ -146,7 +154,28 @@ export const installPythonTemplate = async ({
   const addOnDependencies = getAdditionalDependencies(vectorDb);
   await addDependencies(root, addOnDependencies);
 
-  console.log(
-    "\nPython project, dependencies won't be installed automatically.\n",
-  );
+  // install python dependencies
+  if (installDependencies) {
+    if (isPoetryAvailable()) {
+      console.log(
+        `Installing python dependencies using poetry. This may take a while...`,
+      );
+      const installSuccessful = tryPoetryInstall();
+      if (!installSuccessful) {
+        console.warn(
+          yellow("Install failed. Please install dependencies manually."),
+        );
+      }
+    } else {
+      console.warn(
+        yellow(
+          `Poetry is not available in the current environment. The Python dependencies will not be installed automatically.
+Please check ${terminalLink(
+            "Poetry Installation",
+            `https://python-poetry.org/docs/#installation`,
+          )} to install poetry first, then install the dependencies manually.`,
+        ),
+      );
+    }
+  }
 };

packages/create-llama/helpers/types.ts

@@ -23,4 +23,5 @@ export interface InstallTemplateArgs {
   communityProjectPath?: string;
   vectorDb?: TemplateVectorDB;
   externalPort?: number;
+  installDependencies?: boolean;
 }

packages/create-llama/helpers/typescript.ts

@@ -39,6 +39,7 @@ export const installTSTemplate = async ({
   customApiPath,
   forBackend,
   vectorDb,
+  installDependencies,
 }: InstallTemplateArgs) => {
   console.log(bold(`Using ${packageManager}.`));
 
@@ -210,15 +211,17 @@ export const installTSTemplate = async ({
     JSON.stringify(packageJson, null, 2) + os.EOL,
   );
 
-  console.log("\nInstalling dependencies:");
-  for (const dependency in packageJson.dependencies)
-    console.log(`- ${cyan(dependency)}`);
+  if (installDependencies) {
+    console.log("\nInstalling dependencies:");
+    for (const dependency in packageJson.dependencies)
+      console.log(`- ${cyan(dependency)}`);
 
-  console.log("\nInstalling devDependencies:");
-  for (const dependency in packageJson.devDependencies)
-    console.log(`- ${cyan(dependency)}`);
+    console.log("\nInstalling devDependencies:");
+    for (const dependency in packageJson.devDependencies)
+      console.log(`- ${cyan(dependency)}`);
 
-  console.log();
+    console.log();
 
-  await callPackageManager(packageManager, isOnline);
+    await callPackageManager(packageManager, isOnline);
+  }
 };

packages/create-llama/index.ts

@@ -117,6 +117,13 @@ const program = new Commander.Command(packageJson.name)
     `
 
 Select external port.
+`,
+  )
+  .option(
+    "--install-dependencies",
+    `
+
+Whether install dependencies (backend/frontend) automatically or not.
 `,
   )
   .allowUnknownOption()
@@ -224,6 +231,7 @@ async function run(): Promise<void> {
     communityProjectPath: program.communityProjectPath,
     vectorDb: program.vectorDb,
     externalPort: program.externalPort,
+    installDependencies: program.installDependencies,
   });
   conf.set("preferences", preferences);
 }