RELEASING: Releasing 2 package(s)

Releases:
  llamaindex@0.0.48
  create-llama@0.0.16

[skip ci]

.eslintrc.js

@@ -7,4 +7,8 @@ module.exports = {
       rootDir: ["apps/*/"],
     },
   },
+  rules: {
+    "max-params": ["error", 4],
+  },
+  ignorePatterns: ["dist/"],
 };

.github/workflows/e2e.yml

@@ -8,6 +8,9 @@ on:
       - ".github/workflows/e2e.yml"
     branches: [main]
 
+env:
+  POETRY_VERSION: "1.6.1"
+
 jobs:
   e2e:
     name: create-llama
@@ -16,10 +19,19 @@ jobs:
       fail-fast: true
       matrix:
         node-version: [18, 20]
+        python-version: ["3.11"]
         os: [macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
       - uses: pnpm/action-setup@v2
       - name: Setup Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4

.gitignore

@@ -39,9 +39,6 @@ yarn-error.log*
 dist/
 lib/
 
-# vs code
-.vscode/launch.json
-
 .cache
 test-results/
 playwright-report/

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Example",
+      "skipFiles": ["<node_internals>/**"],
+      "runtimeExecutable": "pnpm",
+      "cwd": "${workspaceFolder}/examples",
+      "runtimeArgs": ["ts-node", "${fileBasename}"]
+    }
+  ]
+}

README.md

@@ -106,7 +106,6 @@ const nextConfig = {
       ...config.resolve.alias,
       sharp$: false,
       "onnxruntime-node$": false,
-      mongodb$: false,
     };
     return config;
   },

apps/docs/docs/modules/high_level/chat_engine.md

@@ -11,7 +11,16 @@ const retriever = index.asRetriever();
 const chatEngine = new ContextChatEngine({ retriever });
 
 // start chatting
-const response = await chatEngine.chat(query);
+const response = await chatEngine.chat({ message: query });
+```
+
+The `chat` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await chatEngine.chat({ message: query, stream: true });
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
 ```
 
 ## Api References

apps/docs/docs/modules/high_level/query_engine.md

@@ -8,7 +8,16 @@ A query engine wraps a `Retriever` and a `ResponseSynthesizer` into a pipeline,
 
 ```typescript
 const queryEngine = index.asQueryEngine();
-const response = await queryEngine.query("query string");
+const response = await queryEngine.query({ query: "query string" });
+```
+
+The `query` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await queryEngine.query({ query: "query string", stream: true });
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
 ```
 
 ## Sub Question Query Engine

examples/astradb/query.ts

@@ -18,7 +18,9 @@ async function main() {
 
     const queryEngine = await index.asQueryEngine({ retriever });
 
-    const results = await queryEngine.query("What is the best reviewed movie?");
+    const results = await queryEngine.query({
+      query: "What is the best reviewed movie?",
+    });
 
     console.log(results.response);
   } catch (e) {

examples/chatEngine.ts

@@ -25,8 +25,11 @@ async function main() {
 
   while (true) {
     const query = await rl.question("Query: ");
-    const response = await chatEngine.chat(query);
-    console.log(response.toString());
+    const stream = await chatEngine.chat({ message: query, stream: true });
+    console.log();
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.response);
+    }
   }
 }
 

examples/chatHistory.ts

@@ -0,0 +1,34 @@
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+
+import { OpenAI, SimpleChatEngine, SummaryChatHistory } from "llamaindex";
+
+async function main() {
+  // Set maxTokens to 75% of the context window size of 4096
+  // This will trigger the summarizer once the chat history reaches 25% of the context window size (1024 tokens)
+  const llm = new OpenAI({ model: "gpt-3.5-turbo", maxTokens: 4096 * 0.75 });
+  const chatHistory = new SummaryChatHistory({ llm });
+  const chatEngine = new SimpleChatEngine({ llm });
+  const rl = readline.createInterface({ input, output });
+
+  while (true) {
+    const query = await rl.question("Query: ");
+    const stream = await chatEngine.chat({
+      message: query,
+      chatHistory,
+      stream: true,
+    });
+    if (chatHistory.getLastSummary()) {
+      // Print the summary of the conversation so far that is produced by the SummaryChatHistory
+      console.log(`Summary: ${chatHistory.getLastSummary()?.content}`);
+    }
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.response);
+    }
+    console.log();
+  }
+}
+
+main().catch(console.error);

examples/chromadb/test.ts

@@ -28,9 +28,9 @@ async function main() {
 
     console.log("Querying index");
     const queryEngine = index.asQueryEngine();
-    const response = await queryEngine.query(
-      "Tell me about Godfrey Cheshire's rating of La Sapienza.",
-    );
+    const response = await queryEngine.query({
+      query: "Tell me about Godfrey Cheshire's rating of La Sapienza.",
+    });
     console.log(response.toString());
   } catch (e) {
     console.error(e);

examples/huggingface.ts

@@ -32,12 +32,15 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const stream = await queryEngine.query({
+    query: "What did the author do in college?",
+    stream: true,
+  });
 
   // Output response
-  console.log(response.toString());
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.response);
+  }
 }
 
 main().catch(console.error);

examples/keywordIndex.ts

@@ -20,9 +20,9 @@ async function main() {
         mode,
       }),
     });
-    const response = await queryEngine.query(
-      "What did the author do growing up?",
-    );
+    const response = await queryEngine.query({
+      query: "What did the author do growing up?",
+    });
     console.log(response.toString());
   });
 }

examples/markdown.ts

@@ -11,7 +11,9 @@ async function main() {
   // Query the index
   const queryEngine = index.asQueryEngine();
 
-  const response = await queryEngine.query("What does the example code do?");
+  const response = await queryEngine.query({
+    query: "What does the example code do?",
+  });
 
   // Output response
   console.log(response.toString());

examples/mistral.ts

@@ -27,7 +27,7 @@ async function rag(llm: LLM, embedModel: BaseEmbedding, query: string) {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(query);
+  const response = await queryEngine.query({ query });
   return response.response;
 }
 

examples/mongo.ts

@@ -54,7 +54,7 @@ async function main() {
       break;
     }
 
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
 
     // Output response
     console.log(response.toString());

examples/mongodb/3_query.ts

@@ -24,9 +24,9 @@ async function query() {
 
   const retriever = index.asRetriever({ similarityTopK: 20 });
   const queryEngine = index.asQueryEngine({ retriever });
-  const result = await queryEngine.query(
-    "What does the author think of web frameworks?",
-  );
+  const result = await queryEngine.query({
+    query: "What does the author think of web frameworks?",
+  });
   console.log(result.response);
   await client.close();
 }

examples/multimodal/rag.ts

@@ -46,9 +46,9 @@ async function main() {
     responseSynthesizer: new MultiModalResponseSynthesizer({ serviceContext }),
     retriever: index.asRetriever({ similarityTopK: 3, imageSimilarityTopK: 1 }),
   });
-  const result = await queryEngine.query(
-    "Tell me more about Vincent van Gogh's famous paintings",
-  );
+  const result = await queryEngine.query({
+    query: "Tell me more about Vincent van Gogh's famous paintings",
+  });
   console.log(result.response, "\n");
   images.forEach((image) =>
     console.log(`Image retrieved and used in inference: ${image.toString()}`),

examples/pg-vector-store/query.ts

@@ -31,7 +31,7 @@ async function main() {
       }
 
       try {
-        const answer = await queryEngine.query(question);
+        const answer = await queryEngine.query({ query: question });
         console.log(answer.response);
       } catch (error) {
         console.error("Error:", error);

examples/pinecone-vector-store/query.ts

@@ -29,7 +29,7 @@ async function main() {
       }
 
       try {
-        const answer = await queryEngine.query(question);
+        const answer = await queryEngine.query({ query: question });
         console.log(answer.response);
       } catch (error) {
         console.error("Error:", error);

examples/readers/load-assemblyai.ts

@@ -48,7 +48,7 @@ program
         break;
       }
 
-      const response = await queryEngine.query(query);
+      const response = await queryEngine.query({ query });
 
       console.log(response.toString());
     }

examples/readers/load-csv.ts

@@ -38,9 +38,9 @@ Given the CSV file, generate me Typescript code to answer the question: ${query}
   const queryEngine = index.asQueryEngine({ responseSynthesizer });
 
   // Query the index
-  const response = await queryEngine.query(
-    "What is the correlation between survival and age?",
-  );
+  const response = await queryEngine.query({
+    query: "What is the correlation between survival and age?",
+  });
 
   // Output response
   console.log(response.toString());

examples/readers/load-docx.ts

@@ -15,7 +15,7 @@ async function main() {
 
   // Test query
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(SAMPLE_QUERY);
+  const response = await queryEngine.query({ query: SAMPLE_QUERY });
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 

examples/readers/load-html.ts

@@ -10,9 +10,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What were the notable changes in 18.1?",
-  );
+  const response = await queryEngine.query({
+    query: "What were the notable changes in 18.1?",
+  });
 
   // Output response
   console.log(response.toString());

examples/readers/load-md.ts

@@ -15,7 +15,7 @@ async function main() {
 
   // Test query
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(SAMPLE_QUERY);
+  const response = await queryEngine.query({ query: SAMPLE_QUERY });
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 

examples/readers/load-notion.ts

@@ -79,7 +79,7 @@ program
         break;
       }
 
-      const response = await queryEngine.query(query);
+      const response = await queryEngine.query({ query });
 
       // Output response
       console.log(response.toString());

examples/readers/load-pdf.ts

@@ -10,7 +10,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query("What mistakes did they make?");
+  const response = await queryEngine.query({
+    query: "What mistakes did they make?",
+  });
 
   // Output response
   console.log(response.toString());

examples/sentenceWindow.ts

@@ -31,9 +31,9 @@ async function main() {
   const queryEngine = index.asQueryEngine({
     nodePostprocessors: [new MetadataReplacementPostProcessor("window")],
   });
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/storageContext.ts

@@ -20,9 +20,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());
@@ -35,9 +35,9 @@ async function main() {
     storageContext: secondStorageContext,
   });
   const loadedQueryEngine = loadedIndex.asQueryEngine();
-  const loadedResponse = await loadedQueryEngine.query(
-    "What did the author do growing up?",
-  );
+  const loadedResponse = await loadedQueryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(loadedResponse.toString());
 }
 

examples/subquestion.ts

@@ -18,9 +18,9 @@ import essay from "./essay";
     ],
   });
 
-  const response = await queryEngine.query(
-    "How was Paul Grahams life different before and after YC?",
-  );
+  const response = await queryEngine.query({
+    query: "How was Paul Grahams life different before and after YC?",
+  });
 
   console.log(response.toString());
 })();

examples/summaryIndex.ts

@@ -20,9 +20,9 @@ async function main() {
   const queryEngine = index.asQueryEngine({
     retriever: index.asRetriever({ mode: SummaryRetrieverMode.LLM }),
   });
-  const response = await queryEngine.query(
-    "What did the author do growing up?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(response.toString());
 }
 

examples/together-ai/vector-index.ts

@@ -0,0 +1,39 @@
+import fs from "node:fs/promises";
+
+import {
+  Document,
+  TogetherEmbedding,
+  TogetherLLM,
+  VectorStoreIndex,
+  serviceContextFromDefaults,
+} from "llamaindex";
+
+async function main() {
+  const apiKey = process.env.TOGETHER_API_KEY;
+  if (!apiKey) {
+    throw new Error("Missing TOGETHER_API_KEY");
+  }
+  const path = require.resolve("llamaindex/examples/abramov.txt");
+  const essay = await fs.readFile(path, "utf-8");
+
+  const document = new Document({ text: essay, id_: path });
+
+  const serviceContext = serviceContextFromDefaults({
+    llm: new TogetherLLM({ model: "mistralai/Mixtral-8x7B-Instruct-v0.1" }),
+    embedModel: new TogetherEmbedding(),
+  });
+
+  const index = await VectorStoreIndex.fromDocuments([document], {
+    serviceContext,
+  });
+
+  const queryEngine = index.asQueryEngine();
+
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);

examples/vectorIndex.ts

@@ -16,9 +16,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vectorIndexAnthropic.ts

@@ -35,9 +35,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine({ responseSynthesizer });
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vectorIndexCustomize.ts

@@ -33,9 +33,9 @@ async function main() {
     [nodePostprocessor],
   );
 
-  const response = await queryEngine.query(
-    "What did the author do growing up?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(response.response);
 }
 

examples/vectorIndexFromVectorStore.ts

@@ -189,7 +189,9 @@ async function main() {
     },
   });
 
-  const response = await queryEngine.query("How many results do you have?");
+  const response = await queryEngine.query({
+    query: "How many results do you have?",
+  });
 
   console.log(response.toString());
 }

examples/vectorIndexGPT4.ts

@@ -25,9 +25,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

packages/core/CHANGELOG.md

@@ -1,5 +1,18 @@
 # llamaindex
 
+## 0.0.48
+
+### Patch Changes
+
+- 34a26e5: Remove HistoryChatEngine and use ChatHistory for all chat engines
+
+## 0.0.47
+
+### Patch Changes
+
+- 844029d: Add streaming support for QueryEngine (and unify streaming interface with ChatEngine)
+- 844029d: Breaking: Use parameter object for query and chat methods of ChatEngine and QueryEngine
+
 ## 0.0.46
 
 ### Patch Changes

packages/core/README.md

@@ -106,7 +106,6 @@ const nextConfig = {
       ...config.resolve.alias,
       sharp$: false,
       "onnxruntime-node$": false,
-      mongodb$: false,
     };
     return config;
   },

packages/core/jest.config.cjs

@@ -2,4 +2,5 @@
 module.exports = {
   preset: "ts-jest",
   testEnvironment: "node",
+  testPathIgnorePatterns: ["/lib/"],
 };

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.0.46",
+  "version": "0.0.48",
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.9.1",
@@ -49,7 +49,8 @@
       "types": "./dist/index.d.mts",
       "import": "./dist/index.mjs",
       "require": "./dist/index.js"
-    }
+    },
+    "./examples/*": "./examples/*"
   },
   "files": [
     "dist",

packages/core/src/ChatEngine.ts

@@ -1,450 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { ChatHistory } from "./ChatHistory";
-import { NodeWithScore, TextNode } from "./Node";
-import {
-  CondenseQuestionPrompt,
-  ContextSystemPrompt,
-  defaultCondenseQuestionPrompt,
-  defaultContextSystemPrompt,
-  messagesToHistoryStr,
-} from "./Prompt";
-import { BaseQueryEngine } from "./QueryEngine";
-import { Response } from "./Response";
-import { BaseRetriever } from "./Retriever";
-import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { Event } from "./callbacks/CallbackManager";
-import { ChatMessage, LLM, OpenAI } from "./llm";
-import { BaseNodePostprocessor } from "./postprocessors";
-
-/**
- * A ChatEngine is used to handle back and forth chats between the application and the LLM.
- */
-export interface ChatEngine {
-  /**
-   * Send message along with the class's current chat history to the LLM.
-   * @param message
-   * @param chatHistory optional chat history if you want to customize the chat history
-   * @param streaming optional streaming flag, which auto-sets the return value if True.
-   */
-  chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-    streaming?: T,
-  ): Promise<R>;
-
-  /**
-   * Resets the chat history so that it's empty.
-   */
-  reset(): void;
-}
-
-/**
- * SimpleChatEngine is the simplest possible chat engine. Useful for using your own custom prompts.
- */
-export class SimpleChatEngine implements ChatEngine {
-  chatHistory: ChatMessage[];
-  llm: LLM;
-
-  constructor(init?: Partial<SimpleChatEngine>) {
-    this.chatHistory = init?.chatHistory ?? [];
-    this.llm = init?.llm ?? new OpenAI();
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-    streaming?: T,
-  ): Promise<R> {
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-
-    //Non-streaming option
-    chatHistory = chatHistory ?? this.chatHistory;
-    chatHistory.push({ content: message, role: "user" });
-    const response = await this.llm.chat({ messages: chatHistory });
-    chatHistory.push(response.message);
-    this.chatHistory = chatHistory;
-    return new Response(response.message.content) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-  ): AsyncGenerator<string, void, unknown> {
-    chatHistory = chatHistory ?? this.chatHistory;
-    chatHistory.push({ content: message, role: "user" });
-    const response_generator = await this.llm.chat({
-      messages: chatHistory,
-      stream: true,
-    });
-
-    var accumulator: string = "";
-    for await (const part of response_generator) {
-      accumulator += part.delta;
-      yield part.delta;
-    }
-
-    chatHistory.push({ content: accumulator, role: "assistant" });
-    this.chatHistory = chatHistory;
-    return;
-  }
-
-  reset() {
-    this.chatHistory = [];
-  }
-}
-
-/**
- * CondenseQuestionChatEngine is used in conjunction with a Index (for example VectorStoreIndex).
- * It does two steps on taking a user's chat message: first, it condenses the chat message
- * with the previous chat history into a question with more context.
- * Then, it queries the underlying Index using the new question with context and returns
- * the response.
- * CondenseQuestionChatEngine performs well when the input is primarily questions about the
- * underlying data. It performs less well when the chat messages are not questions about the
- * data, or are very referential to previous context.
- */
-export class CondenseQuestionChatEngine implements ChatEngine {
-  queryEngine: BaseQueryEngine;
-  chatHistory: ChatMessage[];
-  serviceContext: ServiceContext;
-  condenseMessagePrompt: CondenseQuestionPrompt;
-
-  constructor(init: {
-    queryEngine: BaseQueryEngine;
-    chatHistory: ChatMessage[];
-    serviceContext?: ServiceContext;
-    condenseMessagePrompt?: CondenseQuestionPrompt;
-  }) {
-    this.queryEngine = init.queryEngine;
-    this.chatHistory = init?.chatHistory ?? [];
-    this.serviceContext =
-      init?.serviceContext ?? serviceContextFromDefaults({});
-    this.condenseMessagePrompt =
-      init?.condenseMessagePrompt ?? defaultCondenseQuestionPrompt;
-  }
-
-  private async condenseQuestion(chatHistory: ChatMessage[], question: string) {
-    const chatHistoryStr = messagesToHistoryStr(chatHistory);
-
-    return this.serviceContext.llm.complete({
-      prompt: defaultCondenseQuestionPrompt({
-        question: question,
-        chatHistory: chatHistoryStr,
-      }),
-    });
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    const condensedQuestion = (
-      await this.condenseQuestion(chatHistory, extractText(message))
-    ).text;
-
-    const response = await this.queryEngine.query(condensedQuestion);
-
-    chatHistory.push({ content: message, role: "user" });
-    chatHistory.push({ content: response.response, role: "assistant" });
-
-    return response as R;
-  }
-
-  reset() {
-    this.chatHistory = [];
-  }
-}
-
-export interface Context {
-  message: ChatMessage;
-  nodes: NodeWithScore[];
-}
-
-export interface ContextGenerator {
-  generate(message: string, parentEvent?: Event): Promise<Context>;
-}
-
-export class DefaultContextGenerator implements ContextGenerator {
-  retriever: BaseRetriever;
-  contextSystemPrompt: ContextSystemPrompt;
-  nodePostprocessors: BaseNodePostprocessor[];
-
-  constructor(init: {
-    retriever: BaseRetriever;
-    contextSystemPrompt?: ContextSystemPrompt;
-    nodePostprocessors?: BaseNodePostprocessor[];
-  }) {
-    this.retriever = init.retriever;
-    this.contextSystemPrompt =
-      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
-    this.nodePostprocessors = init.nodePostprocessors || [];
-  }
-
-  private applyNodePostprocessors(nodes: NodeWithScore[]) {
-    return this.nodePostprocessors.reduce(
-      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
-      nodes,
-    );
-  }
-
-  async generate(message: string, parentEvent?: Event): Promise<Context> {
-    if (!parentEvent) {
-      parentEvent = {
-        id: randomUUID(),
-        type: "wrapper",
-        tags: ["final"],
-      };
-    }
-    const sourceNodesWithScore = await this.retriever.retrieve(
-      message,
-      parentEvent,
-    );
-
-    const nodes = this.applyNodePostprocessors(sourceNodesWithScore);
-
-    return {
-      message: {
-        content: this.contextSystemPrompt({
-          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
-        }),
-        role: "system",
-      },
-      nodes,
-    };
-  }
-}
-
-/**
- * ContextChatEngine uses the Index to get the appropriate context for each query.
- * The context is stored in the system prompt, and the chat history is preserved,
- * ideally allowing the appropriate context to be surfaced for each query.
- */
-export class ContextChatEngine implements ChatEngine {
-  chatModel: LLM;
-  chatHistory: ChatMessage[];
-  contextGenerator: ContextGenerator;
-
-  constructor(init: {
-    retriever: BaseRetriever;
-    chatModel?: LLM;
-    chatHistory?: ChatMessage[];
-    contextSystemPrompt?: ContextSystemPrompt;
-    nodePostprocessors?: BaseNodePostprocessor[];
-  }) {
-    this.chatModel =
-      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
-    this.chatHistory = init?.chatHistory ?? [];
-    this.contextGenerator = new DefaultContextGenerator({
-      retriever: init.retriever,
-      contextSystemPrompt: init?.contextSystemPrompt,
-    });
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-
-    const parentEvent: Event = {
-      id: randomUUID(),
-      type: "wrapper",
-      tags: ["final"],
-    };
-    const context = await this.contextGenerator.generate(
-      extractText(message),
-      parentEvent,
-    );
-
-    chatHistory.push({ content: message, role: "user" });
-
-    const response = await this.chatModel.chat({
-      messages: [context.message, ...chatHistory],
-      parentEvent,
-    });
-    chatHistory.push(response.message);
-
-    this.chatHistory = chatHistory;
-
-    return new Response(
-      response.message.content,
-      context.nodes.map((r) => r.node),
-    ) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-  ): AsyncGenerator<string, void, unknown> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    const parentEvent: Event = {
-      id: randomUUID(),
-      type: "wrapper",
-      tags: ["final"],
-    };
-    const context = await this.contextGenerator.generate(
-      extractText(message),
-      parentEvent,
-    );
-
-    chatHistory.push({ content: message, role: "user" });
-
-    const response_stream = await this.chatModel.chat({
-      messages: [context.message, ...chatHistory],
-      parentEvent,
-      stream: true,
-    });
-    var accumulator: string = "";
-    for await (const part of response_stream) {
-      accumulator += part.delta;
-      yield part.delta;
-    }
-
-    chatHistory.push({ content: accumulator, role: "assistant" });
-
-    this.chatHistory = chatHistory;
-
-    return;
-  }
-
-  reset() {
-    this.chatHistory = [];
-  }
-}
-
-export interface MessageContentDetail {
-  type: "text" | "image_url";
-  text?: string;
-  image_url?: { url: string };
-}
-
-/**
- * Extended type for the content of a message that allows for multi-modal messages.
- */
-export type MessageContent = string | MessageContentDetail[];
-
-/**
- * Extracts just the text from a multi-modal message or the message itself if it's just text.
- *
- * @param message The message to extract text from.
- * @returns The extracted text
- */
-function extractText(message: MessageContent): string {
-  if (Array.isArray(message)) {
-    // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
-    // so we can pass them to the context generator
-    return (message as MessageContentDetail[])
-      .filter((c) => c.type === "text")
-      .map((c) => c.text)
-      .join("\n\n");
-  }
-  return message;
-}
-
-/**
- * HistoryChatEngine is a ChatEngine that uses a `ChatHistory` object
- * to keeps track of chat's message history.
- * A `ChatHistory` object is passed as a parameter for each call to the `chat` method,
- * so the state of the chat engine is preserved between calls.
- * Optionally, a `ContextGenerator` can be used to generate an additional context for each call to `chat`.
- */
-export class HistoryChatEngine {
-  llm: LLM;
-  contextGenerator?: ContextGenerator;
-
-  constructor(init?: Partial<HistoryChatEngine>) {
-    this.llm = init?.llm ?? new OpenAI();
-    this.contextGenerator = init?.contextGenerator;
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-    streaming?: T,
-  ): Promise<R> {
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-    const requestMessages = await this.prepareRequestMessages(
-      message,
-      chatHistory,
-    );
-    const response = await this.llm.chat({ messages: requestMessages });
-    chatHistory.addMessage(response.message);
-    return new Response(response.message.content) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-  ): AsyncGenerator<string, void, unknown> {
-    const requestMessages = await this.prepareRequestMessages(
-      message,
-      chatHistory,
-    );
-    const response_stream = await this.llm.chat({
-      messages: requestMessages,
-      stream: true,
-    });
-
-    var accumulator = "";
-    for await (const part of response_stream) {
-      accumulator += part.delta;
-      yield part.delta;
-    }
-    chatHistory.addMessage({
-      content: accumulator,
-      role: "assistant",
-    });
-    return;
-  }
-
-  private async prepareRequestMessages(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-  ) {
-    chatHistory.addMessage({
-      content: message,
-      role: "user",
-    });
-    let requestMessages;
-    let context;
-    if (this.contextGenerator) {
-      const textOnly = extractText(message);
-      context = await this.contextGenerator.generate(textOnly);
-    }
-    requestMessages = await chatHistory.requestMessages(
-      context ? [context.message] : undefined,
-    );
-    return requestMessages;
-  }
-}

packages/core/src/ChatHistory.ts

@@ -1,4 +1,5 @@
-import { ChatMessage, LLM, MessageType, OpenAI } from "./llm/LLM";
+import { OpenAI } from "./llm/LLM";
+import { ChatMessage, LLM, MessageType } from "./llm/types";
 import {
   defaultSummaryPrompt,
   messagesToHistoryStr,
@@ -8,35 +9,38 @@ import {
 /**
  * A ChatHistory is used to keep the state of back and forth chat messages
  */
-export interface ChatHistory {
-  messages: ChatMessage[];
+export abstract class ChatHistory {
+  abstract get messages(): ChatMessage[];
   /**
    * Adds a message to the chat history.
    * @param message
    */
-  addMessage(message: ChatMessage): void;
+  abstract addMessage(message: ChatMessage): void;
 
   /**
    * Returns the messages that should be used as input to the LLM.
    */
-  requestMessages(transientMessages?: ChatMessage[]): Promise<ChatMessage[]>;
+  abstract requestMessages(
+    transientMessages?: ChatMessage[],
+  ): Promise<ChatMessage[]>;
 
   /**
    * Resets the chat history so that it's empty.
    */
-  reset(): void;
+  abstract reset(): void;
 
   /**
    * Returns the new messages since the last call to this function (or since calling the constructor)
    */
-  newMessages(): ChatMessage[];
+  abstract newMessages(): ChatMessage[];
 }
 
-export class SimpleChatHistory implements ChatHistory {
+export class SimpleChatHistory extends ChatHistory {
   messages: ChatMessage[];
   private messagesBefore: number;
 
   constructor(init?: Partial<SimpleChatHistory>) {
+    super();
     this.messages = init?.messages ?? [];
     this.messagesBefore = this.messages.length;
   }
@@ -60,7 +64,7 @@ export class SimpleChatHistory implements ChatHistory {
   }
 }
 
-export class SummaryChatHistory implements ChatHistory {
+export class SummaryChatHistory extends ChatHistory {
   tokensToSummarize: number;
   messages: ChatMessage[];
   summaryPrompt: SummaryPrompt;
@@ -68,6 +72,7 @@ export class SummaryChatHistory implements ChatHistory {
   private messagesBefore: number;
 
   constructor(init?: Partial<SummaryChatHistory>) {
+    super();
     this.messages = init?.messages ?? [];
     this.messagesBefore = this.messages.length;
     this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
@@ -79,6 +84,11 @@ export class SummaryChatHistory implements ChatHistory {
     }
     this.tokensToSummarize =
       this.llm.metadata.contextWindow - this.llm.metadata.maxTokens;
+    if (this.tokensToSummarize < this.llm.metadata.contextWindow * 0.25) {
+      throw new Error(
+        "The number of tokens that trigger the summarize process are less than 25% of the context window. Try lowering maxTokens or use a model with a larger context window.",
+      );
+    }
   }
 
   private async summarize(): Promise<ChatMessage> {
@@ -119,6 +129,11 @@ export class SummaryChatHistory implements ChatHistory {
     return this.messages.length - 1 - index;
   }
 
+  public getLastSummary(): ChatMessage | null {
+    const lastSummaryIndex = this.getLastSummaryIndex();
+    return lastSummaryIndex ? this.messages[lastSummaryIndex] : null;
+  }
+
   private get systemMessages() {
     // get array of all system messages
     return this.messages.filter((message) => message.role === "system");
@@ -198,3 +213,12 @@ export class SummaryChatHistory implements ChatHistory {
     return newMessages;
   }
 }
+
+export function getHistory(
+  chatHistory?: ChatMessage[] | ChatHistory,
+): ChatHistory {
+  if (chatHistory instanceof ChatHistory) {
+    return chatHistory;
+  }
+  return new SimpleChatHistory({ messages: chatHistory });
+}

packages/core/src/Prompt.ts

@@ -1,4 +1,4 @@
-import { ChatMessage } from "./llm/LLM";
+import { ChatMessage } from "./llm/types";
 import { SubQuestion } from "./QuestionGenerator";
 import { ToolMetadata } from "./Tool";
 

packages/core/src/PromptHelper.ts

@@ -37,6 +37,7 @@ export class PromptHelper {
   tokenizer: (text: string) => Uint32Array;
   separator = " ";
 
+  // eslint-disable-next-line max-params
   constructor(
     contextWindow = DEFAULT_CONTEXT_WINDOW,
     numOutput = DEFAULT_NUM_OUTPUTS,

packages/core/src/QueryEngine.ts

@@ -17,16 +17,32 @@ import {
   ResponseSynthesizer,
 } from "./synthesizers";
 
+/**
+ * Parameters for sending a query.
+ */
+export interface QueryEngineParamsBase {
+  query: string;
+  parentEvent?: Event;
+}
+
+export interface QueryEngineParamsStreaming extends QueryEngineParamsBase {
+  stream: true;
+}
+
+export interface QueryEngineParamsNonStreaming extends QueryEngineParamsBase {
+  stream?: false | null;
+}
+
 /**
  * A query engine is a question answerer that can use one or more steps.
  */
 export interface BaseQueryEngine {
   /**
    * Query the query engine and get a response.
-   * @param query
-   * @param parentEvent
+   * @param params
    */
-  query(query: string, parentEvent?: Event): Promise<Response>;
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
 }
 
 /**
@@ -70,17 +86,30 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
     return this.applyNodePostprocessors(nodes);
   }
 
-  async query(query: string, parentEvent?: Event) {
-    const _parentEvent: Event = parentEvent || {
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { query, stream } = params;
+    const parentEvent: Event = params.parentEvent || {
       id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
     };
-    const nodesWithScore = await this.retrieve(query, _parentEvent);
+    const nodesWithScore = await this.retrieve(query, parentEvent);
+    if (stream) {
+      return this.responseSynthesizer.synthesize({
+        query,
+        nodesWithScore,
+        parentEvent,
+        stream: true,
+      });
+    }
     return this.responseSynthesizer.synthesize({
       query,
       nodesWithScore,
-      parentEvent: _parentEvent,
+      parentEvent,
     });
   }
 }
@@ -135,11 +164,16 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
     });
   }
 
-  async query(query: string): Promise<Response> {
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { query, stream } = params;
     const subQuestions = await this.questionGen.generate(this.metadatas, query);
 
     // groups final retrieval+synthesis operation
-    const parentEvent: Event = {
+    const parentEvent: Event = params.parentEvent || {
       id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
@@ -160,6 +194,14 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
     const nodesWithScore = subQNodes
       .filter((node) => node !== null)
       .map((node) => node as NodeWithScore);
+    if (stream) {
+      return this.responseSynthesizer.synthesize({
+        query,
+        nodesWithScore,
+        parentEvent,
+        stream: true,
+      });
+    }
     return this.responseSynthesizer.synthesize({
       query,
       nodesWithScore,
@@ -175,7 +217,10 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
       const question = subQ.subQuestion;
       const queryEngine = this.queryEngines[subQ.toolName];
 
-      const response = await queryEngine.query(question, parentEvent);
+      const response = await queryEngine.query({
+        query: question,
+        parentEvent,
+      });
       const responseText = response.response;
       const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
       const node = new TextNode({ text: nodeText });

packages/core/src/QuestionGenerator.ts

@@ -9,7 +9,8 @@ import {
   defaultSubQuestionPrompt,
 } from "./Prompt";
 import { ToolMetadata } from "./Tool";
-import { LLM, OpenAI } from "./llm/LLM";
+import { OpenAI } from "./llm/LLM";
+import { LLM } from "./llm/types";
 
 export interface SubQuestion {
   subQuestion: string;

packages/core/src/embeddings/utils.ts

@@ -66,6 +66,7 @@ export function similarity(
  * @param similarityCutoff minimum similarity score
  * @returns
  */
+// eslint-disable-next-line max-params
 export function getTopKEmbeddings(
   queryEmbedding: number[],
   embeddings: number[][],
@@ -108,6 +109,7 @@ export function getTopKEmbeddings(
   return [resultSimilarities, resultIds];
 }
 
+// eslint-disable-next-line max-params
 export function getTopKEmbeddingsLearner(
   queryEmbedding: number[],
   embeddings: number[][],
@@ -120,6 +122,7 @@ export function getTopKEmbeddingsLearner(
   // https://github.com/mljs/libsvm which itself hasn't been updated in a while
 }
 
+// eslint-disable-next-line max-params
 export function getTopKMMREmbeddings(
   queryEmbedding: number[],
   embeddings: number[][],

packages/core/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -0,0 +1,104 @@
+import { ChatHistory, getHistory } from "../../ChatHistory";
+import {
+  CondenseQuestionPrompt,
+  defaultCondenseQuestionPrompt,
+  messagesToHistoryStr,
+} from "../../Prompt";
+import { BaseQueryEngine } from "../../QueryEngine";
+import { Response } from "../../Response";
+import {
+  ServiceContext,
+  serviceContextFromDefaults,
+} from "../../ServiceContext";
+import { ChatMessage, LLM } from "../../llm";
+import { extractText, streamReducer } from "../../llm/utils";
+import {
+  ChatEngine,
+  ChatEngineParamsNonStreaming,
+  ChatEngineParamsStreaming,
+} from "./types";
+
+/**
+ * CondenseQuestionChatEngine is used in conjunction with a Index (for example VectorStoreIndex).
+ * It does two steps on taking a user's chat message: first, it condenses the chat message
+ * with the previous chat history into a question with more context.
+ * Then, it queries the underlying Index using the new question with context and returns
+ * the response.
+ * CondenseQuestionChatEngine performs well when the input is primarily questions about the
+ * underlying data. It performs less well when the chat messages are not questions about the
+ * data, or are very referential to previous context.
+ */
+
+export class CondenseQuestionChatEngine implements ChatEngine {
+  queryEngine: BaseQueryEngine;
+  chatHistory: ChatHistory;
+  llm: LLM;
+  condenseMessagePrompt: CondenseQuestionPrompt;
+
+  constructor(init: {
+    queryEngine: BaseQueryEngine;
+    chatHistory: ChatMessage[];
+    serviceContext?: ServiceContext;
+    condenseMessagePrompt?: CondenseQuestionPrompt;
+  }) {
+    this.queryEngine = init.queryEngine;
+    this.chatHistory = getHistory(init?.chatHistory);
+    this.llm = init?.serviceContext?.llm ?? serviceContextFromDefaults().llm;
+    this.condenseMessagePrompt =
+      init?.condenseMessagePrompt ?? defaultCondenseQuestionPrompt;
+  }
+
+  private async condenseQuestion(chatHistory: ChatHistory, question: string) {
+    const chatHistoryStr = messagesToHistoryStr(
+      await chatHistory.requestMessages(),
+    );
+
+    return this.llm.complete({
+      prompt: defaultCondenseQuestionPrompt({
+        question: question,
+        chatHistory: chatHistoryStr,
+      }),
+    });
+  }
+
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+    const chatHistory = params.chatHistory
+      ? getHistory(params.chatHistory)
+      : this.chatHistory;
+
+    const condensedQuestion = (
+      await this.condenseQuestion(chatHistory, extractText(message))
+    ).text;
+    chatHistory.addMessage({ content: message, role: "user" });
+
+    if (stream) {
+      const stream = await this.queryEngine.query({
+        query: condensedQuestion,
+        stream: true,
+      });
+      return streamReducer({
+        stream,
+        initialValue: "",
+        reducer: (accumulator, part) => (accumulator += part.response),
+        finished: (accumulator) => {
+          chatHistory.addMessage({ content: accumulator, role: "assistant" });
+        },
+      });
+    }
+    const response = await this.queryEngine.query({
+      query: condensedQuestion,
+    });
+    chatHistory.addMessage({ content: response.response, role: "assistant" });
+
+    return response;
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+}

packages/core/src/engines/chat/ContextChatEngine.ts

@@ -0,0 +1,113 @@
+import { randomUUID } from "node:crypto";
+import { ChatHistory, getHistory } from "../../ChatHistory";
+import { ContextSystemPrompt } from "../../Prompt";
+import { Response } from "../../Response";
+import { BaseRetriever } from "../../Retriever";
+import { Event } from "../../callbacks/CallbackManager";
+import { ChatMessage, ChatResponseChunk, LLM, OpenAI } from "../../llm";
+import { MessageContent } from "../../llm/types";
+import { extractText, streamConverter, streamReducer } from "../../llm/utils";
+import { BaseNodePostprocessor } from "../../postprocessors";
+import { DefaultContextGenerator } from "./DefaultContextGenerator";
+import {
+  ChatEngine,
+  ChatEngineParamsNonStreaming,
+  ChatEngineParamsStreaming,
+  ContextGenerator,
+} from "./types";
+
+/**
+ * ContextChatEngine uses the Index to get the appropriate context for each query.
+ * The context is stored in the system prompt, and the chat history is preserved,
+ * ideally allowing the appropriate context to be surfaced for each query.
+ */
+export class ContextChatEngine implements ChatEngine {
+  chatModel: LLM;
+  chatHistory: ChatHistory;
+  contextGenerator: ContextGenerator;
+
+  constructor(init: {
+    retriever: BaseRetriever;
+    chatModel?: LLM;
+    chatHistory?: ChatMessage[];
+    contextSystemPrompt?: ContextSystemPrompt;
+    nodePostprocessors?: BaseNodePostprocessor[];
+  }) {
+    this.chatModel =
+      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
+    this.chatHistory = getHistory(init?.chatHistory);
+    this.contextGenerator = new DefaultContextGenerator({
+      retriever: init.retriever,
+      contextSystemPrompt: init?.contextSystemPrompt,
+      nodePostprocessors: init?.nodePostprocessors,
+    });
+  }
+
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+    const chatHistory = params.chatHistory
+      ? getHistory(params.chatHistory)
+      : this.chatHistory;
+    const parentEvent: Event = {
+      id: randomUUID(),
+      type: "wrapper",
+      tags: ["final"],
+    };
+    const requestMessages = await this.prepareRequestMessages(
+      message,
+      chatHistory,
+      parentEvent,
+    );
+
+    if (stream) {
+      const stream = await this.chatModel.chat({
+        messages: requestMessages.messages,
+        parentEvent,
+        stream: true,
+      });
+      return streamConverter(
+        streamReducer({
+          stream,
+          initialValue: "",
+          reducer: (accumulator, part) => (accumulator += part.delta),
+          finished: (accumulator) => {
+            chatHistory.addMessage({ content: accumulator, role: "assistant" });
+          },
+        }),
+        (r: ChatResponseChunk) => new Response(r.delta, requestMessages.nodes),
+      );
+    }
+    const response = await this.chatModel.chat({
+      messages: requestMessages.messages,
+      parentEvent,
+    });
+    chatHistory.addMessage(response.message);
+    return new Response(response.message.content, requestMessages.nodes);
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+
+  private async prepareRequestMessages(
+    message: MessageContent,
+    chatHistory: ChatHistory,
+    parentEvent?: Event,
+  ) {
+    chatHistory.addMessage({
+      content: message,
+      role: "user",
+    });
+    const textOnly = extractText(message);
+    const context = await this.contextGenerator.generate(textOnly, parentEvent);
+    const nodes = context.nodes.map((r) => r.node);
+    const messages = await chatHistory.requestMessages(
+      context ? [context.message] : undefined,
+    );
+    return { nodes, messages };
+  }
+}

packages/core/src/engines/chat/DefaultContextGenerator.ts

@@ -0,0 +1,57 @@
+import { randomUUID } from "node:crypto";
+import { NodeWithScore, TextNode } from "../../Node";
+import { ContextSystemPrompt, defaultContextSystemPrompt } from "../../Prompt";
+import { BaseRetriever } from "../../Retriever";
+import { Event } from "../../callbacks/CallbackManager";
+import { BaseNodePostprocessor } from "../../postprocessors";
+import { Context, ContextGenerator } from "./types";
+
+export class DefaultContextGenerator implements ContextGenerator {
+  retriever: BaseRetriever;
+  contextSystemPrompt: ContextSystemPrompt;
+  nodePostprocessors: BaseNodePostprocessor[];
+
+  constructor(init: {
+    retriever: BaseRetriever;
+    contextSystemPrompt?: ContextSystemPrompt;
+    nodePostprocessors?: BaseNodePostprocessor[];
+  }) {
+    this.retriever = init.retriever;
+    this.contextSystemPrompt =
+      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
+    this.nodePostprocessors = init.nodePostprocessors || [];
+  }
+
+  private applyNodePostprocessors(nodes: NodeWithScore[]) {
+    return this.nodePostprocessors.reduce(
+      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
+      nodes,
+    );
+  }
+
+  async generate(message: string, parentEvent?: Event): Promise<Context> {
+    if (!parentEvent) {
+      parentEvent = {
+        id: randomUUID(),
+        type: "wrapper",
+        tags: ["final"],
+      };
+    }
+    const sourceNodesWithScore = await this.retriever.retrieve(
+      message,
+      parentEvent,
+    );
+
+    const nodes = this.applyNodePostprocessors(sourceNodesWithScore);
+
+    return {
+      message: {
+        content: this.contextSystemPrompt({
+          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
+        }),
+        role: "system",
+      },
+      nodes,
+    };
+  }
+}

packages/core/src/engines/chat/SimpleChatEngine.ts

@@ -0,0 +1,64 @@
+import { ChatHistory, getHistory } from "../../ChatHistory";
+import { Response } from "../../Response";
+import { ChatResponseChunk, LLM, OpenAI } from "../../llm";
+import { streamConverter, streamReducer } from "../../llm/utils";
+import {
+  ChatEngine,
+  ChatEngineParamsNonStreaming,
+  ChatEngineParamsStreaming,
+} from "./types";
+
+/**
+ * SimpleChatEngine is the simplest possible chat engine. Useful for using your own custom prompts.
+ */
+
+export class SimpleChatEngine implements ChatEngine {
+  chatHistory: ChatHistory;
+  llm: LLM;
+
+  constructor(init?: Partial<SimpleChatEngine>) {
+    this.chatHistory = getHistory(init?.chatHistory);
+    this.llm = init?.llm ?? new OpenAI();
+  }
+
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+
+    const chatHistory = params.chatHistory
+      ? getHistory(params.chatHistory)
+      : this.chatHistory;
+    chatHistory.addMessage({ content: message, role: "user" });
+
+    if (stream) {
+      const stream = await this.llm.chat({
+        messages: await chatHistory.requestMessages(),
+        stream: true,
+      });
+      return streamConverter(
+        streamReducer({
+          stream,
+          initialValue: "",
+          reducer: (accumulator, part) => (accumulator += part.delta),
+          finished: (accumulator) => {
+            chatHistory.addMessage({ content: accumulator, role: "assistant" });
+          },
+        }),
+        (r: ChatResponseChunk) => new Response(r.delta),
+      );
+    }
+
+    const response = await this.llm.chat({
+      messages: await chatHistory.requestMessages(),
+    });
+    chatHistory.addMessage(response.message);
+    return new Response(response.message.content);
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+}

packages/core/src/engines/chat/index.ts

@@ -0,0 +1,4 @@
+export { CondenseQuestionChatEngine } from "./CondenseQuestionChatEngine";
+export { ContextChatEngine } from "./ContextChatEngine";
+export { SimpleChatEngine } from "./SimpleChatEngine";
+export * from "./types";

packages/core/src/engines/chat/types.ts

@@ -0,0 +1,54 @@
+import { ChatHistory } from "../../ChatHistory";
+import { NodeWithScore } from "../../Node";
+import { Response } from "../../Response";
+import { Event } from "../../callbacks/CallbackManager";
+import { ChatMessage } from "../../llm";
+import { MessageContent } from "../../llm/types";
+
+/**
+ * Represents the base parameters for ChatEngine.
+ */
+export interface ChatEngineParamsBase {
+  message: MessageContent;
+  /**
+   * Optional chat history if you want to customize the chat history.
+   */
+  chatHistory?: ChatMessage[] | ChatHistory;
+}
+
+export interface ChatEngineParamsStreaming extends ChatEngineParamsBase {
+  stream: true;
+}
+
+export interface ChatEngineParamsNonStreaming extends ChatEngineParamsBase {
+  stream?: false | null;
+}
+
+/**
+ * A ChatEngine is used to handle back and forth chats between the application and the LLM.
+ */
+export interface ChatEngine {
+  /**
+   * Send message along with the class's current chat history to the LLM.
+   * @param params
+   */
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+
+  /**
+   * Resets the chat history so that it's empty.
+   */
+  reset(): void;
+}
+
+export interface Context {
+  message: ChatMessage;
+  nodes: NodeWithScore[];
+}
+
+/**
+ * A ContextGenerator is used to generate a context based on a message's text content
+ */
+export interface ContextGenerator {
+  generate(message: string, parentEvent?: Event): Promise<Context>;
+}

packages/core/src/index.ts

@@ -1,4 +1,3 @@
-export * from "./ChatEngine";
 export * from "./ChatHistory";
 export * from "./GlobalsHelper";
 export * from "./Node";
@@ -15,6 +14,7 @@ export * from "./Tool";
 export * from "./callbacks/CallbackManager";
 export * from "./constants";
 export * from "./embeddings";
+export * from "./engines/chat";
 export * from "./indices";
 export * from "./llm";
 export * from "./nodeParsers";

packages/core/src/indices/summary/SummaryIndexRetriever.ts

@@ -61,6 +61,7 @@ export class SummaryIndexLLMRetriever implements BaseRetriever {
   parseChoiceSelectAnswerFn: ChoiceSelectParserFunction;
   serviceContext: ServiceContext;
 
+  // eslint-disable-next-line max-params
   constructor(
     index: SummaryIndex,
     choiceSelectPrompt?: ChoiceSelectPrompt,

packages/core/src/llm/LLM.ts

@@ -24,149 +24,19 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure";
+import { BaseLLM } from "./base";
 import { OpenAISession, getOpenAISession } from "./openai";
 import { PortkeySession, getPortkeySession } from "./portkey";
 import { ReplicateSession } from "./replicate";
-
-export type MessageType =
-  | "user"
-  | "assistant"
-  | "system"
-  | "generic"
-  | "function"
-  | "memory";
-
-export interface ChatMessage {
-  content: any;
-  role: MessageType;
-}
-
-export interface ChatResponse {
-  message: ChatMessage;
-  raw?: Record<string, any>;
-}
-
-export interface ChatResponseChunk {
-  delta: string;
-}
-
-export interface CompletionResponse {
-  text: string;
-  raw?: Record<string, any>;
-}
-
-export interface LLMMetadata {
-  model: string;
-  temperature: number;
-  topP: number;
-  maxTokens?: number;
-  contextWindow: number;
-  tokenizer: Tokenizers | undefined;
-}
-
-export interface LLMChatParamsBase {
-  messages: ChatMessage[];
-  parentEvent?: Event;
-  extraParams?: Record<string, any>;
-}
-
-export interface LLMChatParamsStreaming extends LLMChatParamsBase {
-  stream: true;
-}
-
-export interface LLMChatParamsNonStreaming extends LLMChatParamsBase {
-  stream?: false | null;
-}
-
-export interface LLMCompletionParamsBase {
-  prompt: any;
-  parentEvent?: Event;
-}
-
-export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
-  stream: true;
-}
-
-export interface LLMCompletionParamsNonStreaming
-  extends LLMCompletionParamsBase {
-  stream?: false | null;
-}
-
-/**
- * Unified language model interface
- */
-export interface LLM {
-  metadata: LLMMetadata;
-  /**
-   * Get a chat response from the LLM
-   *
-   * @param params
-   */
-  chat(
-    params: LLMChatParamsStreaming,
-  ): Promise<AsyncIterable<ChatResponseChunk>>;
-  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
-
-  /**
-   * Get a prompt completion from the LLM
-   * @param params
-   */
-  complete(
-    params: LLMCompletionParamsStreaming,
-  ): Promise<AsyncIterable<CompletionResponse>>;
-  complete(
-    params: LLMCompletionParamsNonStreaming,
-  ): Promise<CompletionResponse>;
-
-  /**
-   * Calculates the number of tokens needed for the given chat messages
-   */
-  tokens(messages: ChatMessage[]): number;
-}
-
-export abstract class BaseLLM implements LLM {
-  abstract metadata: LLMMetadata;
-
-  private async *chatToComplete(
-    stream: AsyncIterable<ChatResponseChunk>,
-  ): AsyncIterable<CompletionResponse> {
-    for await (const chunk of stream) {
-      yield { text: chunk.delta };
-    }
-  }
-
-  complete(
-    params: LLMCompletionParamsStreaming,
-  ): Promise<AsyncIterable<CompletionResponse>>;
-  complete(
-    params: LLMCompletionParamsNonStreaming,
-  ): Promise<CompletionResponse>;
-  async complete(
-    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
-  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
-    const { prompt, parentEvent, stream } = params;
-    if (stream) {
-      const stream = await this.chat({
-        messages: [{ content: prompt, role: "user" }],
-        parentEvent,
-        stream: true,
-      });
-      return this.chatToComplete(stream);
-    }
-    const chatResponse = await this.chat({
-      messages: [{ content: prompt, role: "user" }],
-      parentEvent,
-    });
-    return { text: chatResponse.message.content as string };
-  }
-
-  abstract chat(
-    params: LLMChatParamsStreaming,
-  ): Promise<AsyncIterable<ChatResponseChunk>>;
-  abstract chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
-
-  abstract tokens(messages: ChatMessage[]): number;
-}
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMMetadata,
+  MessageType,
+} from "./types";
 
 export const GPT4_MODELS = {
   "gpt-4": { contextWindow: 8192 },
@@ -392,6 +262,7 @@ export class OpenAI extends BaseLLM {
           type: "llmPredict" as EventType,
         };
 
+    // TODO: add callback to streamConverter and use streamConverter here
     //Indices
     var idx_counter: number = 0;
     for await (const part of chunk_stream) {

packages/core/src/llm/base.ts

@@ -0,0 +1,53 @@
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+} from "./types";
+import { streamConverter } from "./utils";
+
+export abstract class BaseLLM implements LLM {
+  abstract metadata: LLMMetadata;
+
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const { prompt, parentEvent, stream } = params;
+    if (stream) {
+      const stream = await this.chat({
+        messages: [{ content: prompt, role: "user" }],
+        parentEvent,
+        stream: true,
+      });
+      return streamConverter(stream, (chunk) => {
+        return {
+          text: chunk.delta,
+        };
+      });
+    }
+    const chatResponse = await this.chat({
+      messages: [{ content: prompt, role: "user" }],
+      parentEvent,
+    });
+    return { text: chatResponse.message.content as string };
+  }
+
+  abstract chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  abstract chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+
+  abstract tokens(messages: ChatMessage[]): number;
+}

packages/core/src/llm/index.ts

@@ -2,3 +2,4 @@ export * from "./LLM";
 export * from "./mistral";
 export { Ollama } from "./ollama";
 export { TogetherLLM } from "./together";
+export * from "./types";

packages/core/src/llm/mistral.ts

@@ -4,14 +4,14 @@ import {
   EventType,
   StreamCallbackResponse,
 } from "../callbacks/CallbackManager";
+import { BaseLLM } from "./base";
 import {
-  BaseLLM,
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
-} from "./LLM";
+} from "./types";
 
 export const ALL_AVAILABLE_MISTRAL_MODELS = {
   "mistral-tiny": { contextWindow: 32000 },

packages/core/src/llm/ollama.ts

@@ -12,7 +12,7 @@ import {
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
   LLMMetadata,
-} from "./LLM";
+} from "./types";
 
 const messageAccessor = (data: any): ChatResponseChunk => {
   return {

packages/core/src/llm/types.ts

@@ -0,0 +1,110 @@
+import { Tokenizers } from "../GlobalsHelper";
+import { Event } from "../callbacks/CallbackManager";
+
+/**
+ * Unified language model interface
+ */
+export interface LLM {
+  metadata: LLMMetadata;
+  /**
+   * Get a chat response from the LLM
+   *
+   * @param params
+   */
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+
+  /**
+   * Get a prompt completion from the LLM
+   * @param params
+   */
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+
+  /**
+   * Calculates the number of tokens needed for the given chat messages
+   */
+  tokens(messages: ChatMessage[]): number;
+}
+
+export type MessageType =
+  | "user"
+  | "assistant"
+  | "system"
+  | "generic"
+  | "function"
+  | "memory";
+
+export interface ChatMessage {
+  // TODO: use MessageContent
+  content: any;
+  role: MessageType;
+}
+
+export interface ChatResponse {
+  message: ChatMessage;
+  raw?: Record<string, any>;
+}
+
+export interface ChatResponseChunk {
+  delta: string;
+}
+
+export interface CompletionResponse {
+  text: string;
+  raw?: Record<string, any>;
+}
+
+export interface LLMMetadata {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  contextWindow: number;
+  tokenizer: Tokenizers | undefined;
+}
+
+export interface LLMChatParamsBase {
+  messages: ChatMessage[];
+  parentEvent?: Event;
+  extraParams?: Record<string, any>;
+}
+
+export interface LLMChatParamsStreaming extends LLMChatParamsBase {
+  stream: true;
+}
+
+export interface LLMChatParamsNonStreaming extends LLMChatParamsBase {
+  stream?: false | null;
+}
+
+export interface LLMCompletionParamsBase {
+  prompt: any;
+  parentEvent?: Event;
+}
+
+export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
+  stream: true;
+}
+
+export interface LLMCompletionParamsNonStreaming
+  extends LLMCompletionParamsBase {
+  stream?: false | null;
+}
+
+export interface MessageContentDetail {
+  type: "text" | "image_url";
+  text?: string;
+  image_url?: { url: string };
+}
+
+/**
+ * Extended type for the content of a message that allows for multi-modal messages.
+ */
+export type MessageContent = string | MessageContentDetail[];

packages/core/src/llm/utils.ts

@@ -1,4 +1,4 @@
-// TODO: use for LLM.ts
+import { MessageContent, MessageContentDetail } from "./types";
 
 export async function* streamConverter<S, D>(
   stream: AsyncIterable<S>,
@@ -8,3 +8,37 @@ export async function* streamConverter<S, D>(
     yield converter(data);
   }
 }
+
+export async function* streamReducer<S, D>(params: {
+  stream: AsyncIterable<S>;
+  reducer: (previousValue: D, currentValue: S) => D;
+  initialValue: D;
+  finished?: (value: D | undefined) => void;
+}): AsyncIterable<S> {
+  let value = params.initialValue;
+  for await (const data of params.stream) {
+    value = params.reducer(value, data);
+    yield data;
+  }
+  if (params.finished) {
+    params.finished(value);
+  }
+}
+/**
+ * Extracts just the text from a multi-modal message or the message itself if it's just text.
+ *
+ * @param message The message to extract text from.
+ * @returns The extracted text
+ */
+
+export function extractText(message: MessageContent): string {
+  if (Array.isArray(message)) {
+    // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
+    // so we can pass them to the context generator
+    return (message as MessageContentDetail[])
+      .filter((c) => c.type === "text")
+      .map((c) => c.text)
+      .join("\n\n");
+  }
+  return message;
+}

packages/core/src/readers/SimpleMongoReader.ts

@@ -36,6 +36,7 @@ export class SimpleMongoReader implements BaseReader {
    * @returns {Promise<Document[]>}
    * @throws If a field specified in fieldNames or metadataNames is not found in a MongoDB document.
    */
+  // eslint-disable-next-line max-params
   public async loadData(
     dbName: string,
     collectionName: string,

packages/core/src/storage/StorageContext.ts

@@ -1,4 +1,4 @@
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "./FileSystem";
 import {
   DEFAULT_FS,

packages/core/src/storage/docStore/SimpleDocumentStore.ts

@@ -1,5 +1,5 @@
 import _ from "lodash";
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "../FileSystem";
 import {
   DEFAULT_DOC_STORE_PERSIST_FILENAME,

packages/core/src/storage/indexStore/SimpleIndexStore.ts

@@ -1,4 +1,4 @@
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "../FileSystem";
 import {
   DEFAULT_FS,

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -1,5 +1,5 @@
 import _ from "lodash";
-import * as path from "path";
+import path from "path";
 import { BaseNode } from "../../Node";
 import {
   getTopKEmbeddings,

packages/core/src/synthesizers/MultiModalResponseSynthesizer.ts

@@ -1,8 +1,8 @@
-import { MessageContentDetail } from "../ChatEngine";
 import { ImageNode, MetadataMode, splitNodesByType } from "../Node";
 import { Response } from "../Response";
 import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext";
 import { imageToDataUrl } from "../embeddings";
+import { MessageContentDetail } from "../llm/types";
 import { TextQaPrompt, defaultTextQaPrompt } from "./../Prompt";
 import {
   BaseSynthesizer,

packages/core/src/synthesizers/builders.ts

@@ -168,6 +168,7 @@ export class Refine implements ResponseBuilder {
     return response;
   }
 
+  // eslint-disable-next-line max-params
   private async refineResponseSingle(
     initialReponse: string,
     queryStr: string,

packages/core/src/tests/CallbackManager.test.ts

@@ -67,7 +67,7 @@ describe("CallbackManager: onLLMStream and onRetrieve", () => {
     });
     const queryEngine = vectorStoreIndex.asQueryEngine();
     const query = "What is the author's name?";
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
     expect(response.toString()).toBe("MOCK_TOKEN_1-MOCK_TOKEN_2");
     expect(streamCallbackData).toEqual([
       {
@@ -145,7 +145,7 @@ describe("CallbackManager: onLLMStream and onRetrieve", () => {
       responseSynthesizer,
     });
     const query = "What is the author's name?";
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
     expect(response.toString()).toBe("MOCK_TOKEN_1-MOCK_TOKEN_2");
     expect(streamCallbackData).toEqual([
       {

packages/core/src/tests/utility/mockOpenAI.ts

@@ -1,7 +1,8 @@
 import { CallbackManager } from "../../callbacks/CallbackManager";
 import { OpenAIEmbedding } from "../../embeddings";
 import { globalsHelper } from "../../GlobalsHelper";
-import { LLMChatParamsBase, OpenAI } from "../../llm/LLM";
+import { OpenAI } from "../../llm/LLM";
+import { LLMChatParamsBase } from "../../llm/types";
 
 export function mockLlmGeneration({
   languageModel,

packages/create-llama/CHANGELOG.md

@@ -1,5 +1,13 @@
 # create-llama
 
+## 0.0.16
+
+### Patch Changes
+
+- a73942d: Fix: Bundle mongo dependency with NextJS
+- 9492cc6: Feat: Added option to automatically install dependencies (for Python and TS)
+- f74dea5: Feat: Show images in chat messages using GPT4 Vision (Express and NextJS only)
+
 ## 0.0.15
 
 ### Patch Changes

packages/create-llama/create-app.ts

@@ -34,6 +34,7 @@ export async function createApp({
   communityProjectPath,
   vectorDb,
   externalPort,
+  installDependencies,
 }: InstallAppArgs): Promise<void> {
   const root = path.resolve(appPath);
 
@@ -75,6 +76,7 @@ export async function createApp({
     communityProjectPath,
     vectorDb,
     externalPort,
+    installDependencies,
   };
 
   if (frontend) {

packages/create-llama/e2e/utils.ts

@@ -7,6 +7,7 @@ import waitPort from "wait-port";
 export type AppType = "--frontend" | "--no-frontend" | "";
 const MODEL = "gpt-3.5-turbo";
 
+// eslint-disable-next-line max-params
 export async function runApp(
   cwd: string,
   name: string,
@@ -65,6 +66,7 @@ async function createProcess(command: string, cwd: string, port: number) {
   return cp;
 }
 
+// eslint-disable-next-line max-params
 export function runCreateLlama(
   cwd: string,
   templateType: string,
@@ -104,6 +106,7 @@ export function runCreateLlama(
     "--use-npm",
     "--external-port",
     externalPort,
+    "--install-dependencies",
   ].join(" ");
   console.log(`running command '${command}' in ${cwd}`);
   execSync(command, {

packages/create-llama/helpers/index.ts

@@ -7,6 +7,7 @@ import { cyan } from "picocolors";
 
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./constant";
 import { PackageManager } from "./get-pkg-manager";
+import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry";
 import { installPythonTemplate } from "./python";
 import { downloadAndExtractRepo } from "./repo";
 import {
@@ -69,6 +70,7 @@ const copyTestData = async (
   engine?: TemplateEngine,
   openAiKey?: string,
   vectorDb?: TemplateVectorDB,
+  // eslint-disable-next-line max-params
 ) => {
   if (engine === "context") {
     const srcPath = path.join(
@@ -89,18 +91,23 @@ const copyTestData = async (
   if (packageManager && engine === "context") {
     const runGenerate = `${cyan(
       framework === "fastapi"
-        ? "python app/engine/generate.py"
+        ? "poetry run python app/engine/generate.py"
         : `${packageManager} run generate`,
     )}`;
     const hasOpenAiKey = openAiKey || process.env["OPENAI_API_KEY"];
     const hasVectorDb = vectorDb && vectorDb !== "none";
-    const shouldRunGenerateAfterInstall =
-      hasOpenAiKey && framework !== "fastapi" && vectorDb === "none";
-    if (shouldRunGenerateAfterInstall) {
-      console.log(`\nRunning ${runGenerate} to generate the context data.\n`);
-      await callPackageManager(packageManager, true, ["run", "generate"]);
-      console.log();
-      return;
+    if (framework === "fastapi") {
+      if (hasOpenAiKey && vectorDb === "none" && isHavingPoetryLockFile()) {
+        console.log(`Running ${runGenerate} to generate the context data.`);
+        tryPoetryRun("python app/engine/generate.py");
+        return;
+      }
+    } else {
+      if (hasOpenAiKey && vectorDb === "none") {
+        console.log(`Running ${runGenerate} to generate the context data.`);
+        await callPackageManager(packageManager, true, ["run", "generate"]);
+        return;
+      }
     }
 
     const settings = [];

packages/create-llama/helpers/poetry.ts

@@ -0,0 +1,34 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { execSync } from "child_process";
+import fs from "fs";
+
+export function isPoetryAvailable(): boolean {
+  try {
+    execSync("poetry --version", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function tryPoetryInstall(): boolean {
+  try {
+    execSync("poetry install", { stdio: "inherit" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function tryPoetryRun(command: string): boolean {
+  try {
+    execSync(`poetry run ${command}`, { stdio: "inherit" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function isHavingPoetryLockFile(): boolean {
+  try {
+    return fs.existsSync("poetry.lock");
+  } catch (_) {}
+  return false;
+}

packages/create-llama/helpers/python.ts

@@ -1,8 +1,10 @@
 import fs from "fs/promises";
 import path from "path";
-import { cyan } from "picocolors";
+import { cyan, yellow } from "picocolors";
 import { parse, stringify } from "smol-toml";
+import terminalLink from "terminal-link";
 import { copy } from "./copy";
+import { isPoetryAvailable, tryPoetryInstall } from "./poetry";
 import { InstallTemplateArgs, TemplateVectorDB } from "./types";
 
 interface Dependency {
@@ -96,9 +98,15 @@ export const installPythonTemplate = async ({
   framework,
   engine,
   vectorDb,
+  installDependencies,
 }: Pick<
   InstallTemplateArgs,
-  "root" | "framework" | "template" | "engine" | "vectorDb"
+  | "root"
+  | "framework"
+  | "template"
+  | "engine"
+  | "vectorDb"
+  | "installDependencies"
 >) => {
   console.log("\nInitializing Python project with template:", template, "\n");
   const templatePath = path.join(
@@ -146,7 +154,28 @@ export const installPythonTemplate = async ({
   const addOnDependencies = getAdditionalDependencies(vectorDb);
   await addDependencies(root, addOnDependencies);
 
-  console.log(
-    "\nPython project, dependencies won't be installed automatically.\n",
-  );
+  // install python dependencies
+  if (installDependencies) {
+    if (isPoetryAvailable()) {
+      console.log(
+        `Installing python dependencies using poetry. This may take a while...`,
+      );
+      const installSuccessful = tryPoetryInstall();
+      if (!installSuccessful) {
+        console.warn(
+          yellow("Install failed. Please install dependencies manually."),
+        );
+      }
+    } else {
+      console.warn(
+        yellow(
+          `Poetry is not available in the current environment. The Python dependencies will not be installed automatically.
+Please check ${terminalLink(
+            "Poetry Installation",
+            `https://python-poetry.org/docs/#installation`,
+          )} to install poetry first, then install the dependencies manually.`,
+        ),
+      );
+    }
+  }
 };

packages/create-llama/helpers/types.ts

@@ -23,4 +23,5 @@ export interface InstallTemplateArgs {
   communityProjectPath?: string;
   vectorDb?: TemplateVectorDB;
   externalPort?: number;
+  installDependencies?: boolean;
 }

packages/create-llama/helpers/typescript.ts

@@ -39,6 +39,7 @@ export const installTSTemplate = async ({
   customApiPath,
   forBackend,
   vectorDb,
+  installDependencies,
 }: InstallTemplateArgs) => {
   console.log(bold(`Using ${packageManager}.`));
 
@@ -210,15 +211,17 @@ export const installTSTemplate = async ({
     JSON.stringify(packageJson, null, 2) + os.EOL,
   );
 
-  console.log("\nInstalling dependencies:");
-  for (const dependency in packageJson.dependencies)
-    console.log(`- ${cyan(dependency)}`);
+  if (installDependencies) {
+    console.log("\nInstalling dependencies:");
+    for (const dependency in packageJson.dependencies)
+      console.log(`- ${cyan(dependency)}`);
 
-  console.log("\nInstalling devDependencies:");
-  for (const dependency in packageJson.devDependencies)
-    console.log(`- ${cyan(dependency)}`);
+    console.log("\nInstalling devDependencies:");
+    for (const dependency in packageJson.devDependencies)
+      console.log(`- ${cyan(dependency)}`);
 
-  console.log();
+    console.log();
 
-  await callPackageManager(packageManager, isOnline);
+    await callPackageManager(packageManager, isOnline);
+  }
 };

packages/create-llama/index.ts

@@ -117,6 +117,13 @@ const program = new Commander.Command(packageJson.name)
     `
 
 Select external port.
+`,
+  )
+  .option(
+    "--install-dependencies",
+    `
+
+Whether install dependencies (backend/frontend) automatically or not.
 `,
   )
   .allowUnknownOption()
@@ -224,6 +231,7 @@ async function run(): Promise<void> {
     communityProjectPath: program.communityProjectPath,
     vectorDb: program.vectorDb,
     externalPort: program.externalPort,
+    installDependencies: program.installDependencies,
   });
   conf.set("preferences", preferences);
 }

packages/create-llama/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "keywords": [
     "rag",
     "llamaindex",

packages/create-llama/questions.ts

@@ -211,6 +211,19 @@ export const askQuestions = async (
     }
   }
 
+  if (program.installDependencies === undefined) {
+    const { installDependencies } = await prompts({
+      onState: onPromptState,
+      type: "toggle",
+      name: "installDependencies",
+      message: `Would you like to install dependencies automatically? This may take a while`,
+      initial: getPrefOrDefault("installDependencies"),
+      active: "Yes",
+      inactive: "No",
+    });
+    program.installDependencies = Boolean(installDependencies);
+  }
+
   if (!program.model) {
     if (ciInfo.isCI) {
       program.model = getPrefOrDefault("model");

packages/create-llama/templates/types/simple/express/src/controllers/chat.controller.ts

@@ -2,7 +2,7 @@ import { Request, Response } from "express";
 import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
 import { createChatEngine } from "./engine";
 
-const getLastMessageContent = (
+const convertMessageContent = (
   textMessage: string,
   imageUrl: string | undefined,
 ): MessageContent => {
@@ -24,8 +24,8 @@ const getLastMessageContent = (
 export const chat = async (req: Request, res: Response) => {
   try {
     const { messages, data }: { messages: ChatMessage[]; data: any } = req.body;
-    const lastMessage = messages.pop();
-    if (!messages || !lastMessage || lastMessage.role !== "user") {
+    const userMessage = messages.pop();
+    if (!messages || !userMessage || userMessage.role !== "user") {
       return res.status(400).json({
         error:
           "messages are required in the request body and the last message must be from the user",
@@ -36,17 +36,20 @@ export const chat = async (req: Request, res: Response) => {
       model: process.env.MODEL || "gpt-3.5-turbo",
     });
 
-    const lastMessageContent = getLastMessageContent(
-      lastMessage.content,
+    // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
+    // Note: The non-streaming template does not need the Vercel/AI format, we're still using it for consistency with the streaming template
+    const userMessageContent = convertMessageContent(
+      userMessage.content,
       data?.imageUrl,
     );
 
     const chatEngine = await createChatEngine(llm);
 
-    const response = await chatEngine.chat(
-      lastMessageContent as MessageContent,
+    // Calling LlamaIndex's ChatEngine to get a response
+    const response = await chatEngine.chat({
+      message: userMessageContent,
       messages,
-    );
+    });
     const result: ChatMessage = {
       role: "assistant",
       content: response.response,

packages/create-llama/templates/types/streaming/express/src/controllers/chat.controller.ts

@@ -4,7 +4,7 @@ import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
 import { createChatEngine } from "./engine";
 import { LlamaIndexStream } from "./llamaindex-stream";
 
-const getLastMessageContent = (
+const convertMessageContent = (
   textMessage: string,
   imageUrl: string | undefined,
 ): MessageContent => {
@@ -26,8 +26,8 @@ const getLastMessageContent = (
 export const chat = async (req: Request, res: Response) => {
   try {
     const { messages, data }: { messages: ChatMessage[]; data: any } = req.body;
-    const lastMessage = messages.pop();
-    if (!messages || !lastMessage || lastMessage.role !== "user") {
+    const userMessage = messages.pop();
+    if (!messages || !userMessage || userMessage.role !== "user") {
       return res.status(400).json({
         error:
           "messages are required in the request body and the last message must be from the user",
@@ -35,26 +35,43 @@ export const chat = async (req: Request, res: Response) => {
     }
 
     const llm = new OpenAI({
-      model: process.env.MODEL || "gpt-3.5-turbo",
+      model: (process.env.MODEL as any) || "gpt-3.5-turbo",
     });
 
     const chatEngine = await createChatEngine(llm);
 
-    const lastMessageContent = getLastMessageContent(
-      lastMessage.content,
+    // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
+    const userMessageContent = convertMessageContent(
+      userMessage.content,
       data?.imageUrl,
     );
 
-    const response = await chatEngine.chat(
-      lastMessageContent as MessageContent,
-      messages,
-      true,
-    );
+    // Calling LlamaIndex's ChatEngine to get a streamed response
+    const response = await chatEngine.chat({
+      message: userMessageContent,
+      chatHistory: messages,
+      stream: true,
+    });
 
-    // Transform the response into a readable stream
-    const stream = LlamaIndexStream(response);
+    // Return a stream, which can be consumed by the Vercel/AI client
+    const { stream, data: streamData } = LlamaIndexStream(response, {
+      parserOptions: {
+        image_url: data?.imageUrl,
+      },
+    });
 
-    streamToResponse(stream, res);
+    // Pipe LlamaIndexStream to response
+    const processedStream = stream.pipeThrough(streamData.stream);
+    return streamToResponse(processedStream, res, {
+      headers: {
+        // response MUST have the `X-Experimental-Stream-Data: 'true'` header
+        // so that the client uses the correct parsing logic, see
+        // https://sdk.vercel.ai/docs/api-reference/stream-data#on-the-server
+        "X-Experimental-Stream-Data": "true",
+        "Content-Type": "text/plain; charset=utf-8",
+        "Access-Control-Expose-Headers": "X-Experimental-Stream-Data",
+      },
+    });
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return res.status(500).json({

packages/create-llama/templates/types/streaming/express/src/controllers/llamaindex-stream.ts

@@ -1,21 +1,49 @@
 import {
+  JSONValue,
   createCallbacksTransformer,
   createStreamDataTransformer,
+  experimental_StreamData,
   trimStartOfStreamHelper,
   type AIStreamCallbacksAndOptions,
 } from "ai";
+import { Response } from "llamaindex";
 
-function createParser(res: AsyncGenerator<any>) {
+type ParserOptions = {
+  image_url?: string;
+};
+
+function createParser(
+  res: AsyncIterable<Response>,
+  data: experimental_StreamData,
+  opts?: ParserOptions,
+) {
+  const it = res[Symbol.asyncIterator]();
   const trimStartOfStream = trimStartOfStreamHelper();
   return new ReadableStream<string>({
+    start() {
+      // if image_url is provided, send it via the data stream
+      if (opts?.image_url) {
+        const message: JSONValue = {
+          type: "image_url",
+          image_url: {
+            url: opts.image_url,
+          },
+        };
+        data.append(message);
+      } else {
+        data.append({}); // send an empty image response for the user's message
+      }
+    },
     async pull(controller): Promise<void> {
-      const { value, done } = await res.next();
+      const { value, done } = await it.next();
       if (done) {
         controller.close();
+        data.append({}); // send an empty image response for the assistant's message
+        data.close();
         return;
       }
 
-      const text = trimStartOfStream(value ?? "");
+      const text = trimStartOfStream(value.response ?? "");
       if (text) {
         controller.enqueue(text);
       }
@@ -24,12 +52,17 @@ function createParser(res: AsyncGenerator<any>) {
 }
 
 export function LlamaIndexStream(
-  res: AsyncGenerator<any>,
-  callbacks?: AIStreamCallbacksAndOptions,
-): ReadableStream {
-  return createParser(res)
-    .pipeThrough(createCallbacksTransformer(callbacks))
-    .pipeThrough(
-      createStreamDataTransformer(callbacks?.experimental_streamData),
-    );
+  res: AsyncIterable<Response>,
+  opts?: {
+    callbacks?: AIStreamCallbacksAndOptions;
+    parserOptions?: ParserOptions;
+  },
+): { stream: ReadableStream; data: experimental_StreamData } {
+  const data = new experimental_StreamData();
+  return {
+    stream: createParser(res, data, opts?.parserOptions)
+      .pipeThrough(createCallbacksTransformer(opts?.callbacks))
+      .pipeThrough(createStreamDataTransformer(true)),
+    data,
+  };
 }

packages/create-llama/templates/types/streaming/nextjs/app/api/chat/llamaindex-stream.ts

@@ -6,16 +6,18 @@ import {
   trimStartOfStreamHelper,
   type AIStreamCallbacksAndOptions,
 } from "ai";
+import { Response } from "llamaindex";
 
 type ParserOptions = {
   image_url?: string;
 };
 
 function createParser(
-  res: AsyncGenerator<any>,
+  res: AsyncIterable<Response>,
   data: experimental_StreamData,
   opts?: ParserOptions,
 ) {
+  const it = res[Symbol.asyncIterator]();
   const trimStartOfStream = trimStartOfStreamHelper();
   return new ReadableStream<string>({
     start() {
@@ -33,7 +35,7 @@ function createParser(
       }
     },
     async pull(controller): Promise<void> {
-      const { value, done } = await res.next();
+      const { value, done } = await it.next();
       if (done) {
         controller.close();
         data.append({}); // send an empty image response for the assistant's message
@@ -41,7 +43,7 @@ function createParser(
         return;
       }
 
-      const text = trimStartOfStream(value ?? "");
+      const text = trimStartOfStream(value.response ?? "");
       if (text) {
         controller.enqueue(text);
       }
@@ -50,7 +52,7 @@ function createParser(
 }
 
 export function LlamaIndexStream(
-  res: AsyncGenerator<any>,
+  res: AsyncIterable<Response>,
   opts?: {
     callbacks?: AIStreamCallbacksAndOptions;
     parserOptions?: ParserOptions;

packages/create-llama/templates/types/streaming/nextjs/app/api/chat/route.ts

@@ -1,4 +1,4 @@
-import { Message, StreamingTextResponse } from "ai";
+import { StreamingTextResponse } from "ai";
 import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
 import { NextRequest, NextResponse } from "next/server";
 import { createChatEngine } from "./engine";
@@ -7,7 +7,7 @@ import { LlamaIndexStream } from "./llamaindex-stream";
 export const runtime = "nodejs";
 export const dynamic = "force-dynamic";
 
-const getLastMessageContent = (
+const convertMessageContent = (
   textMessage: string,
   imageUrl: string | undefined,
 ): MessageContent => {
@@ -29,9 +29,9 @@ const getLastMessageContent = (
 export async function POST(request: NextRequest) {
   try {
     const body = await request.json();
-    const { messages, data }: { messages: Message[]; data: any } = body;
-    const lastMessage = messages.pop();
-    if (!messages || !lastMessage || lastMessage.role !== "user") {
+    const { messages, data }: { messages: ChatMessage[]; data: any } = body;
+    const userMessage = messages.pop();
+    if (!messages || !userMessage || userMessage.role !== "user") {
       return NextResponse.json(
         {
           error:
@@ -48,25 +48,27 @@ export async function POST(request: NextRequest) {
 
     const chatEngine = await createChatEngine(llm);
 
-    const lastMessageContent = getLastMessageContent(
-      lastMessage.content,
+    // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
+    const userMessageContent = convertMessageContent(
+      userMessage.content,
       data?.imageUrl,
     );
 
-    const response = await chatEngine.chat(
-      lastMessageContent as MessageContent,
-      messages as ChatMessage[],
-      true,
-    );
+    // Calling LlamaIndex's ChatEngine to get a streamed response
+    const response = await chatEngine.chat({
+      message: userMessageContent,
+      chatHistory: messages,
+      stream: true,
+    });
 
-    // Transform the response into a readable stream
+    // Transform LlamaIndex stream to Vercel/AI format
     const { stream, data: streamData } = LlamaIndexStream(response, {
       parserOptions: {
         image_url: data?.imageUrl,
       },
     });
 
-    // Return a StreamingTextResponse, which can be consumed by the client
+    // Return a StreamingTextResponse, which can be consumed by the Vercel/AI client
     return new StreamingTextResponse(stream, {}, streamData);
   } catch (error) {
     console.error("[LlamaIndex]", error);

packages/create-llama/templates/types/streaming/nextjs/next.config.app.js

@@ -6,7 +6,6 @@ const nextConfig = {
       ...config.resolve.alias,
       sharp$: false,
       "onnxruntime-node$": false,
-      mongodb$: false,
     };
     return config;
   },

packages/create-llama/templates/types/streaming/nextjs/next.config.static.js

@@ -8,7 +8,6 @@ const nextConfig = {
       ...config.resolve.alias,
       sharp$: false,
       "onnxruntime-node$": false,
-      mongodb$: false,
     };
     return config;
   },