llamaindex@0.0.47

Releases:
  llamaindex@0.0.46
  create-llama@0.0.15

[skip ci]

.changeset/silly-bugs-fold.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Added option to automatically install dependencies (for Python and TS)

.changeset/spicy-colts-dream.md

@@ -1,5 +0,0 @@
----
-"create-llama": patch
----
-
-feat: support showing image on chat message

.changeset/tame-planes-relax.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-refactor: Updated low-level streaming interface

.github/workflows/e2e.yml

@@ -8,6 +8,9 @@ on:
       - ".github/workflows/e2e.yml"
     branches: [main]
 
+env:
+  POETRY_VERSION: "1.6.1"
+
 jobs:
   e2e:
     name: create-llama
@@ -16,10 +19,19 @@ jobs:
       fail-fast: true
       matrix:
         node-version: [18, 20]
+        python-version: ["3.11"]
         os: [macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
       - uses: pnpm/action-setup@v2
       - name: Setup Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4

apps/docs/docs/modules/high_level/chat_engine.md

@@ -11,7 +11,16 @@ const retriever = index.asRetriever();
 const chatEngine = new ContextChatEngine({ retriever });
 
 // start chatting
-const response = await chatEngine.chat(query);
+const response = await chatEngine.chat({ message: query });
+```
+
+The `chat` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await chatEngine.chat({ message: query, stream: true });
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
 ```
 
 ## Api References

apps/docs/docs/modules/high_level/query_engine.md

@@ -8,7 +8,16 @@ A query engine wraps a `Retriever` and a `ResponseSynthesizer` into a pipeline,
 
 ```typescript
 const queryEngine = index.asQueryEngine();
-const response = await queryEngine.query("query string");
+const response = await queryEngine.query({ query: "query string" });
+```
+
+The `query` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await queryEngine.query({ query: "query string", stream: true });
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
 ```
 
 ## Sub Question Query Engine

examples/astradb/query.ts

@@ -18,7 +18,9 @@ async function main() {
 
     const queryEngine = await index.asQueryEngine({ retriever });
 
-    const results = await queryEngine.query("What is the best reviewed movie?");
+    const results = await queryEngine.query({
+      query: "What is the best reviewed movie?",
+    });
 
     console.log(results.response);
   } catch (e) {

examples/chatEngine.ts

@@ -25,8 +25,11 @@ async function main() {
 
   while (true) {
     const query = await rl.question("Query: ");
-    const response = await chatEngine.chat(query);
-    console.log(response.toString());
+    const stream = await chatEngine.chat({ message: query, stream: true });
+    console.log();
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.response);
+    }
   }
 }
 

examples/chromadb/test.ts

@@ -28,9 +28,9 @@ async function main() {
 
     console.log("Querying index");
     const queryEngine = index.asQueryEngine();
-    const response = await queryEngine.query(
-      "Tell me about Godfrey Cheshire's rating of La Sapienza.",
-    );
+    const response = await queryEngine.query({
+      query: "Tell me about Godfrey Cheshire's rating of La Sapienza.",
+    });
     console.log(response.toString());
   } catch (e) {
     console.error(e);

examples/huggingface.ts

@@ -32,12 +32,15 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const stream = await queryEngine.query({
+    query: "What did the author do in college?",
+    stream: true,
+  });
 
   // Output response
-  console.log(response.toString());
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.response);
+  }
 }
 
 main().catch(console.error);

examples/keywordIndex.ts

@@ -20,9 +20,9 @@ async function main() {
         mode,
       }),
     });
-    const response = await queryEngine.query(
-      "What did the author do growing up?",
-    );
+    const response = await queryEngine.query({
+      query: "What did the author do growing up?",
+    });
     console.log(response.toString());
   });
 }

examples/markdown.ts

@@ -11,7 +11,9 @@ async function main() {
   // Query the index
   const queryEngine = index.asQueryEngine();
 
-  const response = await queryEngine.query("What does the example code do?");
+  const response = await queryEngine.query({
+    query: "What does the example code do?",
+  });
 
   // Output response
   console.log(response.toString());

examples/mistral.ts

@@ -27,7 +27,7 @@ async function rag(llm: LLM, embedModel: BaseEmbedding, query: string) {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(query);
+  const response = await queryEngine.query({ query });
   return response.response;
 }
 

examples/mongo.ts

@@ -54,7 +54,7 @@ async function main() {
       break;
     }
 
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
 
     // Output response
     console.log(response.toString());

examples/mongodb/3_query.ts

@@ -24,9 +24,9 @@ async function query() {
 
   const retriever = index.asRetriever({ similarityTopK: 20 });
   const queryEngine = index.asQueryEngine({ retriever });
-  const result = await queryEngine.query(
-    "What does the author think of web frameworks?",
-  );
+  const result = await queryEngine.query({
+    query: "What does the author think of web frameworks?",
+  });
   console.log(result.response);
   await client.close();
 }

examples/multimodal/rag.ts

@@ -46,9 +46,9 @@ async function main() {
     responseSynthesizer: new MultiModalResponseSynthesizer({ serviceContext }),
     retriever: index.asRetriever({ similarityTopK: 3, imageSimilarityTopK: 1 }),
   });
-  const result = await queryEngine.query(
-    "Tell me more about Vincent van Gogh's famous paintings",
-  );
+  const result = await queryEngine.query({
+    query: "Tell me more about Vincent van Gogh's famous paintings",
+  });
   console.log(result.response, "\n");
   images.forEach((image) =>
     console.log(`Image retrieved and used in inference: ${image.toString()}`),

examples/pg-vector-store/query.ts

@@ -31,7 +31,7 @@ async function main() {
       }
 
       try {
-        const answer = await queryEngine.query(question);
+        const answer = await queryEngine.query({ query: question });
         console.log(answer.response);
       } catch (error) {
         console.error("Error:", error);

examples/pinecone-vector-store/query.ts

@@ -29,7 +29,7 @@ async function main() {
       }
 
       try {
-        const answer = await queryEngine.query(question);
+        const answer = await queryEngine.query({ query: question });
         console.log(answer.response);
       } catch (error) {
         console.error("Error:", error);

examples/readers/load-assemblyai.ts

@@ -48,7 +48,7 @@ program
         break;
       }
 
-      const response = await queryEngine.query(query);
+      const response = await queryEngine.query({ query });
 
       console.log(response.toString());
     }

examples/readers/load-csv.ts

@@ -38,9 +38,9 @@ Given the CSV file, generate me Typescript code to answer the question: ${query}
   const queryEngine = index.asQueryEngine({ responseSynthesizer });
 
   // Query the index
-  const response = await queryEngine.query(
-    "What is the correlation between survival and age?",
-  );
+  const response = await queryEngine.query({
+    query: "What is the correlation between survival and age?",
+  });
 
   // Output response
   console.log(response.toString());

examples/readers/load-docx.ts

@@ -15,7 +15,7 @@ async function main() {
 
   // Test query
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(SAMPLE_QUERY);
+  const response = await queryEngine.query({ query: SAMPLE_QUERY });
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 

examples/readers/load-html.ts

@@ -10,9 +10,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What were the notable changes in 18.1?",
-  );
+  const response = await queryEngine.query({
+    query: "What were the notable changes in 18.1?",
+  });
 
   // Output response
   console.log(response.toString());

examples/readers/load-md.ts

@@ -15,7 +15,7 @@ async function main() {
 
   // Test query
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(SAMPLE_QUERY);
+  const response = await queryEngine.query({ query: SAMPLE_QUERY });
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 

examples/readers/load-notion.ts

@@ -79,7 +79,7 @@ program
         break;
       }
 
-      const response = await queryEngine.query(query);
+      const response = await queryEngine.query({ query });
 
       // Output response
       console.log(response.toString());

examples/readers/load-pdf.ts

@@ -10,7 +10,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query("What mistakes did they make?");
+  const response = await queryEngine.query({
+    query: "What mistakes did they make?",
+  });
 
   // Output response
   console.log(response.toString());

examples/sentenceWindow.ts

@@ -31,9 +31,9 @@ async function main() {
   const queryEngine = index.asQueryEngine({
     nodePostprocessors: [new MetadataReplacementPostProcessor("window")],
   });
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/storageContext.ts

@@ -20,9 +20,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());
@@ -35,9 +35,9 @@ async function main() {
     storageContext: secondStorageContext,
   });
   const loadedQueryEngine = loadedIndex.asQueryEngine();
-  const loadedResponse = await loadedQueryEngine.query(
-    "What did the author do growing up?",
-  );
+  const loadedResponse = await loadedQueryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(loadedResponse.toString());
 }
 

examples/subquestion.ts

@@ -18,9 +18,9 @@ import essay from "./essay";
     ],
   });
 
-  const response = await queryEngine.query(
-    "How was Paul Grahams life different before and after YC?",
-  );
+  const response = await queryEngine.query({
+    query: "How was Paul Grahams life different before and after YC?",
+  });
 
   console.log(response.toString());
 })();

examples/summaryIndex.ts

@@ -20,9 +20,9 @@ async function main() {
   const queryEngine = index.asQueryEngine({
     retriever: index.asRetriever({ mode: SummaryRetrieverMode.LLM }),
   });
-  const response = await queryEngine.query(
-    "What did the author do growing up?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(response.toString());
 }
 

examples/together-ai/vector-index.ts

@@ -0,0 +1,39 @@
+import fs from "node:fs/promises";
+
+import {
+  Document,
+  TogetherEmbedding,
+  TogetherLLM,
+  VectorStoreIndex,
+  serviceContextFromDefaults,
+} from "llamaindex";
+
+async function main() {
+  const apiKey = process.env.TOGETHER_API_KEY;
+  if (!apiKey) {
+    throw new Error("Missing TOGETHER_API_KEY");
+  }
+  const path = require.resolve("llamaindex/examples/abramov.txt");
+  const essay = await fs.readFile(path, "utf-8");
+
+  const document = new Document({ text: essay, id_: path });
+
+  const serviceContext = serviceContextFromDefaults({
+    llm: new TogetherLLM({ model: "mistralai/Mixtral-8x7B-Instruct-v0.1" }),
+    embedModel: new TogetherEmbedding(),
+  });
+
+  const index = await VectorStoreIndex.fromDocuments([document], {
+    serviceContext,
+  });
+
+  const queryEngine = index.asQueryEngine();
+
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);

examples/vectorIndex.ts

@@ -16,9 +16,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vectorIndexAnthropic.ts

@@ -35,9 +35,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine({ responseSynthesizer });
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vectorIndexCustomize.ts

@@ -33,9 +33,9 @@ async function main() {
     [nodePostprocessor],
   );
 
-  const response = await queryEngine.query(
-    "What did the author do growing up?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(response.response);
 }
 

examples/vectorIndexFromVectorStore.ts

@@ -189,7 +189,9 @@ async function main() {
     },
   });
 
-  const response = await queryEngine.query("How many results do you have?");
+  const response = await queryEngine.query({
+    query: "How many results do you have?",
+  });
 
   console.log(response.toString());
 }

examples/vectorIndexGPT4.ts

@@ -25,9 +25,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

packages/core/CHANGELOG.md

@@ -1,5 +1,20 @@
 # llamaindex
 
+## 0.0.47
+
+### Patch Changes
+
+- 844029d: Add streaming support for QueryEngine (and unify streaming interface with ChatEngine)
+- 844029d: Breaking: Use parameter object for query and chat methods of ChatEngine and QueryEngine
+
+## 0.0.46
+
+### Patch Changes
+
+- 977f284: fixing import statement
+- 5d3bb66: fix: class SimpleKVStore might throw error in ES module
+- f18c9f6: refactor: Updated low-level streaming interface
+
 ## 0.0.45
 
 ### Patch Changes

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.0.45",
+  "version": "0.0.47",
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.9.1",
@@ -49,7 +49,8 @@
       "types": "./dist/index.d.mts",
       "import": "./dist/index.mjs",
       "require": "./dist/index.js"
-    }
+    },
+    "./examples/*": "./examples/*"
   },
   "files": [
     "dist",

packages/core/src/ChatEngine.ts

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { ChatHistory } from "./ChatHistory";
+import { ChatHistory, SimpleChatHistory } from "./ChatHistory";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   CondenseQuestionPrompt,
@@ -13,27 +13,40 @@ import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { Event } from "./callbacks/CallbackManager";
-import { ChatMessage, LLM, OpenAI } from "./llm";
+import { ChatMessage, ChatResponseChunk, LLM, OpenAI } from "./llm";
+import { streamConverter, streamReducer } from "./llm/utils";
 import { BaseNodePostprocessor } from "./postprocessors";
 
+/**
+ * Represents the base parameters for ChatEngine.
+ */
+export interface ChatEngineParamsBase {
+  message: MessageContent;
+  /**
+   * Optional chat history if you want to customize the chat history.
+   */
+  chatHistory?: ChatMessage[];
+  history?: ChatHistory;
+}
+
+export interface ChatEngineParamsStreaming extends ChatEngineParamsBase {
+  stream: true;
+}
+
+export interface ChatEngineParamsNonStreaming extends ChatEngineParamsBase {
+  stream?: false | null;
+}
+
 /**
  * A ChatEngine is used to handle back and forth chats between the application and the LLM.
  */
 export interface ChatEngine {
   /**
    * Send message along with the class's current chat history to the LLM.
-   * @param message
-   * @param chatHistory optional chat history if you want to customize the chat history
-   * @param streaming optional streaming flag, which auto-sets the return value if True.
+   * @param params
    */
-  chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-    streaming?: T,
-  ): Promise<R>;
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
 
   /**
    * Resets the chat history so that it's empty.
@@ -53,48 +66,37 @@ export class SimpleChatEngine implements ChatEngine {
     this.llm = init?.llm ?? new OpenAI();
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-    streaming?: T,
-  ): Promise<R> {
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+
+    const chatHistory = params.chatHistory ?? this.chatHistory;
+    chatHistory.push({ content: message, role: "user" });
+
+    if (stream) {
+      const stream = await this.llm.chat({
+        messages: chatHistory,
+        stream: true,
+      });
+      return streamConverter(
+        streamReducer({
+          stream,
+          initialValue: "",
+          reducer: (accumulator, part) => (accumulator += part.delta),
+          finished: (accumulator) => {
+            chatHistory.push({ content: accumulator, role: "assistant" });
+          },
+        }),
+        (r: ChatResponseChunk) => new Response(r.delta),
+      );
     }
 
-    //Non-streaming option
-    chatHistory = chatHistory ?? this.chatHistory;
-    chatHistory.push({ content: message, role: "user" });
     const response = await this.llm.chat({ messages: chatHistory });
     chatHistory.push(response.message);
-    this.chatHistory = chatHistory;
-    return new Response(response.message.content) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-  ): AsyncGenerator<string, void, unknown> {
-    chatHistory = chatHistory ?? this.chatHistory;
-    chatHistory.push({ content: message, role: "user" });
-    const response_generator = await this.llm.chat({
-      messages: chatHistory,
-      stream: true,
-    });
-
-    var accumulator: string = "";
-    for await (const part of response_generator) {
-      accumulator += part.delta;
-      yield part.delta;
-    }
-
-    chatHistory.push({ content: accumulator, role: "assistant" });
-    this.chatHistory = chatHistory;
-    return;
+    return new Response(response.message.content);
   }
 
   reset() {
@@ -115,7 +117,7 @@ export class SimpleChatEngine implements ChatEngine {
 export class CondenseQuestionChatEngine implements ChatEngine {
   queryEngine: BaseQueryEngine;
   chatHistory: ChatMessage[];
-  serviceContext: ServiceContext;
+  llm: LLM;
   condenseMessagePrompt: CondenseQuestionPrompt;
 
   constructor(init: {
@@ -126,8 +128,7 @@ export class CondenseQuestionChatEngine implements ChatEngine {
   }) {
     this.queryEngine = init.queryEngine;
     this.chatHistory = init?.chatHistory ?? [];
-    this.serviceContext =
-      init?.serviceContext ?? serviceContextFromDefaults({});
+    this.llm = init?.serviceContext?.llm ?? serviceContextFromDefaults().llm;
     this.condenseMessagePrompt =
       init?.condenseMessagePrompt ?? defaultCondenseQuestionPrompt;
   }
@@ -135,7 +136,7 @@ export class CondenseQuestionChatEngine implements ChatEngine {
   private async condenseQuestion(chatHistory: ChatMessage[], question: string) {
     const chatHistoryStr = messagesToHistoryStr(chatHistory);
 
-    return this.serviceContext.llm.complete({
+    return this.llm.complete({
       prompt: defaultCondenseQuestionPrompt({
         question: question,
         chatHistory: chatHistoryStr,
@@ -143,26 +144,39 @@ export class CondenseQuestionChatEngine implements ChatEngine {
     });
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    chatHistory = chatHistory ?? this.chatHistory;
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+    const chatHistory = params.chatHistory ?? this.chatHistory;
 
     const condensedQuestion = (
       await this.condenseQuestion(chatHistory, extractText(message))
     ).text;
-
-    const response = await this.queryEngine.query(condensedQuestion);
-
     chatHistory.push({ content: message, role: "user" });
+
+    if (stream) {
+      const stream = await this.queryEngine.query({
+        query: condensedQuestion,
+        stream: true,
+      });
+      return streamReducer({
+        stream,
+        initialValue: "",
+        reducer: (accumulator, part) => (accumulator += part.response),
+        finished: (accumulator) => {
+          chatHistory.push({ content: accumulator, role: "assistant" });
+        },
+      });
+    }
+    const response = await this.queryEngine.query({
+      query: condensedQuestion,
+    });
     chatHistory.push({ content: response.response, role: "assistant" });
 
-    return response as R;
+    return response;
   }
 
   reset() {
@@ -255,21 +269,13 @@ export class ContextChatEngine implements ChatEngine {
     });
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+    const chatHistory = params.chatHistory ?? this.chatHistory;
     const parentEvent: Event = {
       id: randomUUID(),
       type: "wrapper",
@@ -279,57 +285,33 @@ export class ContextChatEngine implements ChatEngine {
       extractText(message),
       parentEvent,
     );
-
+    const nodes = context.nodes.map((r) => r.node);
     chatHistory.push({ content: message, role: "user" });
 
+    if (stream) {
+      const stream = await this.chatModel.chat({
+        messages: [context.message, ...chatHistory],
+        parentEvent,
+        stream: true,
+      });
+      return streamConverter(
+        streamReducer({
+          stream,
+          initialValue: "",
+          reducer: (accumulator, part) => (accumulator += part.delta),
+          finished: (accumulator) => {
+            chatHistory.push({ content: accumulator, role: "assistant" });
+          },
+        }),
+        (r: ChatResponseChunk) => new Response(r.delta, nodes),
+      );
+    }
     const response = await this.chatModel.chat({
       messages: [context.message, ...chatHistory],
       parentEvent,
     });
     chatHistory.push(response.message);
-
-    this.chatHistory = chatHistory;
-
-    return new Response(
-      response.message.content,
-      context.nodes.map((r) => r.node),
-    ) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-  ): AsyncGenerator<string, void, unknown> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    const parentEvent: Event = {
-      id: randomUUID(),
-      type: "wrapper",
-      tags: ["final"],
-    };
-    const context = await this.contextGenerator.generate(
-      extractText(message),
-      parentEvent,
-    );
-
-    chatHistory.push({ content: message, role: "user" });
-
-    const response_stream = await this.chatModel.chat({
-      messages: [context.message, ...chatHistory],
-      parentEvent,
-      stream: true,
-    });
-    var accumulator: string = "";
-    for await (const part of response_stream) {
-      accumulator += part.delta;
-      yield part.delta;
-    }
-
-    chatHistory.push({ content: accumulator, role: "assistant" });
-
-    this.chatHistory = chatHistory;
-
-    return;
+    return new Response(response.message.content, nodes);
   }
 
   reset() {
@@ -382,50 +364,38 @@ export class HistoryChatEngine {
     this.contextGenerator = init?.contextGenerator;
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-    streaming?: T,
-  ): Promise<R> {
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream, history } = params;
+    const chatHistory = history ?? new SimpleChatHistory();
     const requestMessages = await this.prepareRequestMessages(
       message,
       chatHistory,
     );
+
+    if (stream) {
+      const stream = await this.llm.chat({
+        messages: requestMessages,
+        stream: true,
+      });
+      return streamConverter(
+        streamReducer({
+          stream,
+          initialValue: "",
+          reducer: (accumulator, part) => (accumulator += part.delta),
+          finished: (accumulator) => {
+            chatHistory.addMessage({ content: accumulator, role: "assistant" });
+          },
+        }),
+        (r: ChatResponseChunk) => new Response(r.delta),
+      );
+    }
     const response = await this.llm.chat({ messages: requestMessages });
     chatHistory.addMessage(response.message);
-    return new Response(response.message.content) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-  ): AsyncGenerator<string, void, unknown> {
-    const requestMessages = await this.prepareRequestMessages(
-      message,
-      chatHistory,
-    );
-    const response_stream = await this.llm.chat({
-      messages: requestMessages,
-      stream: true,
-    });
-
-    var accumulator = "";
-    for await (const part of response_stream) {
-      accumulator += part.delta;
-      yield part.delta;
-    }
-    chatHistory.addMessage({
-      content: accumulator,
-      role: "assistant",
-    });
-    return;
+    return new Response(response.message.content);
   }
 
   private async prepareRequestMessages(

packages/core/src/QueryEngine.ts

@@ -17,16 +17,32 @@ import {
   ResponseSynthesizer,
 } from "./synthesizers";
 
+/**
+ * Parameters for sending a query.
+ */
+export interface QueryEngineParamsBase {
+  query: string;
+  parentEvent?: Event;
+}
+
+export interface QueryEngineParamsStreaming extends QueryEngineParamsBase {
+  stream: true;
+}
+
+export interface QueryEngineParamsNonStreaming extends QueryEngineParamsBase {
+  stream?: false | null;
+}
+
 /**
  * A query engine is a question answerer that can use one or more steps.
  */
 export interface BaseQueryEngine {
   /**
    * Query the query engine and get a response.
-   * @param query
-   * @param parentEvent
+   * @param params
    */
-  query(query: string, parentEvent?: Event): Promise<Response>;
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
 }
 
 /**
@@ -70,17 +86,30 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
     return this.applyNodePostprocessors(nodes);
   }
 
-  async query(query: string, parentEvent?: Event) {
-    const _parentEvent: Event = parentEvent || {
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { query, stream } = params;
+    const parentEvent: Event = params.parentEvent || {
       id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
     };
-    const nodesWithScore = await this.retrieve(query, _parentEvent);
+    const nodesWithScore = await this.retrieve(query, parentEvent);
+    if (stream) {
+      return this.responseSynthesizer.synthesize({
+        query,
+        nodesWithScore,
+        parentEvent,
+        stream: true,
+      });
+    }
     return this.responseSynthesizer.synthesize({
       query,
       nodesWithScore,
-      parentEvent: _parentEvent,
+      parentEvent,
     });
   }
 }
@@ -135,11 +164,16 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
     });
   }
 
-  async query(query: string): Promise<Response> {
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { query, stream } = params;
     const subQuestions = await this.questionGen.generate(this.metadatas, query);
 
     // groups final retrieval+synthesis operation
-    const parentEvent: Event = {
+    const parentEvent: Event = params.parentEvent || {
       id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
@@ -160,6 +194,14 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
     const nodesWithScore = subQNodes
       .filter((node) => node !== null)
       .map((node) => node as NodeWithScore);
+    if (stream) {
+      return this.responseSynthesizer.synthesize({
+        query,
+        nodesWithScore,
+        parentEvent,
+        stream: true,
+      });
+    }
     return this.responseSynthesizer.synthesize({
       query,
       nodesWithScore,
@@ -175,7 +217,10 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
       const question = subQ.subQuestion;
       const queryEngine = this.queryEngines[subQ.toolName];
 
-      const response = await queryEngine.query(question, parentEvent);
+      const response = await queryEngine.query({
+        query: question,
+        parentEvent,
+      });
       const responseText = response.response;
       const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
       const node = new TextNode({ text: nodeText });

packages/core/src/llm/LLM.ts

@@ -27,6 +27,7 @@ import {
 import { OpenAISession, getOpenAISession } from "./openai";
 import { PortkeySession, getPortkeySession } from "./portkey";
 import { ReplicateSession } from "./replicate";
+import { streamConverter } from "./utils";
 
 export type MessageType =
   | "user"
@@ -127,14 +128,6 @@ export interface LLM {
 export abstract class BaseLLM implements LLM {
   abstract metadata: LLMMetadata;
 
-  private async *chatToComplete(
-    stream: AsyncIterable<ChatResponseChunk>,
-  ): AsyncIterable<CompletionResponse> {
-    for await (const chunk of stream) {
-      yield { text: chunk.delta };
-    }
-  }
-
   complete(
     params: LLMCompletionParamsStreaming,
   ): Promise<AsyncIterable<CompletionResponse>>;
@@ -151,7 +144,11 @@ export abstract class BaseLLM implements LLM {
         parentEvent,
         stream: true,
       });
-      return this.chatToComplete(stream);
+      return streamConverter(stream, (chunk) => {
+        return {
+          text: chunk.delta,
+        };
+      });
     }
     const chatResponse = await this.chat({
       messages: [{ content: prompt, role: "user" }],
@@ -392,6 +389,7 @@ export class OpenAI extends BaseLLM {
           type: "llmPredict" as EventType,
         };
 
+    // TODO: add callback to streamConverter and use streamConverter here
     //Indices
     var idx_counter: number = 0;
     for await (const part of chunk_stream) {

packages/core/src/llm/utils.ts

@@ -1,5 +1,3 @@
-// TODO: use for LLM.ts
-
 export async function* streamConverter<S, D>(
   stream: AsyncIterable<S>,
   converter: (s: S) => D,
@@ -8,3 +6,19 @@ export async function* streamConverter<S, D>(
     yield converter(data);
   }
 }
+
+export async function* streamReducer<S, D>(params: {
+  stream: AsyncIterable<S>;
+  reducer: (previousValue: D, currentValue: S) => D;
+  initialValue: D;
+  finished?: (value: D | undefined) => void;
+}): AsyncIterable<S> {
+  let value = params.initialValue;
+  for await (const data of params.stream) {
+    value = params.reducer(value, data);
+    yield data;
+  }
+  if (params.finished) {
+    params.finished(value);
+  }
+}

packages/core/src/storage/StorageContext.ts

@@ -1,4 +1,4 @@
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "./FileSystem";
 import {
   DEFAULT_FS,

packages/core/src/storage/docStore/SimpleDocumentStore.ts

@@ -1,5 +1,5 @@
 import _ from "lodash";
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "../FileSystem";
 import {
   DEFAULT_DOC_STORE_PERSIST_FILENAME,

packages/core/src/storage/indexStore/SimpleIndexStore.ts

@@ -1,4 +1,4 @@
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "../FileSystem";
 import {
   DEFAULT_FS,

packages/core/src/storage/kvStore/SimpleKVStore.ts

@@ -1,5 +1,5 @@
-import * as _ from "lodash";
-import * as path from "path";
+import _ from "lodash";
+import path from "path";
 import { GenericFileSystem, exists } from "../FileSystem";
 import { DEFAULT_COLLECTION, DEFAULT_FS } from "../constants";
 import { BaseKVStore } from "./types";

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -1,5 +1,5 @@
 import _ from "lodash";
-import * as path from "path";
+import path from "path";
 import { BaseNode } from "../../Node";
 import {
   getTopKEmbeddings,

packages/core/src/tests/CallbackManager.test.ts

@@ -67,7 +67,7 @@ describe("CallbackManager: onLLMStream and onRetrieve", () => {
     });
     const queryEngine = vectorStoreIndex.asQueryEngine();
     const query = "What is the author's name?";
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
     expect(response.toString()).toBe("MOCK_TOKEN_1-MOCK_TOKEN_2");
     expect(streamCallbackData).toEqual([
       {
@@ -145,7 +145,7 @@ describe("CallbackManager: onLLMStream and onRetrieve", () => {
       responseSynthesizer,
     });
     const query = "What is the author's name?";
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
     expect(response.toString()).toBe("MOCK_TOKEN_1-MOCK_TOKEN_2");
     expect(streamCallbackData).toEqual([
       {

packages/create-llama/CHANGELOG.md

@@ -1,5 +1,11 @@
 # create-llama
 
+## 0.0.15
+
+### Patch Changes
+
+- 8e124e5: feat: support showing image on chat message
+
 ## 0.0.14
 
 ### Patch Changes

packages/create-llama/create-app.ts

@@ -34,6 +34,7 @@ export async function createApp({
   communityProjectPath,
   vectorDb,
   externalPort,
+  installDependencies,
 }: InstallAppArgs): Promise<void> {
   const root = path.resolve(appPath);
 
@@ -75,6 +76,7 @@ export async function createApp({
     communityProjectPath,
     vectorDb,
     externalPort,
+    installDependencies,
   };
 
   if (frontend) {

packages/create-llama/e2e/utils.ts

@@ -104,6 +104,7 @@ export function runCreateLlama(
     "--use-npm",
     "--external-port",
     externalPort,
+    "--install-dependencies",
   ].join(" ");
   console.log(`running command '${command}' in ${cwd}`);
   execSync(command, {

packages/create-llama/helpers/index.ts

@@ -7,6 +7,7 @@ import { cyan } from "picocolors";
 
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./constant";
 import { PackageManager } from "./get-pkg-manager";
+import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry";
 import { installPythonTemplate } from "./python";
 import { downloadAndExtractRepo } from "./repo";
 import {
@@ -89,18 +90,23 @@ const copyTestData = async (
   if (packageManager && engine === "context") {
     const runGenerate = `${cyan(
       framework === "fastapi"
-        ? "python app/engine/generate.py"
+        ? "poetry run python app/engine/generate.py"
         : `${packageManager} run generate`,
     )}`;
     const hasOpenAiKey = openAiKey || process.env["OPENAI_API_KEY"];
     const hasVectorDb = vectorDb && vectorDb !== "none";
-    const shouldRunGenerateAfterInstall =
-      hasOpenAiKey && framework !== "fastapi" && vectorDb === "none";
-    if (shouldRunGenerateAfterInstall) {
-      console.log(`\nRunning ${runGenerate} to generate the context data.\n`);
-      await callPackageManager(packageManager, true, ["run", "generate"]);
-      console.log();
-      return;
+    if (framework === "fastapi") {
+      if (hasOpenAiKey && vectorDb === "none" && isHavingPoetryLockFile()) {
+        console.log(`Running ${runGenerate} to generate the context data.`);
+        tryPoetryRun("python app/engine/generate.py");
+        return;
+      }
+    } else {
+      if (hasOpenAiKey && vectorDb === "none") {
+        console.log(`Running ${runGenerate} to generate the context data.`);
+        await callPackageManager(packageManager, true, ["run", "generate"]);
+        return;
+      }
     }
 
     const settings = [];

packages/create-llama/helpers/poetry.ts

@@ -0,0 +1,34 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { execSync } from "child_process";
+import fs from "fs";
+
+export function isPoetryAvailable(): boolean {
+  try {
+    execSync("poetry --version", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function tryPoetryInstall(): boolean {
+  try {
+    execSync("poetry install", { stdio: "inherit" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function tryPoetryRun(command: string): boolean {
+  try {
+    execSync(`poetry run ${command}`, { stdio: "inherit" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function isHavingPoetryLockFile(): boolean {
+  try {
+    return fs.existsSync("poetry.lock");
+  } catch (_) {}
+  return false;
+}

packages/create-llama/helpers/python.ts

@@ -1,8 +1,10 @@
 import fs from "fs/promises";
 import path from "path";
-import { cyan } from "picocolors";
+import { cyan, yellow } from "picocolors";
 import { parse, stringify } from "smol-toml";
+import terminalLink from "terminal-link";
 import { copy } from "./copy";
+import { isPoetryAvailable, tryPoetryInstall } from "./poetry";
 import { InstallTemplateArgs, TemplateVectorDB } from "./types";
 
 interface Dependency {
@@ -96,9 +98,15 @@ export const installPythonTemplate = async ({
   framework,
   engine,
   vectorDb,
+  installDependencies,
 }: Pick<
   InstallTemplateArgs,
-  "root" | "framework" | "template" | "engine" | "vectorDb"
+  | "root"
+  | "framework"
+  | "template"
+  | "engine"
+  | "vectorDb"
+  | "installDependencies"
 >) => {
   console.log("\nInitializing Python project with template:", template, "\n");
   const templatePath = path.join(
@@ -146,7 +154,28 @@ export const installPythonTemplate = async ({
   const addOnDependencies = getAdditionalDependencies(vectorDb);
   await addDependencies(root, addOnDependencies);
 
-  console.log(
-    "\nPython project, dependencies won't be installed automatically.\n",
-  );
+  // install python dependencies
+  if (installDependencies) {
+    if (isPoetryAvailable()) {
+      console.log(
+        `Installing python dependencies using poetry. This may take a while...`,
+      );
+      const installSuccessful = tryPoetryInstall();
+      if (!installSuccessful) {
+        console.warn(
+          yellow("Install failed. Please install dependencies manually."),
+        );
+      }
+    } else {
+      console.warn(
+        yellow(
+          `Poetry is not available in the current environment. The Python dependencies will not be installed automatically.
+Please check ${terminalLink(
+            "Poetry Installation",
+            `https://python-poetry.org/docs/#installation`,
+          )} to install poetry first, then install the dependencies manually.`,
+        ),
+      );
+    }
+  }
 };

packages/create-llama/helpers/types.ts

@@ -23,4 +23,5 @@ export interface InstallTemplateArgs {
   communityProjectPath?: string;
   vectorDb?: TemplateVectorDB;
   externalPort?: number;
+  installDependencies?: boolean;
 }

packages/create-llama/helpers/typescript.ts

@@ -39,6 +39,7 @@ export const installTSTemplate = async ({
   customApiPath,
   forBackend,
   vectorDb,
+  installDependencies,
 }: InstallTemplateArgs) => {
   console.log(bold(`Using ${packageManager}.`));
 
@@ -210,15 +211,17 @@ export const installTSTemplate = async ({
     JSON.stringify(packageJson, null, 2) + os.EOL,
   );
 
-  console.log("\nInstalling dependencies:");
-  for (const dependency in packageJson.dependencies)
-    console.log(`- ${cyan(dependency)}`);
+  if (installDependencies) {
+    console.log("\nInstalling dependencies:");
+    for (const dependency in packageJson.dependencies)
+      console.log(`- ${cyan(dependency)}`);
 
-  console.log("\nInstalling devDependencies:");
-  for (const dependency in packageJson.devDependencies)
-    console.log(`- ${cyan(dependency)}`);
+    console.log("\nInstalling devDependencies:");
+    for (const dependency in packageJson.devDependencies)
+      console.log(`- ${cyan(dependency)}`);
 
-  console.log();
+    console.log();
 
-  await callPackageManager(packageManager, isOnline);
+    await callPackageManager(packageManager, isOnline);
+  }
 };

packages/create-llama/index.ts

@@ -117,6 +117,13 @@ const program = new Commander.Command(packageJson.name)
     `
 
 Select external port.
+`,
+  )
+  .option(
+    "--install-dependencies",
+    `
+
+Whether install dependencies (backend/frontend) automatically or not.
 `,
   )
   .allowUnknownOption()
@@ -224,6 +231,7 @@ async function run(): Promise<void> {
     communityProjectPath: program.communityProjectPath,
     vectorDb: program.vectorDb,
     externalPort: program.externalPort,
+    installDependencies: program.installDependencies,
   });
   conf.set("preferences", preferences);
 }

packages/create-llama/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.0.14",
+  "version": "0.0.15",
   "keywords": [
     "rag",
     "llamaindex",

packages/create-llama/questions.ts

@@ -211,6 +211,19 @@ export const askQuestions = async (
     }
   }
 
+  if (program.installDependencies === undefined) {
+    const { installDependencies } = await prompts({
+      onState: onPromptState,
+      type: "toggle",
+      name: "installDependencies",
+      message: `Would you like to install dependencies automatically? This may take a while`,
+      initial: getPrefOrDefault("installDependencies"),
+      active: "Yes",
+      inactive: "No",
+    });
+    program.installDependencies = Boolean(installDependencies);
+  }
+
   if (!program.model) {
     if (ciInfo.isCI) {
       program.model = getPrefOrDefault("model");