Release 0.11.19 (#2105)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>

apps/next/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/doc
 
+## 0.2.40
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+- Updated dependencies [a1fdb07]
+  - @llamaindex/core@0.6.15
+  - @llamaindex/workflow@1.1.15
+  - @llamaindex/openai@0.4.9
+  - @llamaindex/cloud@4.0.24
+  - llamaindex@0.11.19
+  - @llamaindex/node-parser@2.0.15
+  - @llamaindex/readers@3.1.14
+
 ## 0.2.39
 
 ### Patch Changes

apps/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/doc",
-  "version": "0.2.39",
+  "version": "0.2.40",
   "private": true,
   "scripts": {
     "postinstall": "fumadocs-mdx",

apps/next/src/app/api/chat/route.ts

@@ -1,4 +1,4 @@
-import { MockLLM } from "@llamaindex/core/utils";
+import { MockLLM } from "@llamaindex/core/llms/mock";
 import { LlamaIndexAdapter, type Message } from "ai";
 import { Settings, SimpleChatEngine, type ChatMessage } from "llamaindex";
 import { NextResponse, type NextRequest } from "next/server";

e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.180
+
+### Patch Changes
+
+- llamaindex@0.11.19
+
 ## 0.0.179
 
 ### Patch Changes

e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.179",
+  "version": "0.0.180",
   "type": "module",
   "private": true,
   "scripts": {

e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.79
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.24
+
 ## 0.0.78
 
 ### Patch Changes

e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.78",
+  "version": "0.0.79",
   "type": "module",
   "scripts": {
     "dev": "vite",

e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/next-agent-test
 
+## 0.1.180
+
+### Patch Changes
+
+- llamaindex@0.11.19
+
 ## 0.1.179
 
 ### Patch Changes

e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.179",
+  "version": "0.1.180",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,11 @@
 # test-edge-runtime
 
+## 0.1.179
+
+### Patch Changes
+
+- llamaindex@0.11.19
+
 ## 0.1.178
 
 ### Patch Changes

e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.178",
+  "version": "0.1.179",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/next-node-runtime
 
+## 0.1.48
+
+### Patch Changes
+
+- llamaindex@0.11.19
+- @llamaindex/huggingface@0.1.19
+- @llamaindex/readers@3.1.14
+
 ## 0.1.47
 
 ### Patch Changes

e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.47",
+  "version": "0.1.48",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/vite-import-llamaindex/CHANGELOG.md

@@ -1,5 +1,11 @@
 # vite-import-llamaindex
 
+## 0.0.46
+
+### Patch Changes
+
+- llamaindex@0.11.19
+
 ## 0.0.45
 
 ### Patch Changes

e2e/examples/vite-import-llamaindex/package.json

@@ -1,7 +1,7 @@
 {
   "name": "vite-import-llamaindex",
   "private": true,
-  "version": "0.0.45",
+  "version": "0.0.46",
   "type": "module",
   "scripts": {
     "build": "vite build",

e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.180
+
+### Patch Changes
+
+- llamaindex@0.11.19
+
 ## 0.0.179
 
 ### Patch Changes

e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.179",
+  "version": "0.0.180",
   "type": "module",
   "private": true,
   "scripts": {

examples/CHANGELOG.md

@@ -1,5 +1,60 @@
 # examples
 
+## 0.3.30
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+- Updated dependencies [a1fdb07]
+- Updated dependencies [ddc0eaf]
+  - @llamaindex/core@0.6.15
+  - @llamaindex/tools@0.1.5
+  - @llamaindex/workflow@1.1.15
+  - @llamaindex/openai@0.4.9
+  - @llamaindex/anthropic@0.3.17
+  - @llamaindex/cloud@4.0.24
+  - llamaindex@0.11.19
+  - @llamaindex/node-parser@2.0.15
+  - @llamaindex/assemblyai@0.1.14
+  - @llamaindex/clip@0.0.65
+  - @llamaindex/cohere@0.0.29
+  - @llamaindex/deepinfra@0.0.65
+  - @llamaindex/discord@0.1.14
+  - @llamaindex/google@0.3.14
+  - @llamaindex/huggingface@0.1.19
+  - @llamaindex/jinaai@0.0.25
+  - @llamaindex/mistral@0.1.15
+  - @llamaindex/mixedbread@0.0.29
+  - @llamaindex/notion@0.1.14
+  - @llamaindex/ollama@0.1.15
+  - @llamaindex/perplexity@0.0.22
+  - @llamaindex/portkey-ai@0.0.57
+  - @llamaindex/replicate@0.0.57
+  - @llamaindex/bm25-retriever@0.0.4
+  - @llamaindex/astra@0.0.29
+  - @llamaindex/azure@0.1.26
+  - @llamaindex/chroma@0.0.29
+  - @llamaindex/elastic-search@0.1.15
+  - @llamaindex/firestore@1.0.22
+  - @llamaindex/milvus@0.1.24
+  - @llamaindex/mongodb@0.0.30
+  - @llamaindex/pinecone@0.1.15
+  - @llamaindex/postgres@0.0.58
+  - @llamaindex/qdrant@0.1.25
+  - @llamaindex/supabase@0.1.15
+  - @llamaindex/upstash@0.0.29
+  - @llamaindex/weaviate@0.0.30
+  - @llamaindex/vercel@0.1.15
+  - @llamaindex/voyage-ai@1.0.21
+  - @llamaindex/readers@3.1.14
+  - @llamaindex/deepseek@0.0.26
+  - @llamaindex/fireworks@0.0.25
+  - @llamaindex/groq@0.0.81
+  - @llamaindex/together@0.0.25
+  - @llamaindex/vllm@0.0.51
+  - @llamaindex/xai@0.0.12
+
 ## 0.3.29
 
 ### Patch Changes

examples/agents/agent/single-agent.ts

@@ -3,7 +3,7 @@
  */
 import { openai } from "@llamaindex/openai";
 import { agent } from "@llamaindex/workflow";
-import { getWeatherTool } from "../../deprecated/agents/utils/tools";
+import { getWeatherTool } from "../tools/tools";
 
 async function main() {
   const weatherAgent = agent({

examples/agents/agent/with-ollama.ts

@@ -1,6 +1,6 @@
 import { ollama } from "@llamaindex/ollama";
 import { agent } from "@llamaindex/workflow";
-import { getWeatherTool } from "../../deprecated/agents/utils/tools";
+import { getWeatherTool } from "../tools/tools";
 
 async function main() {
   const myAgent = agent({

examples/agents/tools/chat-stream.ts

@@ -1,7 +1,7 @@
-import { OpenAI } from "@llamaindex/openai";
+import { openai } from "@llamaindex/openai";
 
 async function main() {
-  const llm = new OpenAI({ model: "gpt-4-turbo" });
+  const llm = openai({ model: "gpt-4.1-mini" });
   const args: Parameters<typeof llm.chat>[0] = {
     additionalChatOptions: {
       tool_choice: "auto",

examples/agents/tools/exec-stream.ts

@@ -0,0 +1,46 @@
+import { openai } from "@llamaindex/openai";
+import { tool } from "llamaindex";
+import z from "zod";
+
+import { ChatMessage } from "llamaindex";
+
+async function main() {
+  const llm = openai({ model: "gpt-4.1-mini" });
+  const messages = [
+    {
+      content: `What's the weather like in San Francisco?`,
+      role: "user",
+    } as ChatMessage,
+  ];
+
+  let exit = false;
+  do {
+    const { stream, newMessages, toolCalls } = await llm.exec({
+      messages,
+      tools: [
+        tool({
+          name: "get_weather",
+          description: "Get the current weather for a location",
+          parameters: z.object({
+            address: z.string().describe("The address"),
+          }),
+          execute: ({ address }) => {
+            return `It's sunny in ${address}!`;
+          },
+        }),
+      ],
+      stream: true,
+    });
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.delta);
+    }
+    messages.push(...newMessages());
+    // exit condition to stop the agent loop
+    // here we can also check for specific tool calls or limit the number of llm.exec calls
+    exit = toolCalls.length === 0;
+  } while (!exit);
+}
+
+(async function () {
+  await main();
+})();

examples/agents/tools/exec.ts

@@ -0,0 +1,43 @@
+import { openai } from "@llamaindex/openai";
+import { ChatMessage, tool } from "llamaindex";
+import z from "zod";
+
+async function main() {
+  const llm = openai({ model: "gpt-4.1-mini" });
+  const messages = [
+    {
+      content: `What's the weather like in San Francisco?`,
+      role: "user",
+    } as ChatMessage,
+  ];
+
+  let exit = false;
+  do {
+    const { newMessages, toolCalls } = await llm.exec({
+      messages,
+      tools: [
+        tool({
+          name: "get_weather",
+          description: "Get the current weather for a location",
+          parameters: z.object({
+            address: z.string().describe("The address"),
+          }),
+          execute: ({ address }) => {
+            return `It's sunny in ${address}!`;
+          },
+        }),
+      ],
+    });
+    console.log(newMessages);
+    messages.push(...newMessages);
+    // exit condition to stop the agent loop
+    // here we can also check for specific tool calls or limit the number of llm.exec calls
+    exit = toolCalls.length === 0;
+  } while (!exit);
+}
+
+(async function () {
+  console.log("Starting...");
+  await main();
+  console.log("Done");
+})();

examples/deprecated/agents/openai-task.ts

@@ -4,7 +4,7 @@ import {
   getCurrentIDTool,
   getUserInfoTool,
   getWeatherTool,
-} from "./utils/tools";
+} from "../../agents/tools/tools";
 
 async function main() {
   // Create an OpenAIAgent with the function tools

examples/deprecated/agents/react-task.ts

@@ -3,7 +3,7 @@ import {
   getCurrentIDTool,
   getUserInfoTool,
   getWeatherTool,
-} from "./utils/tools";
+} from "../../agents/tools/tools";
 
 async function main() {
   // Create an OpenAIAgent with the function tools

examples/models/openai/responses/image-gen.ts

@@ -30,6 +30,12 @@ async function main() {
   );
   // and print out the text part
   console.log(textPart?.text);
+
+  const imageId = response.message.options?.image_id;
+  if (imageId) {
+    console.log("Image ID for multi-turn generation:", imageId);
+    console.log("Use this image_id in subsequent requests to modify the image");
+  }
 }
 
 main().catch(console.error);

examples/models/openai/responses/multi-turn-image-gen.ts

@@ -0,0 +1,89 @@
+import { openaiResponses } from "@llamaindex/openai";
+import fs from "fs";
+import { MessageContentDetail } from "llamaindex";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4.1-mini",
+    builtInTools: [{ type: "image_generation" }],
+  });
+
+  // First turn: Generate initial image
+  console.log("=== First Turn: Generate initial image ===");
+  const firstResponse = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content:
+          "Generate an image of a cute tiny llama wearing a hat playing with a cat on a meadow",
+      },
+    ],
+  });
+
+  const firstContent = firstResponse.message.content as MessageContentDetail[];
+  const firstImagePart = firstContent.find((part) => part.type === "image");
+  const firstTextPart = firstContent.find((part) => part.type === "text");
+
+  // Save the first image
+  if (firstImagePart?.data) {
+    fs.writeFileSync(
+      "llama-initial.png",
+      Buffer.from(firstImagePart.data as string, "base64"),
+    );
+    console.log("First image saved as 'llama-initial.png'");
+  }
+
+  if (firstTextPart?.text) {
+    console.log("First response:", firstTextPart.text);
+  }
+
+  // Get the image_id from the response options for multi-turn
+  const imageId = firstResponse.message.options?.image_id;
+  console.log("Image ID for multi-turn:", imageId);
+
+  if (imageId) {
+    // Second turn: Modify the image using the image_id
+    console.log("\n=== Second Turn: Modify the image ===");
+    const secondResponse = await llm.chat({
+      messages: [
+        {
+          role: "user",
+          content:
+            "Generate an image of a cute tiny llama wearing a hat playing with a cat on a meadow",
+        },
+        {
+          role: "assistant",
+          content: firstContent,
+          options: { image_id: imageId },
+        },
+        {
+          role: "user",
+          content:
+            "Now add a rainbow in the background and make the llama's hat blue",
+        },
+      ],
+    });
+
+    const secondContent = secondResponse.message
+      .content as MessageContentDetail[];
+    const secondImagePart = secondContent.find((part) => part.type === "image");
+    const secondTextPart = secondContent.find((part) => part.type === "text");
+
+    // Save the modified image
+    if (secondImagePart?.data) {
+      fs.writeFileSync(
+        "llama-modified.png",
+        Buffer.from(secondImagePart.data as string, "base64"),
+      );
+      console.log("Modified image saved as 'llama-modified.png'");
+    }
+
+    if (secondTextPart?.text) {
+      console.log("Second response:", secondTextPart.text);
+    }
+  } else {
+    console.log("No image_id received, cannot perform multi-turn generation");
+  }
+}
+
+main().catch(console.error);

examples/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/examples",
-  "version": "0.3.29",
+  "version": "0.3.30",
   "private": true,
   "scripts": {
     "lint": "eslint .",
@@ -11,52 +11,52 @@
     "@azure/cosmos": "^4.1.1",
     "@azure/identity": "^4.4.1",
     "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.16",
-    "@llamaindex/assemblyai": "^0.1.13",
-    "@llamaindex/astra": "^0.0.28",
-    "@llamaindex/azure": "^0.1.25",
-    "@llamaindex/bm25-retriever": "^0.0.3",
-    "@llamaindex/chroma": "^0.0.28",
-    "@llamaindex/clip": "^0.0.64",
-    "@llamaindex/cloud": "^4.0.19",
-    "@llamaindex/cohere": "^0.0.28",
-    "@llamaindex/core": "^0.6.14",
-    "@llamaindex/deepinfra": "^0.0.64",
-    "@llamaindex/deepseek": "^0.0.25",
-    "@llamaindex/discord": "^0.1.13",
-    "@llamaindex/elastic-search": "^0.1.14",
+    "@llamaindex/anthropic": "^0.3.17",
+    "@llamaindex/assemblyai": "^0.1.14",
+    "@llamaindex/astra": "^0.0.29",
+    "@llamaindex/azure": "^0.1.26",
+    "@llamaindex/bm25-retriever": "^0.0.4",
+    "@llamaindex/chroma": "^0.0.29",
+    "@llamaindex/clip": "^0.0.65",
+    "@llamaindex/cloud": "^4.0.24",
+    "@llamaindex/cohere": "^0.0.29",
+    "@llamaindex/core": "^0.6.15",
+    "@llamaindex/deepinfra": "^0.0.65",
+    "@llamaindex/deepseek": "^0.0.26",
+    "@llamaindex/discord": "^0.1.14",
+    "@llamaindex/elastic-search": "^0.1.15",
     "@llamaindex/env": "^0.1.30",
-    "@llamaindex/firestore": "^1.0.21",
-    "@llamaindex/fireworks": "^0.0.24",
-    "@llamaindex/google": "^0.3.13",
-    "@llamaindex/groq": "^0.0.80",
-    "@llamaindex/huggingface": "^0.1.18",
-    "@llamaindex/jinaai": "^0.0.24",
-    "@llamaindex/milvus": "^0.1.23",
-    "@llamaindex/mistral": "^0.1.14",
-    "@llamaindex/mixedbread": "^0.0.28",
-    "@llamaindex/mongodb": "^0.0.29",
-    "@llamaindex/node-parser": "^2.0.14",
-    "@llamaindex/notion": "^0.1.13",
-    "@llamaindex/ollama": "^0.1.14",
-    "@llamaindex/openai": "^0.4.8",
-    "@llamaindex/perplexity": "^0.0.21",
-    "@llamaindex/pinecone": "^0.1.14",
-    "@llamaindex/portkey-ai": "^0.0.56",
-    "@llamaindex/postgres": "^0.0.57",
-    "@llamaindex/qdrant": "^0.1.24",
-    "@llamaindex/readers": "^3.1.13",
-    "@llamaindex/replicate": "^0.0.56",
-    "@llamaindex/supabase": "^0.1.14",
-    "@llamaindex/together": "^0.0.24",
-    "@llamaindex/tools": "^0.1.4",
-    "@llamaindex/upstash": "^0.0.28",
-    "@llamaindex/vercel": "^0.1.14",
-    "@llamaindex/vllm": "^0.0.50",
-    "@llamaindex/voyage-ai": "^1.0.20",
-    "@llamaindex/weaviate": "^0.0.29",
-    "@llamaindex/workflow": "^1.1.14",
-    "@llamaindex/xai": "^0.0.11",
+    "@llamaindex/firestore": "^1.0.22",
+    "@llamaindex/fireworks": "^0.0.25",
+    "@llamaindex/google": "^0.3.14",
+    "@llamaindex/groq": "^0.0.81",
+    "@llamaindex/huggingface": "^0.1.19",
+    "@llamaindex/jinaai": "^0.0.25",
+    "@llamaindex/milvus": "^0.1.24",
+    "@llamaindex/mistral": "^0.1.15",
+    "@llamaindex/mixedbread": "^0.0.29",
+    "@llamaindex/mongodb": "^0.0.30",
+    "@llamaindex/node-parser": "^2.0.15",
+    "@llamaindex/notion": "^0.1.14",
+    "@llamaindex/ollama": "^0.1.15",
+    "@llamaindex/openai": "^0.4.9",
+    "@llamaindex/perplexity": "^0.0.22",
+    "@llamaindex/pinecone": "^0.1.15",
+    "@llamaindex/portkey-ai": "^0.0.57",
+    "@llamaindex/postgres": "^0.0.58",
+    "@llamaindex/qdrant": "^0.1.25",
+    "@llamaindex/readers": "^3.1.14",
+    "@llamaindex/replicate": "^0.0.57",
+    "@llamaindex/supabase": "^0.1.15",
+    "@llamaindex/together": "^0.0.25",
+    "@llamaindex/tools": "^0.1.5",
+    "@llamaindex/upstash": "^0.0.29",
+    "@llamaindex/vercel": "^0.1.15",
+    "@llamaindex/vllm": "^0.0.51",
+    "@llamaindex/voyage-ai": "^1.0.21",
+    "@llamaindex/weaviate": "^0.0.30",
+    "@llamaindex/workflow": "^1.1.15",
+    "@llamaindex/xai": "^0.0.12",
     "@notionhq/client": "^4.0.0",
     "@pinecone-database/pinecone": "^4.0.0",
     "@vercel/postgres": "^0.10.0",
@@ -65,7 +65,7 @@
     "commander": "^12.1.0",
     "dotenv": "^17.2.0",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.11.14",
+    "llamaindex": "^0.11.19",
     "mongodb": "6.7.0",
     "postgres": "^3.4.4",
     "wikipedia": "^2.1.2",

packages/autotool/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/autotool
 
+## 8.0.19
+
+### Patch Changes
+
+- llamaindex@0.11.19
+
 ## 8.0.18
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.127
+
+### Patch Changes
+
+- llamaindex@0.11.19
+- @llamaindex/autotool@8.0.19
+
 ## 0.0.126
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.126"
+  "version": "0.0.127"
 }

packages/autotool/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
     "directory": "packages/autotool"
   },
-  "version": "8.0.18",
+  "version": "8.0.19",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/cloud
 
+## 4.0.24
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 4.0.23
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "4.0.23",
+  "version": "4.0.24",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/core/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/core
 
+## 0.6.15
+
+### Patch Changes
+
+- 7ad3411: feat: add llm.exec
+- 5da5b3c: add progress callback to embeddings
+
 ## 0.6.14
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.6.14",
+  "version": "0.6.15",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./agent": {
@@ -59,6 +59,17 @@
       },
       "default": "./llms/dist/index.js"
     },
+    "./llms/mock": {
+      "require": {
+        "types": "./llms/mock/dist/index.d.cts",
+        "default": "./llms/mock/dist/index.cjs"
+      },
+      "import": {
+        "types": "./llms/mock/dist/index.d.ts",
+        "default": "./llms/mock/dist/index.js"
+      },
+      "default": "./llms/mock/dist/index.js"
+    },
     "./decorator": {
       "require": {
         "types": "./decorator/dist/index.d.cts",

packages/core/src/embeddings/base.ts

@@ -17,6 +17,7 @@ export type EmbeddingInfo = {
 
 export type BaseEmbeddingOptions = {
   logProgress?: boolean;
+  progressCallback?: (current: number, total: number) => void;
 };
 
 export abstract class BaseEmbedding extends TransformComponent<
@@ -138,9 +139,11 @@ export async function batchEmbeddings<T>(
       const embeddings = await embedFunc(curBatch);
 
       resultEmbeddings.push(...embeddings);
-
+      if (options?.progressCallback) {
+        options?.progressCallback?.(i + 1, queue.length);
+      }
       if (options?.logProgress) {
-        console.log(`getting embedding progress: ${i} / ${queue.length}`);
+        console.log(`getting embedding progress: ${i + 1} / ${queue.length}`);
       }
 
       curBatch.length = 0;

packages/core/src/llms/base.ts

@@ -1,15 +1,20 @@
 import { extractText } from "../utils/llms";
 import { streamConverter } from "../utils/stream";
+import { callTool, getToolCallsFromResponse } from "./tool-call";
 import type {
+  ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   CompletionResponse,
+  ExecResponse,
+  ExecStreamResponse,
   LLM,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
   LLMMetadata,
+  PartialToolCall,
   ToolCallLLMMessageOptions,
 } from "./type";
 
@@ -60,13 +65,180 @@ export abstract class BaseLLM<
       AdditionalChatOptions,
       AdditionalMessageOptions
     >,
-  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  ): Promise<AsyncIterable<ChatResponseChunk<AdditionalMessageOptions>>>;
   abstract chat(
     params: LLMChatParamsNonStreaming<
       AdditionalChatOptions,
       AdditionalMessageOptions
     >,
   ): Promise<ChatResponse<AdditionalMessageOptions>>;
+
+  exec(
+    params: LLMChatParamsStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
+  ): Promise<ExecStreamResponse<AdditionalMessageOptions>>;
+  exec(
+    params: LLMChatParamsNonStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
+  ): Promise<ExecResponse<AdditionalMessageOptions>>;
+  async exec(
+    params:
+      | LLMChatParamsStreaming<AdditionalChatOptions, AdditionalMessageOptions>
+      | LLMChatParamsNonStreaming<
+          AdditionalChatOptions,
+          AdditionalMessageOptions
+        >,
+  ): Promise<
+    | ExecResponse<AdditionalMessageOptions>
+    | ExecStreamResponse<AdditionalMessageOptions>
+  > {
+    if (params.stream) {
+      return this.streamExec(params);
+    }
+    const newMessages: ChatMessage<AdditionalMessageOptions>[] = [];
+    const response = await this.chat(params);
+    newMessages.push(response.message);
+    const toolCalls = getToolCallsFromResponse(response);
+    if (params.tools && toolCalls.length > 0) {
+      for (const toolCall of toolCalls) {
+        const toolResultMessage = await callTool<AdditionalMessageOptions>(
+          params.tools,
+          toolCall,
+        );
+        if (toolResultMessage) {
+          newMessages.push(toolResultMessage);
+        }
+      }
+    }
+    return {
+      newMessages,
+      toolCalls,
+    };
+  }
+
+  async streamExec(
+    params: LLMChatParamsStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
+  ): Promise<ExecStreamResponse<AdditionalMessageOptions>> {
+    const responseStream = await this.chat(params);
+    const iterator = responseStream[Symbol.asyncIterator]();
+    const first = await iterator.next();
+
+    // Set firstChunk to null if empty
+    const firstChunk = !first.done ? first.value : null;
+
+    const hasToolCallsInFirst =
+      firstChunk?.options && "toolCall" in firstChunk.options;
+
+    if (!hasToolCallsInFirst) {
+      let content = firstChunk?.delta ?? "";
+      let finished = false;
+      return {
+        stream: (async function* () {
+          if (firstChunk) {
+            yield firstChunk;
+          }
+          for await (const chunk of {
+            [Symbol.asyncIterator]: () => iterator,
+          }) {
+            content += chunk.delta;
+            yield chunk;
+          }
+          finished = true;
+        })(),
+        toolCalls: [],
+        newMessages() {
+          if (!finished) {
+            throw new Error(
+              "New messages are not ready yet. Call newMessages() after the stream is done.",
+            );
+          }
+          return content
+            ? [
+                {
+                  role: "assistant",
+                  content,
+                } as ChatMessage<AdditionalMessageOptions>,
+              ]
+            : [];
+        },
+      };
+    }
+    // Helper function to process a chunk
+    function processChunk(
+      chunk: ChatResponseChunk,
+      toolCallMap: Map<string, PartialToolCall>,
+    ): ChatResponseChunk | null {
+      if (chunk.options && "toolCall" in chunk.options) {
+        // update tool call map
+        for (const toolCall of chunk.options.toolCall as PartialToolCall[]) {
+          if (toolCall.id) {
+            toolCallMap.set(toolCall.id, toolCall);
+          }
+        }
+        // return the current full response with the tool calls
+        const toolCalls = Array.from(toolCallMap.values());
+        return {
+          ...chunk,
+          options: {
+            ...chunk.options,
+            toolCall: toolCalls,
+          },
+        };
+      }
+      return null;
+    }
+    // Collect for tool call
+    let fullResponse: ChatResponseChunk | null = null;
+    const toolCallMap = new Map<string, PartialToolCall>();
+    // Process first chunk
+    fullResponse = processChunk(firstChunk, toolCallMap);
+    // Process remaining chunks
+    while (true) {
+      const next = await iterator.next();
+      if (next.done) break;
+      const chunk = next.value;
+      const potentialFull = processChunk(chunk, toolCallMap);
+      if (potentialFull) {
+        fullResponse = potentialFull;
+      }
+    }
+    if (params.tools && fullResponse) {
+      const toolCalls = getToolCallsFromResponse(fullResponse);
+      const messages: ChatMessage<AdditionalMessageOptions>[] = [];
+      messages.push({
+        role: "assistant",
+        content: "",
+        options: {
+          toolCall: toolCalls,
+        } as AdditionalMessageOptions,
+      });
+      for (const toolCall of toolCalls) {
+        const toolResultMessage = await callTool<AdditionalMessageOptions>(
+          params.tools,
+          toolCall,
+        );
+        if (toolResultMessage) {
+          messages.push(toolResultMessage);
+        }
+      }
+      return {
+        stream: (async function* () {})(),
+        newMessages() {
+          return messages;
+        },
+        toolCalls,
+      };
+    } else {
+      throw new Error("Cannot get tool calls from response");
+    }
+  }
 }
 
 export abstract class ToolCallLLM<

packages/core/src/llms/mock.ts

@@ -1,5 +1,4 @@
-// TODO: move to a test package
-import { ToolCallLLM } from "../llms/base";
+import { ToolCallLLM } from "./base";
 import type {
   ChatResponse,
   ChatResponseChunk,
@@ -9,7 +8,7 @@ import type {
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
   LLMMetadata,
-} from "../llms/type";
+} from "./type";
 
 export class MockLLM extends ToolCallLLM {
   metadata: LLMMetadata;

packages/core/src/llms/tool-call.ts

@@ -0,0 +1,61 @@
+import { stringifyJSONToMessageContent } from "../utils";
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  ToolCall,
+  ToolCallLLMMessageOptions,
+} from "./type";
+
+export const getToolCallsFromResponse = (
+  response:
+    | ChatResponse<ToolCallLLMMessageOptions>
+    | ChatResponseChunk<ToolCallLLMMessageOptions>,
+): ToolCall[] => {
+  let options;
+
+  if ("message" in response) {
+    options = response.message.options;
+  } else {
+    options = response.options;
+  }
+
+  if (options && "toolCall" in options) {
+    return (options.toolCall as ToolCall[]).map((toolCall) => ({
+      ...toolCall,
+      input:
+        // XXX: this is a hack openai returns parsed object for streaming, but not for
+        // non-streaming
+        typeof toolCall.input === "string"
+          ? JSON.parse(toolCall.input)
+          : toolCall.input,
+    }));
+  }
+  return [];
+};
+
+export const callTool = async <
+  AdditionalMessageOptions extends object = object,
+>(
+  tools: BaseTool[],
+  toolCall: ToolCall,
+): Promise<ChatMessage<AdditionalMessageOptions> | null> => {
+  const tool = tools?.find((t) => t.metadata.name === toolCall.name);
+  // TODO: consider using BaseToolWithCall instead of BaseTool to avoid checking for tool.call
+  if (tool && tool.call) {
+    const result = await tool.call(toolCall.input);
+    const toolResultMessage: ChatMessage<AdditionalMessageOptions> = {
+      role: "user",
+      content: stringifyJSONToMessageContent(result),
+      options: {
+        toolResult: {
+          id: toolCall.id,
+          result,
+        },
+      } as AdditionalMessageOptions,
+    };
+    return toolResultMessage;
+  }
+  return null;
+};

packages/core/src/llms/type.ts

@@ -95,6 +95,22 @@ export type ChatResponseChunk<
   options?: undefined | AdditionalMessageOptions;
 };
 
+export interface ExecResponse<
+  AdditionalMessageOptions extends object = object,
+> {
+  newMessages: ChatMessage<AdditionalMessageOptions>[];
+  toolCalls: ToolCall[];
+}
+
+export interface ExecStreamResponse<
+  AdditionalMessageOptions extends object = object,
+> {
+  stream: AsyncIterable<ChatResponseChunk<AdditionalMessageOptions>>;
+  // this is a function as while streaming, the assistant message is not ready yet - can be called after the stream is done
+  newMessages(): ChatMessage<AdditionalMessageOptions>[];
+  toolCalls: ToolCall[];
+}
+
 export interface CompletionResponse {
   text: string;
   /**
@@ -120,9 +136,9 @@ export interface LLMChatParamsBase<
   AdditionalMessageOptions extends object = object,
 > {
   messages: ChatMessage<AdditionalMessageOptions>[];
-  additionalChatOptions?: AdditionalChatOptions;
-  tools?: BaseTool[];
-  responseFormat?: z.ZodType | object;
+  additionalChatOptions?: AdditionalChatOptions | undefined;
+  tools?: BaseTool[] | undefined;
+  responseFormat?: z.ZodType | object | undefined;
 }
 
 export interface LLMChatParamsStreaming<

packages/core/src/utils/index.ts

@@ -70,8 +70,6 @@ export {
   toToolDescriptions,
 } from "./llms";
 
-export { MockLLM } from "./mock";
-
 export * from "./encoding";
 export { objectEntries } from "./object-entries";
 export * from "./stream";

packages/core/tests/agents.test.ts

@@ -1,5 +1,5 @@
 import { LLMAgent, validateAgentParams } from "@llamaindex/core/agent";
-import { MockLLM } from "@llamaindex/core/utils";
+import { MockLLM } from "@llamaindex/core/llms/mock";
 import { expect, test } from "vitest";
 import { ZodError } from "zod";
 

packages/core/tests/embeddings.test.ts

@@ -1,4 +1,9 @@
-import { truncateMaxTokens } from "@llamaindex/core/embeddings";
+import {
+  BaseEmbedding,
+  batchEmbeddings,
+  truncateMaxTokens,
+  type BaseEmbeddingOptions,
+} from "@llamaindex/core/embeddings";
 import { Tokenizers, tokenizers } from "@llamaindex/env/tokenizers";
 import { describe, expect, test } from "vitest";
 
@@ -27,3 +32,77 @@ describe("truncateMaxTokens", () => {
     expect(t.includes("�")).toBe(false);
   });
 });
+
+describe("BaseEmbedding progressCallback", () => {
+  const mockEmbedFunc = async (text: string): Promise<number[]> => {
+    return Array.from({ length: 10 }, () => Math.random());
+  };
+  const mockBatchEmbedFunc = async (
+    texts: string[],
+  ): Promise<Array<number[]>> => {
+    return await Promise.all(texts.map(mockEmbedFunc));
+  };
+  const mockProgressCallback = (current: number, total: number) => {
+    console.log(`Progress: ${current}/${total}`);
+  };
+  const mockLogProgress = true;
+
+  const mockOptions = {
+    logProgress: mockLogProgress,
+    progressCallback: mockProgressCallback,
+  };
+
+  class MockEmbedding extends BaseEmbedding {
+    constructor(options: BaseEmbeddingOptions) {
+      super();
+      this.options = options;
+    }
+
+    private options: BaseEmbeddingOptions;
+
+    async getTextEmbedding(text: string): Promise<number[]> {
+      return await mockEmbedFunc(text);
+    }
+
+    getTextEmbeddings = async (texts: string[]): Promise<Array<number[]>> => {
+      return await mockBatchEmbedFunc(texts);
+    };
+
+    async getTextEmbeddingsBatch(
+      texts: string[],
+      options?: BaseEmbeddingOptions,
+    ): Promise<Array<number[]>> {
+      const mergedOptions = { ...this.options, ...options };
+
+      expect(mergedOptions.progressCallback).toBeDefined();
+
+      return await batchEmbeddings(
+        texts,
+        this.getTextEmbeddings,
+        this.embedBatchSize,
+        mergedOptions,
+      );
+    }
+  }
+
+  test("should call progressCallback with correct values", async () => {
+    // Import and use a real embedding class instead
+
+    const progressCalls: Array<{ current: number; total: number }> = [];
+    const progressCallback = (current: number, total: number) => {
+      progressCalls.push({ current, total });
+    };
+    const texts = ["text1", "text2", "text3"];
+    const embedding = new MockEmbedding({ progressCallback: progressCallback });
+    embedding.embedBatchSize = 1; // Set batch size to 1 for testing
+    // so that progressCallback is called for each item
+    // (otherwise, we'd only get a callback for 3/3, which is fine but less clear)
+    await embedding.getTextEmbeddingsBatch(texts);
+
+    expect(progressCalls).toEqual([
+      { current: 1, total: 3 },
+      { current: 2, total: 3 },
+      { current: 3, total: 3 },
+    ]);
+  });
+});

packages/core/tests/llms/base.test.ts

@@ -0,0 +1,43 @@
+import { MockLLM } from "@llamaindex/core/llms/mock";
+import { describe, expect, it } from "vitest";
+
+// TODO: add tests for tool calls
+describe("BaseLLM exec", () => {
+  it("should stream text response when no tool call is made", async () => {
+    const responseMessage = "This is a response message while streaming";
+
+    const llm = new MockLLM({ responseMessage });
+
+    const { stream, newMessages, toolCalls } = await llm.exec({
+      messages: [{ content: "Hi", role: "user" }],
+      stream: true,
+    });
+
+    expect(() => newMessages()).toThrowError();
+
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+
+    expect(chunks.map((c) => c.delta).join("")).toBe(responseMessage);
+    expect(toolCalls).toEqual([]);
+    expect(newMessages()).toEqual([
+      { content: responseMessage, role: "assistant" },
+    ]);
+  });
+  it("should return text response when no tool call is made", async () => {
+    const responseMessage = "This is a response message";
+
+    const llm = new MockLLM({ responseMessage });
+
+    const { newMessages, toolCalls } = await llm.exec({
+      messages: [{ content: "Hi", role: "user" }],
+    });
+
+    expect(newMessages).toEqual([
+      { content: responseMessage, role: "assistant" },
+    ]);
+    expect(toolCalls).toEqual([]);
+  });
+});

packages/core/tests/memory/memory.test.ts

@@ -1,7 +1,7 @@
 import { Settings } from "@llamaindex/core/global";
 import type { ChatMessage, LLM } from "@llamaindex/core/llms";
+import { MockLLM } from "@llamaindex/core/llms/mock";
 import { createMemory, Memory, staticBlock } from "@llamaindex/core/memory";
-import { MockLLM } from "@llamaindex/core/utils";
 import type { Tokenizer } from "@llamaindex/env/tokenizers";
 import {
   afterAll,

packages/core/tests/simple-chat-engine.test.ts

@@ -1,6 +1,6 @@
 import { SimpleChatEngine } from "@llamaindex/core/chat-engine";
+import { MockLLM } from "@llamaindex/core/llms/mock";
 import { Memory } from "@llamaindex/core/memory";
-import { MockLLM } from "@llamaindex/core/utils";
 import { describe, expect, test } from "vitest";
 
 describe("SimpleChatEngine", () => {

packages/experimental/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/experimental
 
+## 0.0.196
+
+### Patch Changes
+
+- llamaindex@0.11.19
+
 ## 0.0.195
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.195",
+  "version": "0.0.196",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,16 @@
 # llamaindex
 
+## 0.11.19
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+  - @llamaindex/workflow@1.1.15
+  - @llamaindex/cloud@4.0.24
+  - @llamaindex/node-parser@2.0.15
+
 ## 0.11.18
 
 ### Patch Changes

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.11.18",
+  "version": "0.11.19",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/tests/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/core-test
 
+## 0.1.10
+
+### Patch Changes
+
+- Updated dependencies [a1fdb07]
+  - @llamaindex/openai@0.4.9
+
 ## 0.1.9
 
 ### Patch Changes

packages/llamaindex/tests/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llamaindex-test",
   "private": true,
-  "version": "0.1.9",
+  "version": "0.1.10",
   "type": "module",
   "scripts": {
     "test": "vitest run"

packages/node-parser/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/node-parser
 
+## 2.0.15
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 2.0.14
 
 ### Patch Changes

packages/node-parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/node-parser",
-  "version": "2.0.14",
+  "version": "2.0.15",
   "description": "Node parser for LlamaIndex",
   "type": "module",
   "exports": {

packages/providers/anthropic/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @llamaindex/anthropic
 
+## 0.3.17
+
+### Patch Changes
+
+- ddc0eaf: anthropic: stream partial tool calls
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.3.16
 
 ### Patch Changes

packages/providers/anthropic/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/anthropic",
   "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.3.16",
+  "version": "0.3.17",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/anthropic/src/llm.ts

@@ -578,7 +578,6 @@ export class Anthropic extends ToolCallLLM<
     });
 
     let currentToolCall: PartialToolCall | null = null;
-    let accumulatedToolInput = "";
 
     for await (const part of stream) {
       const textContent =
@@ -607,7 +606,13 @@ export class Anthropic extends ToolCallLLM<
           name: part.content_block.name,
           input: "",
         };
-        accumulatedToolInput = "";
+        yield {
+          raw: part,
+          delta: "",
+          options: {
+            toolCall: [currentToolCall],
+          },
+        };
         continue;
       }
 
@@ -616,7 +621,14 @@ export class Anthropic extends ToolCallLLM<
         part.delta.type === "input_json_delta" &&
         currentToolCall
       ) {
-        accumulatedToolInput += part.delta.partial_json;
+        currentToolCall.input += part.delta.partial_json;
+        yield {
+          raw: part,
+          delta: "",
+          options: {
+            toolCall: [currentToolCall],
+          },
+        };
         continue;
       }
 
@@ -625,13 +637,7 @@ export class Anthropic extends ToolCallLLM<
           raw: part,
           delta: "",
           options: {
-            toolCall: [
-              {
-                id: currentToolCall.id,
-                name: currentToolCall.name,
-                input: accumulatedToolInput,
-              },
-            ],
+            toolCall: [currentToolCall],
           },
         };
         currentToolCall = null;

packages/providers/assemblyai/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/assemblyai
 
+## 0.1.14
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.1.13
 
 ### Patch Changes

packages/providers/assemblyai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/assemblyai",
   "description": "AssemblyAI Reader for LlamaIndex",
-  "version": "0.1.13",
+  "version": "0.1.14",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/aws/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/community
 
+## 0.0.110
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.0.109
 
 ### Patch Changes

packages/providers/aws/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/aws",
   "description": "AWS package for LlamaIndexTS",
-  "version": "0.0.109",
+  "version": "0.0.110",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/providers/clip/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @llamaindex/clip
 
+## 0.0.65
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+- Updated dependencies [a1fdb07]
+  - @llamaindex/core@0.6.15
+  - @llamaindex/openai@0.4.9
+
 ## 0.0.64
 
 ### Patch Changes

packages/providers/clip/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/clip",
   "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.64",
+  "version": "0.0.65",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/cohere/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/cohere
 
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.0.28
 
 ### Patch Changes

packages/providers/cohere/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/cohere",
   "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.28",
+  "version": "0.0.29",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepinfra/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @llamaindex/deepinfra
 
+## 0.0.65
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+- Updated dependencies [a1fdb07]
+  - @llamaindex/core@0.6.15
+  - @llamaindex/openai@0.4.9
+
 ## 0.0.64
 
 ### Patch Changes

packages/providers/deepinfra/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepinfra",
   "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.64",
+  "version": "0.0.65",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepseek/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/deepseek
 
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [a1fdb07]
+  - @llamaindex/openai@0.4.9
+
 ## 0.0.25
 
 ### Patch Changes

packages/providers/deepseek/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepseek",
   "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.25",
+  "version": "0.0.26",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/discord/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/discord
 
+## 0.1.14
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.1.13
 
 ### Patch Changes

packages/providers/discord/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/discord",
   "description": "Discord Reader for LlamaIndex",
-  "version": "0.1.13",
+  "version": "0.1.14",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/excel/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/excel
 
+## 0.1.15
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.1.14
 
 ### Patch Changes

packages/providers/excel/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/excel",
   "description": "Excel Reader for LlamaIndex",
-  "version": "0.1.14",
+  "version": "0.1.15",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/fireworks/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/fireworks
 
+## 0.0.25
+
+### Patch Changes
+
+- Updated dependencies [a1fdb07]
+  - @llamaindex/openai@0.4.9
+
 ## 0.0.24
 
 ### Patch Changes

packages/providers/fireworks/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/fireworks",
   "description": "Fireworks Adapter for LlamaIndex",
-  "version": "0.0.24",
+  "version": "0.0.25",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/google/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/google
 
+## 0.3.14
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.3.13
 
 ### Patch Changes

packages/providers/google/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/google",
   "description": "Google Adapter for LlamaIndex",
-  "version": "0.3.13",
+  "version": "0.3.14",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/groq/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/groq
 
+## 0.0.81
+
+### Patch Changes
+
+- Updated dependencies [a1fdb07]
+  - @llamaindex/openai@0.4.9
+
 ## 0.0.80
 
 ### Patch Changes

packages/providers/groq/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/groq",
   "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.80",
+  "version": "0.0.81",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/huggingface/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @llamaindex/huggingface
 
+## 0.1.19
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+- Updated dependencies [a1fdb07]
+  - @llamaindex/core@0.6.15
+  - @llamaindex/openai@0.4.9
+
 ## 0.1.18
 
 ### Patch Changes

packages/providers/huggingface/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/huggingface",
   "description": "Huggingface Adapter for LlamaIndex",
-  "version": "0.1.18",
+  "version": "0.1.19",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/jinaai/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @llamaindex/jinaai
 
+## 0.0.25
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+- Updated dependencies [a1fdb07]
+  - @llamaindex/core@0.6.15
+  - @llamaindex/openai@0.4.9
+
 ## 0.0.24
 
 ### Patch Changes

packages/providers/jinaai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/jinaai",
   "description": "JinaAI Adapter for LlamaIndex",
-  "version": "0.0.24",
+  "version": "0.0.25",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/mistral/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/mistral
 
+## 0.1.15
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.1.14
 
 ### Patch Changes

packages/providers/mistral/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/mistral",
   "description": "Mistral Adapter for LlamaIndex",
-  "version": "0.1.14",
+  "version": "0.1.15",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/mixedbread/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/mixedbread
 
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.0.28
 
 ### Patch Changes

packages/providers/mixedbread/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/mixedbread",
   "description": "Mixedbread Adapter for LlamaIndex",
-  "version": "0.0.28",
+  "version": "0.0.29",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/notion/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/notion
 
+## 0.1.14
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.1.13
 
 ### Patch Changes

packages/providers/notion/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/notion",
   "description": "Notion Reader for LlamaIndex",
-  "version": "0.1.13",
+  "version": "0.1.14",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/ollama/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/ollama
 
+## 0.1.15
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.1.14
 
 ### Patch Changes

packages/providers/ollama/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/ollama",
   "description": "Ollama Adapter for LlamaIndex",
-  "version": "0.1.14",
+  "version": "0.1.15",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/openai/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @llamaindex/openai
 
+## 0.4.9
+
+### Patch Changes
+
+- a1fdb07: feat: multi-turn image generation support
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.4.8
 
 ### Patch Changes

packages/providers/openai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/openai",
   "description": "OpenAI Adapter for LlamaIndex",
-  "version": "0.4.8",
+  "version": "0.4.9",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/openai/src/responses.ts

@@ -19,9 +19,9 @@ import {
 import type { StoredValue } from "@llamaindex/core/schema";
 import { extractText } from "@llamaindex/core/utils";
 import { getEnv } from "@llamaindex/env";
+import { Tokenizers } from "@llamaindex/env/tokenizers";
 
 import { wrapEventCaller } from "@llamaindex/core/decorator";
-import { Tokenizers } from "@llamaindex/env/tokenizers";
 import {
   ALL_AVAILABLE_OPENAI_MODELS,
   isFunctionCallingModel,
@@ -41,6 +41,11 @@ import {
   type ClientOptions as OpenAIClientOptions,
 } from "openai";
 
+// Extend ToolCallLLMMessageOptions to include image_id for multi-turn support
+export type OpenAIResponsesMessageOptions = ToolCallLLMMessageOptions & {
+  image_id?: string;
+};
+
 export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   model: string;
   temperature: number;
@@ -55,9 +60,8 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   additionalSessionOptions?:
     | undefined
     | Omit<Partial<OpenAIClientOptions>, "apiKey" | "maxRetries" | "timeout">;
-  lazySession: () => Promise<LLMInstance>;
   #session: Promise<LLMInstance> | null = null;
-
+  lazySession: () => Promise<LLMInstance>;
   trackPreviousResponses: boolean;
   store: boolean;
   user: string;
@@ -193,6 +197,17 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
     );
   }
 
+  private isImageGenerationCallWithId(
+    item: OpenAILLM.Responses.ResponseOutputItem,
+  ): item is OpenAILLM.Responses.ResponseOutputItem.ImageGenerationCall & {
+    image_id?: string;
+  } {
+    return (
+      this.isImageGenerationCall(item) &&
+      typeof (item as { image_id?: unknown }).image_id === "string"
+    );
+  }
+
   private isResponseCreatedEvent(
     event: OpenAILLM.Responses.ResponseStreamEvent,
   ): event is OpenAILLM.Responses.ResponseCreatedEvent {
@@ -316,7 +331,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
 
   private parseResponseOutput(
     response: OpenAILLM.Responses.ResponseOutputItem[],
-  ): ChatMessage<ToolCallLLMMessageOptions> {
+  ): ChatMessage<OpenAIResponsesMessageOptions> {
     const message = this.createInitialMessage();
     const options = this.createInitialOptions();
     const toolCall = this.extractToolCalls(response);
@@ -334,6 +349,13 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
           mimeType: "image/png",
         };
         addContentPart(message, imagePart);
+
+        // Add image_id to options if present for multi-turn support
+        if (this.isImageGenerationCallWithId(item) && item.image_id) {
+          if (!options.image_id) {
+            options.image_id = item.image_id;
+          }
+        }
       } else if (this.isBuiltInToolCall(item)) {
         options.built_in_tool_calls.push(item);
       } else if (this.isReasoning(item)) {
@@ -462,15 +484,37 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   }
 
   private createBaseRequestParams(
-    messages: ChatMessage<ToolCallLLMMessageOptions>[],
+    messages: ChatMessage<OpenAIResponsesMessageOptions>[],
     tools: BaseTool[] | undefined,
     additionalChatOptions: OpenAIResponsesChatOptions | undefined,
   ) {
+    // Check for image_id from previous responses
+    let imageId: string | undefined;
+    for (const message of messages) {
+      if (message.options?.image_id) {
+        imageId = message.options.image_id;
+        break;
+      }
+    }
+
+    // Process built-in tools to add image_id if needed
+    const processedBuiltInTools = this.builtInTools
+      ? [...this.builtInTools]
+      : [];
+    if (imageId) {
+      processedBuiltInTools.forEach((tool) => {
+        if (tool.type === "image_generation") {
+          (tool as OpenAILLM.Responses.Tool & { image_id?: string }).image_id =
+            imageId;
+        }
+      });
+    }
+
     const baseRequestParams = <OpenAILLM.Responses.ResponseCreateParams>{
       model: this.model,
       include: this.include,
       input: this.toOpenAIResponseMessages(messages),
-      tools: this.builtInTools ? [...this.builtInTools] : [],
+      tools: processedBuiltInTools,
       instructions: this.instructions,
       max_output_tokens: this.maxOutputTokens,
       previous_response_id: this.previousResponseId,
@@ -497,28 +541,28 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   chat(
     params: LLMChatParamsStreaming<
       OpenAIResponsesChatOptions,
-      ToolCallLLMMessageOptions
+      OpenAIResponsesMessageOptions
     >,
-  ): Promise<AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>>;
+  ): Promise<AsyncIterable<ChatResponseChunk<OpenAIResponsesMessageOptions>>>;
   chat(
     params: LLMChatParamsNonStreaming<
       OpenAIResponsesChatOptions,
-      ToolCallLLMMessageOptions
+      OpenAIResponsesMessageOptions
     >,
-  ): Promise<ChatResponse<ToolCallLLMMessageOptions>>;
+  ): Promise<ChatResponse<OpenAIResponsesMessageOptions>>;
   async chat(
     params:
       | LLMChatParamsNonStreaming<
           OpenAIResponsesChatOptions,
-          ToolCallLLMMessageOptions
+          OpenAIResponsesMessageOptions
         >
       | LLMChatParamsStreaming<
           OpenAIResponsesChatOptions,
-          ToolCallLLMMessageOptions
+          OpenAIResponsesMessageOptions
         >,
   ): Promise<
-    | ChatResponse<ToolCallLLMMessageOptions>
-    | AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>
+    | ChatResponse<OpenAIResponsesMessageOptions>
+    | AsyncIterable<ChatResponseChunk<OpenAIResponsesMessageOptions>>
   > {
     const { messages, stream, tools, responseFormat, additionalChatOptions } =
       params;
@@ -570,7 +614,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   private createResponseChunk(
     event: OpenAILLM.Responses.ResponseStreamEvent,
     state: StreamState,
-  ): ChatResponseChunk<ToolCallLLMMessageOptions> {
+  ): ChatResponseChunk<OpenAIResponsesMessageOptions> {
     return {
       raw: event,
       delta: state.delta,
@@ -585,7 +629,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   @wrapEventCaller
   protected async *streamChat(
     baseRequestParams: OpenAILLM.Responses.ResponseCreateParams,
-  ): AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>> {
+  ): AsyncIterable<ChatResponseChunk<OpenAIResponsesMessageOptions>> {
     const streamState = this.initalizeStreamState();
 
     const stream = await (
@@ -618,18 +662,18 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   }
 
   private isToolResultPresent(
-    options: ToolCallLLMMessageOptions,
+    options: OpenAIResponsesMessageOptions,
   ): options is ToolResultOptions {
     return "toolResult" in options;
   }
 
   private isToolCallPresent(
-    options: ToolCallLLMMessageOptions,
+    options: OpenAIResponsesMessageOptions,
   ): options is ToolCallOptions {
     return "toolCall" in options;
   }
 
-  private isUserMessage(message: ChatMessage<ToolCallLLMMessageOptions>) {
+  private isUserMessage(message: ChatMessage<OpenAIResponsesMessageOptions>) {
     return message.role === "user";
   }
 
@@ -706,7 +750,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   }
 
   private convertToOpenAIUserMessage(
-    message: ChatMessage<ToolCallLLMMessageOptions>,
+    message: ChatMessage<OpenAIResponsesMessageOptions>,
   ) {
     const messageContent = this.processMessageContent(message.content);
     return {
@@ -716,7 +760,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   }
 
   private defaultOpenAIResponseMessage(
-    message: ChatMessage<ToolCallLLMMessageOptions>,
+    message: ChatMessage<OpenAIResponsesMessageOptions>,
   ) {
     const response: OpenAILLM.Responses.ResponseInputItem = {
       role: this.toOpenAIResponsesRole(message.role),
@@ -726,7 +770,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   }
 
   toOpenAIResponseMessage(
-    message: ChatMessage<ToolCallLLMMessageOptions>,
+    message: ChatMessage<OpenAIResponsesMessageOptions>,
   ):
     | OpenAILLM.Responses.ResponseInputItem
     | OpenAILLM.Responses.ResponseInputItem[] {
@@ -744,7 +788,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
   }
 
   toOpenAIResponseMessages(
-    messages: ChatMessage<ToolCallLLMMessageOptions>[],
+    messages: ChatMessage<OpenAIResponsesMessageOptions>[],
   ): OpenAILLM.Responses.ResponseInput {
     const finalMessages: OpenAILLM.Responses.ResponseInputItem[] = [];
     for (const message of messages) {

packages/providers/openai/src/utils.ts

@@ -216,6 +216,7 @@ export type ResponsesAdditionalOptions = {
   refusal?: string;
   reasoning?: OpenAILLM.Responses.ResponseReasoningItem;
   usage?: OpenAILLM.Responses.ResponseUsage;
+  image_id?: string;
 };
 
 export type StreamState = {

packages/providers/perplexity/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @llamaindex/perplexity
 
+## 0.0.22
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+- Updated dependencies [a1fdb07]
+  - @llamaindex/core@0.6.15
+  - @llamaindex/openai@0.4.9
+
 ## 0.0.21
 
 ### Patch Changes

packages/providers/perplexity/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/perplexity",
   "description": "Perplexity Adapter for LlamaIndex",
-  "version": "0.0.21",
+  "version": "0.0.22",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/portkey-ai/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/portkey-ai
 
+## 0.0.57
+
+### Patch Changes
+
+- Updated dependencies [7ad3411]
+- Updated dependencies [5da5b3c]
+  - @llamaindex/core@0.6.15
+
 ## 0.0.56
 
 ### Patch Changes