Merge branch 'main' into ms/add-image-tool

.changeset/tasty-beds-heal.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/openai": patch
+"@llamaindex/core": patch
+---
+
+Add support for builtin image generation tool

examples/models/openai/responses/image-gen.ts

@@ -0,0 +1,35 @@
+import { openaiResponses } from "@llamaindex/openai";
+import fs from "fs";
+import { MessageContentDetail } from "llamaindex";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4.1-mini",
+    builtInTools: [{ type: "image_generation" }],
+  });
+
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content:
+          "Generate an image of a cute tiny llama wearing a hat playing with a cat on a meadow",
+      },
+    ],
+  });
+  const content = response.message.content as MessageContentDetail[];
+
+  // This call returns a message with two parts, an image and a text part, get both parts
+  const imagePart = content.find((part) => part.type === "image");
+  const textPart = content.find((part) => part.type === "text");
+
+  // write the image to a file
+  fs.writeFileSync(
+    "llama.png",
+    Buffer.from(imagePart?.data as string, "base64"),
+  );
+  // and print out the text part
+  console.log(textPart?.text);
+}
+
+main().catch(console.error);

examples/models/openai/responses/interpreter.ts

@@ -0,0 +1,31 @@
+import { openaiResponses } from "@llamaindex/openai";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4.1",
+    builtInTools: [
+      {
+        type: "code_interpreter",
+        container: { type: "auto" },
+      },
+    ],
+  });
+
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "system",
+        content:
+          "You are a personal math tutor. When asked a math question, write and run code to answer the question.",
+      },
+      {
+        role: "user",
+        content: "I need to solve the equation 3x + 11 = 14. Can you help me?",
+      },
+    ],
+  });
+
+  console.log(response.message.content);
+}
+
+main().catch(console.error);

examples/models/openai/responses/web-search.ts

@@ -7,7 +7,6 @@ async function main() {
     builtInTools: [{ type: "web_search_preview" }],
   });
 
-  // Streaming chat example
   const response = await llm.chat({
     messages: [
       {

packages/core/src/llms/index.ts

@@ -36,3 +36,4 @@ export type {
   ToolResult,
   ToolResultOptions,
 } from "./type";
+export { addContentPart } from "./utils";

packages/core/src/llms/utils.ts

@@ -0,0 +1,28 @@
+import type {
+  ChatMessage,
+  MessageContentImageDataDetail,
+  MessageContentTextDetail,
+} from "./type";
+
+export function addContentPart<AdditionalMessageOptions extends object>(
+  message: ChatMessage<AdditionalMessageOptions>,
+  part: MessageContentTextDetail | MessageContentImageDataDetail,
+): void {
+  if (part.type === "text") {
+    if (typeof message.content === "string") {
+      message.content += part.text;
+    } else {
+      message.content.push(part);
+    }
+  } else {
+    if (typeof message.content === "string") {
+      if (message.content === "") {
+        message.content = [part];
+      } else {
+        message.content = [{ type: "text", text: message.content }, part];
+      }
+    } else {
+      message.content.push(part);
+    }
+  }
+}

packages/providers/openai/package.json

@@ -31,15 +31,15 @@
     "test": "vitest"
   },
   "devDependencies": {
-    "zod": "^3.24.2",
     "@llamaindex/core": "workspace:*",
-    "@llamaindex/env": "workspace:*"
+    "@llamaindex/env": "workspace:*",
+    "zod": "^3.24.2"
   },
   "peerDependencies": {
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*"
   },
   "dependencies": {
-    "openai": "^4.90.0"
+    "openai": "^4.102.0"
   }
 }

packages/providers/openai/src/responses.ts

@@ -1,4 +1,5 @@
 import {
+  addContentPart,
   ToolCallLLM,
   type BaseTool,
   type ChatMessage,
@@ -8,6 +9,7 @@ import {
   type LLMChatParamsStreaming,
   type LLMMetadata,
   type MessageContent,
+  type MessageContentImageDataDetail,
   type MessageType,
   type PartialToolCall,
   type ToolCallLLMMessageOptions,
@@ -182,6 +184,15 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
     return item.type === "function_call";
   }
 
+  private isImageGenerationCall(
+    item: OpenAILLM.Responses.ResponseOutputItem,
+  ): item is OpenAILLM.Responses.ResponseOutputItem.ImageGenerationCall {
+    return (
+      item.type === "image_generation_call" &&
+      typeof (item as { result?: unknown }).result === "string"
+    );
+  }
+
   private isResponseCreatedEvent(
     event: OpenAILLM.Responses.ResponseStreamEvent,
   ): event is OpenAILLM.Responses.ResponseCreatedEvent {
@@ -313,7 +324,16 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
     for (const item of response) {
       if (this.isMessageBlock(item)) {
         const outputContent = this.handleResponseOutputMessage(item, options);
-        message.content = outputContent;
+        if (outputContent) {
+          addContentPart(message, { type: "text", text: outputContent });
+        }
+      } else if (this.isImageGenerationCall(item) && item.result) {
+        const imagePart: MessageContentImageDataDetail = {
+          type: "image",
+          data: item.result,
+          mimeType: "image/png",
+        };
+        addContentPart(message, imagePart);
       } else if (this.isBuiltInToolCall(item)) {
         options.built_in_tool_calls.push(item);
       } else if (this.isReasoning(item)) {

packages/providers/openai/src/utils.ts

@@ -129,6 +129,7 @@ export const ALL_AVAILABLE_OPENAI_MODELS = {
   ...O1_MODELS,
   ...O3_MODELS,
   ...O4_MODELS,
+  "codex-mini-latest": { contextWindow: 200000 },
 } satisfies Record<ChatModel, { contextWindow: number }>;
 
 export function isFunctionCallingModel(llm: LLM): llm is OpenAI {

pnpm-lock.yaml

@@ -136,7 +136,7 @@ importers:
         version: 1.6.0(@aws-sdk/credential-provider-web-identity@3.810.0)
       ai:
         specifier: ^3.4.33
-        version: 3.4.33(openai@4.94.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2))(react@19.1.0)(sswr@2.2.0(svelte@5.27.0))(svelte@5.27.0)(vue@3.5.13(typescript@5.7.3))(zod@3.24.2)
+        version: 3.4.33(openai@4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2))(react@19.1.0)(sswr@2.2.0(svelte@5.27.0))(svelte@5.27.0)(vue@3.5.13(typescript@5.7.3))(zod@3.24.2)
       class-variance-authority:
         specifier: ^0.7.0
         version: 0.7.1
@@ -1341,8 +1341,8 @@ importers:
   packages/providers/openai:
     dependencies:
       openai:
-        specifier: ^4.90.0
-        version: 4.94.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2)
+        specifier: ^4.102.0
+        version: 4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2)
     devDependencies:
       '@llamaindex/core':
         specifier: workspace:*
@@ -1450,7 +1450,7 @@ importers:
     dependencies:
       chromadb:
         specifier: 1.10.3
-        version: 1.10.3(@google/generative-ai@0.1.3)(cohere-ai@7.17.0)(openai@4.94.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.25.7))(voyageai@0.0.3-1)
+        version: 1.10.3(@google/generative-ai@0.1.3)(cohere-ai@7.17.0)(openai@4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.25.7))(voyageai@0.0.3-1)
       chromadb-default-embed:
         specifier: ^2.13.2
         version: 2.14.0
@@ -10783,7 +10783,6 @@ packages:
   node-domexception@1.0.0:
     resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
     engines: {node: '>=10.5.0'}
-    deprecated: Use your platform's native DOMException instead
 
   node-fetch-native@1.6.6:
     resolution: {integrity: sha512-8Mc2HhqPdlIfedsuZoc3yioPuzp6b+L5jRCRY1QzuWZh2EGJVQrGppC6V6cF0bLdbW0+O2YpqCA25aF/1lvipQ==}
@@ -10971,6 +10970,18 @@ packages:
     resolution: {integrity: sha512-zy1wx4+P3PfhXSEPJNtZmJXfhkkIaxU1VauWIrDZw1O7uJRDRJtKr9n3Ic4NgbA16KyOxOXO2ng9gYwCdXuSXA==}
     engines: {node: '>=18'}
 
+  openai@4.102.0:
+    resolution: {integrity: sha512-CWk15CMhPSHNZnjz+6rwVYV551xaC8CwOd7/zxImrC1btEo37dX/Ii5tBKWfqqxqyzpJ6p3Y4bICzzKhW03WhQ==}
+    hasBin: true
+    peerDependencies:
+      ws: ^8.18.0
+      zod: ^3.23.8
+    peerDependenciesMeta:
+      ws:
+        optional: true
+      zod:
+        optional: true
+
   openai@4.94.0:
     resolution: {integrity: sha512-WVmr9HWcwfouLJ7R3UHd2A93ClezTPuJljQxkCYQAL15Sjyt+FBNoqEz5MHSdH/ebQrVyvRhFyn/bvdqtSPyIA==}
     hasBin: true
@@ -20298,7 +20309,7 @@ snapshots:
     dependencies:
       humanize-ms: 1.2.1
 
-  ai@3.4.33(openai@4.94.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2))(react@19.1.0)(sswr@2.2.0(svelte@5.27.0))(svelte@5.27.0)(vue@3.5.13(typescript@5.7.3))(zod@3.24.2):
+  ai@3.4.33(openai@4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2))(react@19.1.0)(sswr@2.2.0(svelte@5.27.0))(svelte@5.27.0)(vue@3.5.13(typescript@5.7.3))(zod@3.24.2):
     dependencies:
       '@ai-sdk/provider': 0.0.26
       '@ai-sdk/provider-utils': 1.0.22(zod@3.24.2)
@@ -20314,7 +20325,7 @@ snapshots:
       secure-json-parse: 2.7.0
       zod-to-json-schema: 3.24.5(zod@3.24.2)
     optionalDependencies:
-      openai: 4.94.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2)
+      openai: 4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2)
       react: 19.1.0
       sswr: 2.2.0(svelte@5.27.0)
       svelte: 5.27.0
@@ -20941,14 +20952,14 @@ snapshots:
     transitivePeerDependencies:
       - bare-buffer
 
-  chromadb@1.10.3(@google/generative-ai@0.1.3)(cohere-ai@7.17.0)(openai@4.94.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.25.7))(voyageai@0.0.3-1):
+  chromadb@1.10.3(@google/generative-ai@0.1.3)(cohere-ai@7.17.0)(openai@4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.25.7))(voyageai@0.0.3-1):
     dependencies:
       cliui: 8.0.1
       isomorphic-fetch: 3.0.0
     optionalDependencies:
       '@google/generative-ai': 0.1.3
       cohere-ai: 7.17.0
-      openai: 4.94.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.25.7)
+      openai: 4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.25.7)
       voyageai: 0.0.3-1
     transitivePeerDependencies:
       - encoding
@@ -25770,7 +25781,7 @@ snapshots:
       is-inside-container: 1.0.0
       is-wsl: 3.1.0
 
-  openai@4.94.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2):
+  openai@4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.24.2):
     dependencies:
       '@types/node': 18.19.86
       '@types/node-fetch': 2.6.12
@@ -25785,6 +25796,22 @@ snapshots:
     transitivePeerDependencies:
       - encoding
 
+  openai@4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.25.7):
+    dependencies:
+      '@types/node': 18.19.86
+      '@types/node-fetch': 2.6.12
+      abort-controller: 3.0.0
+      agentkeepalive: 4.6.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.7.0
+    optionalDependencies:
+      ws: 8.18.1(bufferutil@4.0.9)
+      zod: 3.25.7
+    transitivePeerDependencies:
+      - encoding
+    optional: true
+
   openai@4.94.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.25.7):
     dependencies:
       '@types/node': 18.19.86