Release 0.9.15 (#1807)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>

apps/next/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/doc
 
+## 0.2.4
+
+### Patch Changes
+
+- 9c63f3f: Add support for openai responses api
+- Updated dependencies [9c63f3f]
+- Updated dependencies [c515a32]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+  - @llamaindex/workflow@1.0.2
+  - llamaindex@0.9.15
+  - @llamaindex/cloud@4.0.2
+  - @llamaindex/node-parser@2.0.2
+  - @llamaindex/readers@3.0.2
+
 ## 0.2.3
 
 ### Patch Changes

apps/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/doc",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "private": true,
   "scripts": {
     "postinstall": "fumadocs-mdx",

apps/next/src/content/docs/llamaindex/modules/llms/openai.mdx

@@ -46,6 +46,156 @@ or
 Settings.llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0, apiKey: <YOUR_API_KEY>, baseURL: "https://api.scaleway.ai/v1" });
 ```
 
+## Using OpenAI Responses API
+
+The OpenAI Responses API provides enhanced functionality for handling complex interactions, including built-in tools, annotations, and streaming responses. Here's how to use it:
+
+### Basic Setup
+
+```ts
+import { openaiResponses } from "@llamaindex/openai";
+
+const llm = openaiResponses({
+  model: "gpt-4o",
+  temperature: 0.1,
+  maxOutputTokens: 1000
+});
+```
+
+### Message Content Types
+
+The API supports different types of message content, including text and images:
+
+```ts
+const response = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: [
+        {
+          type: "input_text",
+          text: "What's in this image?"
+        },
+        {
+          type: "input_image",
+          image_url: "https://example.com/image.jpg",
+          detail: "auto" // Optional: can be "auto", "low", or "high"
+        }
+      ]
+    }
+  ]
+});
+```
+
+### Advanced Features
+
+#### Built-in Tools
+
+```ts
+const llm = openaiResponses({
+  model: "gpt-4o",
+  builtInTools: [
+    {
+      type: "function",
+      name: "search_files",
+      description: "Search through available files"
+    }
+  ],
+  strict: true // Enable strict mode for tool calls
+});
+```
+
+#### Response Tracking and Storage
+
+```ts
+const llm = openaiResponses({
+  trackPreviousResponses: true, // Enable response tracking
+  store: true, // Store responses for future reference
+  user: "user-123", // Associate responses with a user
+  callMetadata: { // Add custom metadata
+    sessionId: "session-123",
+    context: "customer-support"
+  }
+});
+```
+
+#### Streaming Responses
+
+```ts
+const response = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: "Generate a long response"
+    }
+  ],
+  stream: true // Enable streaming
+});
+
+for await (const chunk of response) {
+  console.log(chunk.delta); // Process each chunk of the response
+}
+```
+
+### Configuration Options
+
+The OpenAI Responses API supports various configuration options:
+
+```ts
+const llm = openaiResponses({
+  // Model and basic settings
+  model: "gpt-4o",
+  temperature: 0.1,
+  topP: 1,
+  maxOutputTokens: 1000,
+  
+  // API configuration
+  apiKey: "your-api-key",
+  baseURL: "custom-endpoint",
+  maxRetries: 10,
+  timeout: 60000,
+  
+  // Response handling
+  trackPreviousResponses: false,
+  store: false,
+  strict: false,
+  
+  // Additional options
+  instructions: "Custom instructions for the model",
+  truncation: "auto", // Can be "auto", "disabled", or null
+  include: ["citations", "reasoning"] // Specify what to include in responses
+});
+```
+
+### Response Structure
+
+The API returns responses with rich metadata and optional annotations:
+
+```ts
+interface ResponseStructure {
+  message: {
+    content: string;
+    role: "assistant";
+    options: {
+      built_in_tool_calls: Array<ToolCall>;
+      annotations?: Array<Citation | URLCitation | FilePath>;
+      refusal?: string;
+      reasoning?: ReasoningItem;
+      usage?: ResponseUsage;
+      toolCall?: Array<PartialToolCall>;
+    }
+  }
+}
+```
+
+### Best Practices
+
+1. Use `trackPreviousResponses` when you need conversation continuity
+2. Enable `strict` mode when using tools to ensure accurate function calls
+3. Set appropriate `maxOutputTokens` to control response length
+4. Use `annotations` to track citations and references in responses
+5. Implement error handling for potential API failures and retries
+
 ## Using JSON Response Format
 
 You can configure OpenAI to return responses in JSON format:
@@ -73,6 +223,112 @@ Settings.llm = new OpenAI({
 });
 ```
 
+## Response Formats
+
+The OpenAI LLM supports different response formats to structure the output in specific ways. There are two main approaches to formatting responses:
+
+### 1. JSON Object Format
+
+The simplest way to get structured JSON responses is using the `json_object` response format:
+
+```ts
+Settings.llm = new OpenAI({ 
+  model: "gpt-4o", 
+  temperature: 0,
+  responseFormat: { type: "json_object" }  
+});
+
+const response = await llm.chat({
+  messages: [
+    {
+      role: "system",
+      content: "You are a helpful assistant that outputs JSON."
+    },
+    {
+      role: "user", 
+      content: "Summarize this meeting transcript"
+    }
+  ]
+});
+
+// Response will be valid JSON
+console.log(response.message.content);
+```
+
+### 2. Schema Validation with Zod
+
+For more robust type safety and validation, you can use Zod schemas to define the expected response structure:
+
+```ts
+import { z } from "zod";
+
+// Define the response schema
+const meetingSchema = z.object({
+  summary: z.string(),
+  participants: z.array(z.string()),
+  actionItems: z.array(z.string()),
+  nextSteps: z.string()
+});
+
+// Configure the LLM with the schema
+Settings.llm = new OpenAI({ 
+  model: "gpt-4o", 
+  temperature: 0,
+  responseFormat: meetingSchema
+});
+
+const response = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: "Summarize this meeting transcript" 
+    }
+  ]
+});
+
+// Response will be typed and validated according to the schema
+const result = response.message.content;
+console.log(result.summary);
+console.log(result.actionItems);
+```
+
+### Response Format Options
+
+The response format can be configured in two ways:
+
+1. At LLM initialization:
+```ts
+const llm = new OpenAI({
+  model: "gpt-4o",
+  responseFormat: { type: "json_object" } // or a Zod schema
+});
+```
+
+2. Per request:
+```ts
+const response = await llm.chat({
+  messages: [...],
+  responseFormat: { type: "json_object" } // or a Zod schema
+});
+```
+
+The response format options are:
+
+- `{ type: "json_object" }` - Returns responses as JSON objects
+- `zodSchema` - A Zod schema that defines and validates the response structure
+
+### Best Practices
+
+1. Use JSON object format for simple structured responses
+2. Use Zod schemas when you need:
+   - Type safety
+   - Response validation
+   - Complex nested structures
+   - Specific field constraints
+3. Set a low temperature (e.g. 0) when using structured outputs for more reliable formatting
+4. Include clear instructions in system or user messages about the expected response format
+5. Handle potential parsing errors when working with JSON responses
+
 ## Load and index documents
 
 For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.

e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.149
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
 ## 0.0.148
 
 ### Patch Changes

e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.148",
+  "version": "0.0.149",
   "type": "module",
   "private": true,
   "scripts": {

e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.57
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.2
+
 ## 0.0.56
 
 ### Patch Changes

e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.56",
+  "version": "0.0.57",
   "type": "module",
   "scripts": {
     "dev": "vite",

e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/next-agent-test
 
+## 0.1.149
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
 ## 0.1.148
 
 ### Patch Changes

e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.148",
+  "version": "0.1.149",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,11 @@
 # test-edge-runtime
 
+## 0.1.148
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
 ## 0.1.147
 
 ### Patch Changes

e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.147",
+  "version": "0.1.148",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/next-node-runtime
 
+## 0.1.15
+
+### Patch Changes
+
+- llamaindex@0.9.15
+- @llamaindex/huggingface@0.1.2
+- @llamaindex/readers@3.0.2
+
 ## 0.1.14
 
 ### Patch Changes

e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.14",
+  "version": "0.1.15",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/vite-import-llamaindex/CHANGELOG.md

@@ -1,5 +1,11 @@
 # vite-import-llamaindex
 
+## 0.0.15
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
 ## 0.0.14
 
 ### Patch Changes

e2e/examples/vite-import-llamaindex/package.json

@@ -1,7 +1,7 @@
 {
   "name": "vite-import-llamaindex",
   "private": true,
-  "version": "0.0.14",
+  "version": "0.0.15",
   "type": "module",
   "scripts": {
     "build": "vite build",

e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.149
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
 ## 0.0.148
 
 ### Patch Changes

e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.148",
+  "version": "0.0.149",
   "type": "module",
   "private": true,
   "scripts": {

examples/CHANGELOG.md

@@ -1,5 +1,54 @@
 # examples
 
+## 0.3.2
+
+### Patch Changes
+
+- 9c63f3f: Add support for openai responses api
+- Updated dependencies [9c63f3f]
+- Updated dependencies [c515a32]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/google@0.2.2
+  - @llamaindex/core@0.6.2
+  - @llamaindex/workflow@1.0.2
+  - llamaindex@0.9.15
+  - @llamaindex/clip@0.0.48
+  - @llamaindex/deepinfra@0.0.48
+  - @llamaindex/deepseek@0.0.8
+  - @llamaindex/fireworks@0.0.8
+  - @llamaindex/groq@0.0.63
+  - @llamaindex/huggingface@0.1.2
+  - @llamaindex/jinaai@0.0.8
+  - @llamaindex/perplexity@0.0.5
+  - @llamaindex/azure@0.1.11
+  - @llamaindex/elastic-search@0.1.2
+  - @llamaindex/milvus@0.1.11
+  - @llamaindex/qdrant@0.1.11
+  - @llamaindex/supabase@0.1.1
+  - @llamaindex/together@0.0.8
+  - @llamaindex/vllm@0.0.34
+  - @llamaindex/cloud@4.0.2
+  - @llamaindex/node-parser@2.0.2
+  - @llamaindex/anthropic@0.3.2
+  - @llamaindex/cohere@0.0.16
+  - @llamaindex/mistral@0.1.2
+  - @llamaindex/mixedbread@0.0.16
+  - @llamaindex/ollama@0.1.2
+  - @llamaindex/portkey-ai@0.0.44
+  - @llamaindex/replicate@0.0.44
+  - @llamaindex/astra@0.0.16
+  - @llamaindex/chroma@0.0.16
+  - @llamaindex/firestore@1.0.9
+  - @llamaindex/mongodb@0.0.16
+  - @llamaindex/pinecone@0.1.2
+  - @llamaindex/postgres@0.0.44
+  - @llamaindex/upstash@0.0.16
+  - @llamaindex/weaviate@0.0.16
+  - @llamaindex/vercel@0.1.2
+  - @llamaindex/voyage-ai@1.0.8
+  - @llamaindex/readers@3.0.2
+  - @llamaindex/tools@0.0.4
+
 ## 0.3.1
 
 ### Patch Changes

examples/openai/responses/agent.ts

@@ -0,0 +1,30 @@
+import { openaiResponses } from "@llamaindex/openai";
+import { wiki } from "@llamaindex/tools";
+import { agent, tool } from "llamaindex";
+import { z } from "zod";
+
+const workflow = agent({
+  tools: [
+    tool({
+      name: "weather",
+      description: "Get the weather",
+      parameters: z.object({
+        location: z.string().describe("The location to get the weather for"),
+      }),
+      execute: ({ location }) => `The weather in ${location} is sunny`,
+    }),
+    wiki(),
+  ],
+  llm: openaiResponses({
+    model: "gpt-4o-mini",
+  }),
+});
+
+async function main() {
+  const result = await workflow.run(
+    "What is the weather in New York? What's the history of New York from Wikipedia in 3 sentences?",
+  );
+  console.log(result.data);
+}
+
+void main();

examples/openai/responses/image.ts

@@ -0,0 +1,33 @@
+import { openaiResponses } from "@llamaindex/openai";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4o",
+    maxOutputTokens: 1000,
+    apiKey: process.env.MY_OPENAI_API_KEY,
+  });
+
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this image? Describe it in detail.",
+          },
+          {
+            type: "image_url",
+            image_url: {
+              url: "https://storage.googleapis.com/cloud-samples-data/vision/face/faces.jpeg",
+            },
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log("Single Image Analysis:", response.message.content);
+}
+
+main().catch(console.error);

examples/openai/responses/openai-responses.ts

@@ -0,0 +1,22 @@
+import { openaiResponses } from "@llamaindex/openai";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4o-mini",
+    temperature: 0.1,
+  });
+
+  // Basic chat example
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: "What is the capital of France?",
+      },
+    ],
+  });
+
+  console.log(response.message.content);
+}
+
+main().catch(console.error);

examples/openai/responses/stream.ts

@@ -0,0 +1,26 @@
+import { openaiResponses } from "@llamaindex/openai";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4o-mini",
+    temperature: 0.1,
+  });
+
+  const stream = await llm.chat({
+    messages: [
+      { content: "You want to talk in rhymes.", role: "system" },
+      {
+        content:
+          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
+        role: "user",
+      },
+    ],
+    stream: true,
+  });
+
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.delta);
+  }
+}
+
+main().catch(console.error);

examples/openai/responses/tool-call.ts

@@ -0,0 +1,37 @@
+import { openaiResponses } from "@llamaindex/openai";
+import { tool } from "llamaindex";
+
+import { z } from "zod";
+async function main() {
+  const weatherTool = tool({
+    name: "weather",
+    description: "Get the weather",
+    parameters: z.object({
+      location: z.string({
+        description: "The location to get the weather for",
+      }),
+    }),
+    execute: ({ location }) => {
+      return `The weather in ${location} is sunny`;
+    },
+  });
+
+  const llm = openaiResponses({
+    model: "gpt-4o-mini",
+    temperature: 0.1,
+  });
+
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: "What is the weather in New York?",
+      },
+    ],
+    tools: [weatherTool],
+  });
+
+  console.log(response.message.options);
+}
+
+main().catch(console.error);

examples/openai/responses/web-search.ts

@@ -0,0 +1,23 @@
+import { openaiResponses } from "@llamaindex/openai";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4o",
+    temperature: 0.1,
+    builtInTools: [{ type: "web_search_preview" }],
+  });
+
+  // Streaming chat example
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: "What are the latest developments in AI?",
+      },
+    ],
+  });
+
+  console.log(response.message.content);
+}
+
+main().catch(console.error);

examples/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/examples",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "private": true,
   "scripts": {
     "lint": "eslint .",
@@ -11,47 +11,47 @@
     "@azure/cosmos": "^4.1.1",
     "@azure/identity": "^4.4.1",
     "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.1",
+    "@llamaindex/anthropic": "^0.3.2",
-    "@llamaindex/astra": "^0.0.15",
+    "@llamaindex/astra": "^0.0.16",
-    "@llamaindex/azure": "^0.1.10",
+    "@llamaindex/azure": "^0.1.11",
-    "@llamaindex/chroma": "^0.0.15",
+    "@llamaindex/chroma": "^0.0.16",
-    "@llamaindex/clip": "^0.0.47",
+    "@llamaindex/clip": "^0.0.48",
-    "@llamaindex/cloud": "^4.0.1",
+    "@llamaindex/cloud": "^4.0.2",
-    "@llamaindex/cohere": "^0.0.15",
+    "@llamaindex/cohere": "^0.0.16",
-    "@llamaindex/core": "^0.6.1",
+    "@llamaindex/core": "^0.6.2",
-    "@llamaindex/deepinfra": "^0.0.47",
+    "@llamaindex/deepinfra": "^0.0.48",
     "@llamaindex/env": "^0.1.29",
-    "@llamaindex/firestore": "^1.0.8",
+    "@llamaindex/firestore": "^1.0.9",
-    "@llamaindex/google": "^0.2.1",
+    "@llamaindex/google": "^0.2.2",
-    "@llamaindex/groq": "^0.0.62",
+    "@llamaindex/groq": "^0.0.63",
-    "@llamaindex/huggingface": "^0.1.1",
+    "@llamaindex/huggingface": "^0.1.2",
-    "@llamaindex/milvus": "^0.1.10",
+    "@llamaindex/milvus": "^0.1.11",
-    "@llamaindex/mistral": "^0.1.1",
+    "@llamaindex/mistral": "^0.1.2",
-    "@llamaindex/mixedbread": "^0.0.15",
+    "@llamaindex/mixedbread": "^0.0.16",
-    "@llamaindex/mongodb": "^0.0.15",
+    "@llamaindex/mongodb": "^0.0.16",
-    "@llamaindex/elastic-search": "^0.1.1",
+    "@llamaindex/elastic-search": "^0.1.2",
-    "@llamaindex/node-parser": "^2.0.1",
+    "@llamaindex/node-parser": "^2.0.2",
-    "@llamaindex/ollama": "^0.1.1",
+    "@llamaindex/ollama": "^0.1.2",
-    "@llamaindex/openai": "^0.2.1",
+    "@llamaindex/openai": "^0.3.0",
-    "@llamaindex/pinecone": "^0.1.1",
+    "@llamaindex/pinecone": "^0.1.2",
-    "@llamaindex/portkey-ai": "^0.0.43",
+    "@llamaindex/portkey-ai": "^0.0.44",
-    "@llamaindex/postgres": "^0.0.43",
+    "@llamaindex/postgres": "^0.0.44",
-    "@llamaindex/qdrant": "^0.1.10",
+    "@llamaindex/qdrant": "^0.1.11",
-    "@llamaindex/readers": "^3.0.1",
+    "@llamaindex/readers": "^3.0.2",
-    "@llamaindex/replicate": "^0.0.43",
+    "@llamaindex/replicate": "^0.0.44",
-    "@llamaindex/upstash": "^0.0.15",
+    "@llamaindex/upstash": "^0.0.16",
-    "@llamaindex/vercel": "^0.1.1",
+    "@llamaindex/vercel": "^0.1.2",
-    "@llamaindex/vllm": "^0.0.33",
+    "@llamaindex/vllm": "^0.0.34",
-    "@llamaindex/voyage-ai": "^1.0.7",
+    "@llamaindex/voyage-ai": "^1.0.8",
-    "@llamaindex/weaviate": "^0.0.15",
+    "@llamaindex/weaviate": "^0.0.16",
-    "@llamaindex/workflow": "^1.0.1",
+    "@llamaindex/workflow": "^1.0.2",
-    "@llamaindex/deepseek": "^0.0.7",
+    "@llamaindex/deepseek": "^0.0.8",
-    "@llamaindex/fireworks": "^0.0.7",
+    "@llamaindex/fireworks": "^0.0.8",
-    "@llamaindex/together": "^0.0.7",
+    "@llamaindex/together": "^0.0.8",
-    "@llamaindex/jinaai": "^0.0.7",
+    "@llamaindex/jinaai": "^0.0.8",
-    "@llamaindex/perplexity": "^0.0.4",
+    "@llamaindex/perplexity": "^0.0.5",
-    "@llamaindex/supabase": "^0.1.0",
+    "@llamaindex/supabase": "^0.1.1",
-    "@llamaindex/tools": "^0.0.3",
+    "@llamaindex/tools": "^0.0.4",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^4.0.0",
     "@vercel/postgres": "^0.10.0",
@@ -60,7 +60,7 @@
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.9.14",
+    "llamaindex": "^0.9.15",
     "mongodb": "6.7.0",
     "postgres": "^3.4.4",
     "wikipedia": "^2.1.2",

package.json

@@ -35,7 +35,8 @@
     "prettier-plugin-tailwindcss": "^0.6.11",
     "turbo": "^2.4.4",
     "typescript": "^5.7.3",
-    "typescript-eslint": "^8.18.0"
+    "typescript-eslint": "^8.18.0",
+    "vitest": "^3.1.1"
   },
   "packageManager": "pnpm@9.12.3",
   "lint-staged": {

packages/autotool/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/autotool
 
+## 6.0.15
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
 ## 6.0.14
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.96
+
+### Patch Changes
+
+- llamaindex@0.9.15
+- @llamaindex/autotool@6.0.15
+
 ## 0.0.95
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.95"
+  "version": "0.0.96"
 }

packages/autotool/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
     "directory": "packages/autotool"
   },
-  "version": "6.0.14",
+  "version": "6.0.15",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/cloud
 
+## 4.0.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 4.0.1
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "4.0.1",
+  "version": "4.0.2",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/community
 
+## 0.0.94
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.0.93
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.93",
+  "version": "0.0.94",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/core
 
+## 0.6.2
+
+### Patch Changes
+
+- 9c63f3f: Add support for openai responses api
+
 ## 0.6.1
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./agent": {

packages/core/src/llms/type.ts

@@ -2,7 +2,6 @@ import type { Tokenizers } from "@llamaindex/env/tokenizers";
 import type { JSONSchemaType } from "ajv";
 import { z } from "zod";
 import type { JSONObject, JSONValue } from "../global";
-
 /**
  * @internal
  */
@@ -55,7 +54,12 @@ export interface LLM<
   ): Promise<CompletionResponse>;
 }
 
-export type MessageType = "user" | "assistant" | "system" | "memory";
+export type MessageType =
+  | "user"
+  | "assistant"
+  | "system"
+  | "memory"
+  | "developer";
 
 export type TextChatMessage<AdditionalMessageOptions extends object = object> =
   {
@@ -156,6 +160,7 @@ export type MessageContentTextDetail = {
 export type MessageContentImageDetail = {
   type: "image_url";
   image_url: { url: string };
+  detail?: "high" | "low" | "auto";
 };
 
 export type MessageContentDetail =

packages/experimental/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/experimental
 
+## 0.0.165
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
 ## 0.0.164
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.164",
+  "version": "0.0.165",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,17 @@
 # llamaindex
 
+## 0.9.15
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+- Updated dependencies [c515a32]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+  - @llamaindex/workflow@1.0.2
+  - @llamaindex/cloud@4.0.2
+  - @llamaindex/node-parser@2.0.2
+
 ## 0.9.14
 
 ### Patch Changes

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.9.14",
+  "version": "0.9.15",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/node-parser/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/node-parser
 
+## 2.0.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 2.0.1
 
 ### Patch Changes

packages/node-parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/node-parser",
-  "version": "2.0.1",
+  "version": "2.0.2",
   "description": "Node parser for LlamaIndex",
   "type": "module",
   "exports": {

packages/providers/anthropic/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/anthropic
 
+## 0.3.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.3.1
 
 ### Patch Changes

packages/providers/anthropic/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/anthropic",
   "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/clip/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/clip
 
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
 ## 0.0.47
 
 ### Patch Changes

packages/providers/clip/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/clip",
   "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.47",
+  "version": "0.0.48",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/cohere/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/cohere
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.0.15
 
 ### Patch Changes

packages/providers/cohere/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/cohere",
   "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepinfra/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/deepinfra
 
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
 ## 0.0.47
 
 ### Patch Changes

packages/providers/deepinfra/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepinfra",
   "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.47",
+  "version": "0.0.48",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepseek/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/deepseek
 
+## 0.0.8
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+
 ## 0.0.7
 
 ### Patch Changes

packages/providers/deepseek/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepseek",
   "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/fireworks/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/fireworks
 
+## 0.0.8
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+
 ## 0.0.7
 
 ### Patch Changes

packages/providers/fireworks/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/fireworks",
   "description": "Fireworks Adapter for LlamaIndex",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/google/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/google
 
+## 0.2.2
+
+### Patch Changes
+
+- 9c63f3f: Add support for openai responses api
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.2.1
 
 ### Patch Changes

packages/providers/google/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/google",
   "description": "Google Adapter for LlamaIndex",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/google/src/utils.ts

@@ -181,7 +181,7 @@ export const mapBaseToolToGeminiFunctionDeclaration = (
 export class GeminiHelper {
   // Gemini only has user and model roles. Put the rest in user role.
   public static readonly ROLES_TO_GEMINI: Record<
-    MessageType,
+    Exclude<MessageType, "developer">,
     GeminiMessageRole
   > = {
     user: "user",
@@ -285,7 +285,9 @@ export class GeminiHelper {
     if (message.options && "toolResult" in message.options) {
       return "function";
     }
-    return GeminiHelper.ROLES_TO_GEMINI[message.role];
+    return GeminiHelper.ROLES_TO_GEMINI[
+      message.role as Exclude<MessageType, "developer">
+    ];
   }
 
   public static chatMessageToGemini(

packages/providers/groq/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/groq
 
+## 0.0.63
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+
 ## 0.0.62
 
 ### Patch Changes

packages/providers/groq/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/groq",
   "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.62",
+  "version": "0.0.63",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/huggingface/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/huggingface
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
 ## 0.1.1
 
 ### Patch Changes

packages/providers/huggingface/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/huggingface",
   "description": "Huggingface Adapter for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/jinaai/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/jinaai
 
+## 0.0.8
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
 ## 0.0.7
 
 ### Patch Changes

packages/providers/jinaai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/jinaai",
   "description": "JinaAI Adapter for LlamaIndex",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/mistral/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/mistral
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.1.1
 
 ### Patch Changes

packages/providers/mistral/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/mistral",
   "description": "Mistral Adapter for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/mixedbread/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/mixedbread
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.0.15
 
 ### Patch Changes

packages/providers/mixedbread/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/mixedbread",
   "description": "Mixedbread Adapter for LlamaIndex",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/ollama/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/ollama
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.1.1
 
 ### Patch Changes

packages/providers/ollama/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/ollama",
   "description": "Ollama Adapter for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/openai/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @llamaindex/openai
 
+## 0.3.0
+
+### Minor Changes
+
+- 9c63f3f: Add support for openai responses api
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.2.1
 
 ### Patch Changes

packages/providers/openai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/openai",
   "description": "OpenAI Adapter for LlamaIndex",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -27,7 +27,8 @@
   },
   "scripts": {
     "build": "bunchee",
-    "dev": "bunchee --watch"
+    "dev": "bunchee --watch",
+    "test": "vitest"
   },
   "devDependencies": {
     "bunchee": "6.4.0"
@@ -35,7 +36,7 @@
   "dependencies": {
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*",
-    "openai": "^4.86.0",
+    "openai": "^4.90.0",
     "zod": "^3.24.2"
   }
 }

packages/providers/openai/src/agent.ts

@@ -5,7 +5,8 @@ import {
 } from "@llamaindex/core/agent";
 import { Settings } from "@llamaindex/core/global";
 import type { ToolCallLLMMessageOptions } from "@llamaindex/core/llms";
-import { OpenAI, type OpenAIAdditionalChatOptions } from "./llm";
+import { OpenAI } from "./llm";
+import type { OpenAIAdditionalChatOptions } from "./utils";
 
 // This is likely not necessary anymore but leaving it here just in case it's in use elsewhere
 

packages/providers/openai/src/index.ts

@@ -1,3 +1,4 @@
 export * from "./agent";
 export * from "./embedding";
 export * from "./llm";
+export * from "./responses";

packages/providers/openai/src/llm.ts

@@ -4,7 +4,6 @@ import {
   type ChatMessage,
   type ChatResponse,
   type ChatResponseChunk,
-  type LLM,
   type LLMChatParamsNonStreaming,
   type LLMChatParamsStreaming,
   type LLMMetadata,
@@ -18,7 +17,6 @@ import { getEnv } from "@llamaindex/env";
 import { Tokenizers } from "@llamaindex/env/tokenizers";
 import type {
   AzureClientOptions,
-  AzureOpenAI as AzureOpenAILLM,
   ClientOptions as OpenAIClientOptions,
   OpenAI as OpenAILLM,
 } from "openai";
@@ -45,153 +43,15 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure.js";
-
+import {
-export const GPT4_MODELS = {
+  ALL_AVAILABLE_OPENAI_MODELS,
-  "chatgpt-4o-latest": {
+  isFunctionCallingModel,
-    contextWindow: 128000,
+  isReasoningModel,
-  },
+  isTemperatureSupported,
-  "gpt-4.5-preview": { contextWindow: 128000 },
+  type LLMInstance,
-  "gpt-4.5-preview-2025-02-27": { contextWindow: 128000 },
+  type OpenAIAdditionalChatOptions,
-  "gpt-4": { contextWindow: 8192 },
+  type OpenAIAdditionalMetadata,
-  "gpt-4-32k": { contextWindow: 32768 },
+} from "./utils.js";
-  "gpt-4-32k-0613": { contextWindow: 32768 },
-  "gpt-4-turbo": { contextWindow: 128000 },
-  "gpt-4-turbo-preview": { contextWindow: 128000 },
-  "gpt-4-1106-preview": { contextWindow: 128000 },
-  "gpt-4-0125-preview": { contextWindow: 128000 },
-  "gpt-4-vision-preview": { contextWindow: 128000 },
-  "gpt-4o": { contextWindow: 128000 },
-  "gpt-4o-2024-05-13": { contextWindow: 128000 },
-  "gpt-4o-mini": { contextWindow: 128000 },
-  "gpt-4o-mini-2024-07-18": { contextWindow: 128000 },
-  "gpt-4o-2024-08-06": { contextWindow: 128000 },
-  "gpt-4o-2024-09-14": { contextWindow: 128000 },
-  "gpt-4o-2024-10-14": { contextWindow: 128000 },
-  "gpt-4-0613": { contextWindow: 128000 },
-  "gpt-4-turbo-2024-04-09": { contextWindow: 128000 },
-  "gpt-4-0314": { contextWindow: 128000 },
-  "gpt-4-32k-0314": { contextWindow: 32768 },
-  "gpt-4o-realtime-preview": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-realtime-preview-2024-10-01": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-audio-preview": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-audio-preview-2024-10-01": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-2024-11-20": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-audio-preview-2024-12-17": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-mini-audio-preview": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-mini-audio-preview-2024-12-17": {
-    contextWindow: 128000,
-  },
-};
-
-// NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
-export const GPT35_MODELS = {
-  "gpt-3.5-turbo": { contextWindow: 16385 },
-  "gpt-3.5-turbo-0613": { contextWindow: 4096 },
-  "gpt-3.5-turbo-16k": { contextWindow: 16385 },
-  "gpt-3.5-turbo-16k-0613": { contextWindow: 16385 },
-  "gpt-3.5-turbo-1106": { contextWindow: 16385 },
-  "gpt-3.5-turbo-0125": { contextWindow: 16385 },
-  "gpt-3.5-turbo-0301": { contextWindow: 16385 },
-};
-
-export const O1_MODELS = {
-  "o1-preview": {
-    contextWindow: 128000,
-  },
-  "o1-preview-2024-09-12": {
-    contextWindow: 128000,
-  },
-  "o1-mini": {
-    contextWindow: 128000,
-  },
-  "o1-mini-2024-09-12": {
-    contextWindow: 128000,
-  },
-  o1: {
-    contextWindow: 128000,
-  },
-  "o1-2024-12-17": {
-    contextWindow: 128000,
-  },
-};
-
-export const O3_MODELS = {
-  "o3-mini": {
-    contextWindow: 200000,
-  },
-  "o3-mini-2025-01-31": {
-    contextWindow: 200000,
-  },
-};
-
-/**
- * We currently support GPT-3.5 and GPT-4 models
- */
-export const ALL_AVAILABLE_OPENAI_MODELS = {
-  ...GPT4_MODELS,
-  ...GPT35_MODELS,
-  ...O1_MODELS,
-  ...O3_MODELS,
-} satisfies Record<ChatModel, { contextWindow: number }>;
-
-function isFunctionCallingModel(llm: LLM): llm is OpenAI {
-  let model: string;
-  if (llm instanceof OpenAI) {
-    model = llm.model;
-  } else if ("model" in llm && typeof llm.model === "string") {
-    model = llm.model;
-  } else {
-    return false;
-  }
-  const isChatModel = Object.keys(ALL_AVAILABLE_OPENAI_MODELS).includes(model);
-  const isOld = model.includes("0314") || model.includes("0301");
-  const isO1 = model.startsWith("o1");
-  return isChatModel && !isOld && !isO1;
-}
-
-function isReasoningModel(model: ChatModel | string): boolean {
-  const isO1 = model.startsWith("o1");
-  const isO3 = model.startsWith("o3");
-  return isO1 || isO3;
-}
-
-function isTemperatureSupported(model: ChatModel | string): boolean {
-  return !model.startsWith("o3");
-}
-
-export type OpenAIAdditionalMetadata = object;
-
-export type OpenAIAdditionalChatOptions = Omit<
-  Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
-  | "max_tokens"
-  | "messages"
-  | "model"
-  | "temperature"
-  | "reasoning_effort"
-  | "top_p"
-  | "stream"
-  | "tools"
-  | "toolChoice"
->;
-
-type LLMInstance = Pick<
-  AzureOpenAILLM | OpenAILLM,
-  "chat" | "apiKey" | "baseURL"
->;
 
 export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
   model:
@@ -230,6 +90,7 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
     },
   ) {
     super();
+
     this.model = init?.model ?? "gpt-4o";
     this.temperature = init?.temperature ?? 0.1;
     this.reasoningEffort = isReasoningModel(this.model)

packages/providers/openai/src/responses.ts

@@ -0,0 +1,774 @@
+import {
+  ToolCallLLM,
+  type BaseTool,
+  type ChatMessage,
+  type ChatResponse,
+  type ChatResponseChunk,
+  type LLMChatParamsNonStreaming,
+  type LLMChatParamsStreaming,
+  type LLMMetadata,
+  type MessageContent,
+  type MessageType,
+  type PartialToolCall,
+  type ToolCallLLMMessageOptions,
+  type ToolCallOptions,
+  type ToolResultOptions,
+} from "@llamaindex/core/llms";
+import type { StoredValue } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
+import { getEnv } from "@llamaindex/env";
+import {
+  OpenAI as OpenAILLM,
+  type AzureClientOptions,
+  type ClientOptions as OpenAIClientOptions,
+} from "openai";
+
+import { wrapEventCaller } from "@llamaindex/core/decorator";
+import { Tokenizers } from "@llamaindex/env/tokenizers";
+import {
+  AzureOpenAIWithUserAgent,
+  getAzureConfigFromEnv,
+  getAzureModel,
+  shouldUseAzure,
+} from "./azure";
+import {
+  ALL_AVAILABLE_OPENAI_MODELS,
+  isFunctionCallingModel,
+  isReasoningModel,
+  isTemperatureSupported,
+  type LLMInstance,
+  type OpenAIAdditionalMetadata,
+  type OpenAIResponsesChatOptions,
+  type OpenAIResponsesRole,
+  type ResponseMessageContent,
+  type ResponsesAdditionalOptions,
+  type StreamState,
+} from "./utils";
+
+export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxOutputTokens?: number | undefined;
+  additionalChatOptions?: OpenAIResponsesChatOptions | undefined;
+  reasoningEffort?: "low" | "medium" | "high" | undefined;
+  apiKey?: string | undefined;
+  baseURL?: string | undefined;
+  maxRetries: number;
+  timeout?: number;
+  additionalSessionOptions?:
+    | undefined
+    | Omit<Partial<OpenAIClientOptions>, "apiKey" | "maxRetries" | "timeout">;
+  lazySession: () => Promise<LLMInstance>;
+  #session: Promise<LLMInstance> | null = null;
+
+  trackPreviousResponses: boolean;
+  store: boolean;
+  user: string;
+  callMetadata: StoredValue;
+  builtInTools: OpenAILLM.Responses.Tool[] | null;
+  strict: boolean;
+  include: OpenAILLM.Responses.ResponseIncludable[] | null;
+  instructions: string;
+  previousResponseId: string | null;
+  truncation: "auto" | "disabled" | null;
+
+  constructor(
+    init?: Omit<Partial<OpenAIResponses>, "session"> & {
+      session?: LLMInstance | undefined;
+      azure?: AzureClientOptions;
+    },
+  ) {
+    super();
+    this.model = init?.model ?? "gpt-4o";
+    this.temperature = init?.temperature ?? 0.1;
+
+    this.reasoningEffort = isReasoningModel(this.model)
+      ? init?.reasoningEffort
+      : undefined;
+
+    this.topP = init?.topP ?? 1;
+    this.maxOutputTokens = init?.maxOutputTokens ?? undefined;
+    this.maxRetries = init?.maxRetries ?? 10;
+
+    this.timeout = init?.timeout ?? 60 * 1000;
+
+    this.apiKey =
+      init?.session?.apiKey ?? init?.apiKey ?? getEnv("OPENAI_API_KEY");
+    this.baseURL =
+      init?.session?.baseURL ?? init?.baseURL ?? getEnv("OPENAI_BASE_URL");
+
+    this.additionalSessionOptions = init?.additionalSessionOptions;
+    this.additionalChatOptions = init?.additionalChatOptions;
+
+    this.trackPreviousResponses = init?.trackPreviousResponses ?? false;
+    this.builtInTools = init?.builtInTools ?? null;
+    this.store = init?.store ?? false;
+    this.user = init?.user ?? "";
+    this.callMetadata = init?.callMetadata ?? {};
+    this.strict = init?.strict ?? false;
+    this.include = init?.include ?? null;
+    this.instructions = init?.instructions ?? "";
+    this.previousResponseId = init?.previousResponseId ?? null;
+    this.truncation = init?.truncation ?? null;
+
+    if (init?.azure || shouldUseAzure()) {
+      const azureConfig = {
+        ...getAzureConfigFromEnv({
+          model: getAzureModel(this.model),
+        }),
+        ...init?.azure,
+      };
+
+      this.lazySession = async () =>
+        init?.session ??
+        import("openai").then(({ AzureOpenAI }) => {
+          AzureOpenAI = AzureOpenAIWithUserAgent(AzureOpenAI);
+
+          return new AzureOpenAI({
+            maxRetries: this.maxRetries,
+            timeout: this.timeout!,
+            ...this.additionalSessionOptions,
+            ...azureConfig,
+          });
+        });
+    } else {
+      this.lazySession = async () =>
+        init?.session ??
+        import("openai").then(({ OpenAI }) => {
+          return new OpenAI({
+            apiKey: this.apiKey,
+            baseURL: this.baseURL,
+            maxRetries: this.maxRetries,
+            timeout: this.timeout!,
+            ...this.additionalSessionOptions,
+          });
+        });
+    }
+  }
+
+  get session() {
+    if (!this.#session) {
+      this.#session = this.lazySession();
+    }
+    return this.#session;
+  }
+
+  get supportToolCall() {
+    return isFunctionCallingModel(this);
+  }
+
+  get metadata(): LLMMetadata & OpenAIAdditionalMetadata {
+    const contextWindow =
+      ALL_AVAILABLE_OPENAI_MODELS[
+        this.model as keyof typeof ALL_AVAILABLE_OPENAI_MODELS
+      ]?.contextWindow ?? 1024;
+
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxOutputTokens,
+      contextWindow,
+      tokenizer: Tokenizers.CL100K_BASE,
+      structuredOutput: true,
+    };
+  }
+
+  private createInitialMessage(): ChatMessage<ToolCallLLMMessageOptions> {
+    return {
+      role: "assistant",
+      content: "",
+    };
+  }
+
+  private createInitialOptions(): ResponsesAdditionalOptions {
+    return {
+      built_in_tool_calls: [],
+    };
+  }
+
+  private isBuiltInToolCall(
+    item: OpenAILLM.Responses.ResponseOutputItem,
+  ): item is
+    | OpenAILLM.Responses.ResponseFileSearchToolCall
+    | OpenAILLM.Responses.ResponseComputerToolCall
+    | OpenAILLM.Responses.ResponseFunctionWebSearch {
+    return ["file_search_call", "computer_call", "web_search_call"].includes(
+      item.type,
+    );
+  }
+
+  private isReasoning(item: OpenAILLM.Responses.ResponseOutputItem) {
+    return item.type === "reasoning";
+  }
+
+  private isMessageBlock(item: OpenAILLM.Responses.ResponseOutputItem) {
+    return item.type === "message";
+  }
+
+  private isFunctionCall(item: OpenAILLM.Responses.ResponseOutputItem) {
+    return item.type === "function_call";
+  }
+
+  private isResponseCreatedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseCreatedEvent {
+    return event.type === "response.created";
+  }
+
+  private isResponseOutputItemAddedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseOutputItemAddedEvent {
+    return event.type === "response.output_item.added";
+  }
+
+  private isToolCallEvent(
+    event: OpenAILLM.Responses.ResponseOutputItemAddedEvent,
+  ): event is OpenAILLM.Responses.ResponseOutputItemAddedEvent & {
+    item: OpenAILLM.Responses.ResponseFunctionToolCall;
+  } {
+    return event.item.type === "function_call";
+  }
+
+  private isResponseTextDeltaEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseTextDeltaEvent {
+    return event.type === "response.output_text.delta";
+  }
+
+  private isResponseFunctionCallArgumentsDeltaEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseFunctionCallArgumentsDeltaEvent {
+    return event.type === "response.function_call_arguments.delta";
+  }
+
+  private isResponseFunctionCallDoneEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseFunctionCallArgumentsDoneEvent {
+    return event.type === "response.function_call_arguments.done";
+  }
+
+  private isResponseOutputTextAnnotationAddedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseTextAnnotationDeltaEvent {
+    return event.type === "response.output_text.annotation.added";
+  }
+
+  private isResponseFileSearchCallCompletedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseFileSearchCallCompletedEvent {
+    return event.type === "response.file_search_call.completed";
+  }
+
+  private isResponseWebSearchCallCompletedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseWebSearchCallCompletedEvent {
+    return event.type === "response.web_search_call.completed";
+  }
+
+  private isResponseCompletedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseCompletedEvent {
+    return event.type === "response.completed";
+  }
+
+  private isTextPresent(
+    part:
+      | OpenAILLM.Responses.ResponseOutputText
+      | OpenAILLM.Responses.ResponseOutputRefusal,
+  ) {
+    return "text" in part;
+  }
+
+  private isRefusalPresent(
+    part:
+      | OpenAILLM.Responses.ResponseOutputText
+      | OpenAILLM.Responses.ResponseOutputRefusal,
+  ) {
+    return "refusal" in part;
+  }
+
+  private isAnnotationPresent(
+    part:
+      | OpenAILLM.Responses.ResponseOutputText
+      | OpenAILLM.Responses.ResponseOutputRefusal,
+  ) {
+    return "annotations" in part;
+  }
+
+  private handleResponseOutputMessage(
+    item: OpenAILLM.Responses.ResponseOutputMessage,
+    options: ResponsesAdditionalOptions,
+  ): string {
+    let outputContent = "";
+    for (const part of item.content) {
+      if (this.isTextPresent(part)) {
+        outputContent += part.text;
+      }
+      if (this.isAnnotationPresent(part)) {
+        options.annotations = part.annotations;
+      }
+      if (this.isRefusalPresent(part)) {
+        options.refusal = part.refusal;
+      }
+    }
+    return outputContent;
+  }
+
+  private extractToolCalls(
+    response: OpenAILLM.Responses.ResponseOutputItem[],
+  ): PartialToolCall[] {
+    return response
+      .filter((item): item is OpenAILLM.Responses.ResponseFunctionToolCall =>
+        this.isFunctionCall(item),
+      )
+      .map((item) => {
+        return {
+          name: item.name,
+          id: item.call_id,
+          input: item.arguments,
+        };
+      });
+  }
+
+  private parseResponseOutput(
+    response: OpenAILLM.Responses.ResponseOutputItem[],
+  ): ChatMessage<ToolCallLLMMessageOptions> {
+    const message = this.createInitialMessage();
+    const options = this.createInitialOptions();
+    const toolCall = this.extractToolCalls(response);
+
+    for (const item of response) {
+      if (this.isMessageBlock(item)) {
+        const outputContent = this.handleResponseOutputMessage(item, options);
+        message.content = outputContent;
+      } else if (this.isBuiltInToolCall(item)) {
+        options.built_in_tool_calls.push(item);
+      } else if (this.isReasoning(item)) {
+        options.reasoning = item;
+      }
+    }
+    message.options = {
+      ...options,
+      toolCall: toolCall,
+    };
+
+    return message;
+  }
+
+  private processStreamEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+    streamState: StreamState,
+  ) {
+    switch (true) {
+      case this.isResponseCreatedEvent(event):
+        this.handleResponseCreatedEvent(event, streamState);
+        break;
+      case this.isResponseOutputItemAddedEvent(event):
+        this.handleOutputItemAddedEvent(event, streamState);
+        break;
+      case this.isResponseTextDeltaEvent(event):
+        this.handleTextDeltaEvent(event, streamState);
+        break;
+      case this.isResponseFunctionCallArgumentsDeltaEvent(event):
+        this.handleFunctionCallArgumentsDeltaEvent(event, streamState);
+        break;
+      case this.isResponseFunctionCallDoneEvent(event):
+        this.handleFunctionCallArgumentsDoneEvent(event, streamState);
+        break;
+      case this.isResponseOutputTextAnnotationAddedEvent(event):
+        this.handleOutputTextAnnotationAddedEvent(event, streamState);
+        break;
+      case this.isResponseFileSearchCallCompletedEvent(event):
+      case this.isResponseWebSearchCallCompletedEvent(event):
+        this.handleBuiltInToolCallCompletedEvent(event, streamState);
+        break;
+      case this.isResponseCompletedEvent(event):
+        this.handleCompletedEvent(event, streamState);
+        break;
+    }
+  }
+
+  private handleResponseCreatedEvent(
+    event: OpenAILLM.Responses.ResponseCreatedEvent,
+    streamState: StreamState,
+  ) {
+    if (this.trackPreviousResponses) {
+      streamState.previousResponseId = event.response.id;
+    }
+  }
+
+  private handleOutputItemAddedEvent(
+    event: OpenAILLM.Responses.ResponseOutputItemAddedEvent,
+    streamState: StreamState,
+  ) {
+    if (this.isToolCallEvent(event)) {
+      streamState.currentToolCall = {
+        name: event.item.name,
+        id: event.item.call_id,
+        input: event.item.arguments,
+      };
+    }
+  }
+
+  private handleTextDeltaEvent(
+    event: OpenAILLM.Responses.ResponseTextDeltaEvent,
+    streamState: StreamState,
+  ) {
+    streamState.delta = event.delta;
+  }
+
+  private handleFunctionCallArgumentsDeltaEvent(
+    event: OpenAILLM.Responses.ResponseFunctionCallArgumentsDeltaEvent,
+    streamState: StreamState,
+  ) {
+    if (streamState.currentToolCall) {
+      streamState.currentToolCall!.input += event.delta;
+    }
+  }
+
+  private handleFunctionCallArgumentsDoneEvent(
+    event: OpenAILLM.Responses.ResponseFunctionCallArgumentsDoneEvent,
+    streamState: StreamState,
+  ) {
+    if (streamState.currentToolCall) {
+      streamState.currentToolCall.input = event.arguments;
+    }
+
+    streamState.shouldEmitToolCall = {
+      ...streamState.currentToolCall!,
+      input: JSON.parse(streamState.currentToolCall!.input),
+    };
+  }
+
+  private handleOutputTextAnnotationAddedEvent(
+    event: OpenAILLM.Responses.ResponseTextAnnotationDeltaEvent,
+    streamState: StreamState,
+  ) {
+    if (!streamState.options.annotations) {
+      streamState.options.annotations = [];
+    }
+    streamState.options.annotations.push(event.annotation);
+  }
+
+  private handleBuiltInToolCallCompletedEvent(
+    event:
+      | OpenAILLM.Responses.ResponseFileSearchCallCompletedEvent
+      | OpenAILLM.Responses.ResponseWebSearchCallCompletedEvent,
+    streamState: StreamState,
+  ) {
+    streamState.options.built_in_tool_calls.push(event);
+  }
+
+  private handleCompletedEvent(
+    event: OpenAILLM.Responses.ResponseCompletedEvent,
+    streamState: StreamState,
+  ) {
+    if (event.response.usage) {
+      streamState.options.usage = event.response.usage;
+    }
+  }
+
+  private createBaseRequestParams(
+    messages: ChatMessage<ToolCallLLMMessageOptions>[],
+    tools: BaseTool[] | undefined,
+    additionalChatOptions: OpenAIResponsesChatOptions | undefined,
+  ) {
+    const baseRequestParams = <OpenAILLM.Responses.ResponseCreateParams>{
+      model: this.model,
+      include: this.include,
+      input: this.toOpenAIResponseMessages(messages),
+      tools: this.builtInTools ? [...this.builtInTools] : [],
+      instructions: this.instructions,
+      max_output_tokens: this.maxOutputTokens,
+      previous_response_id: this.previousResponseId,
+      store: this.store,
+      metadata: this.callMetadata,
+      top_p: this.topP,
+      truncation: this.truncation,
+      user: this.user,
+      ...Object.assign({}, this.additionalChatOptions, additionalChatOptions),
+    };
+
+    if (tools?.length) {
+      if (!baseRequestParams.tools) {
+        baseRequestParams.tools = [];
+      }
+      baseRequestParams.tools.push(
+        ...tools.map(this.toResponsesTool.bind(this)),
+      );
+    }
+
+    return baseRequestParams;
+  }
+
+  chat(
+    params: LLMChatParamsStreaming<
+      OpenAIResponsesChatOptions,
+      ToolCallLLMMessageOptions
+    >,
+  ): Promise<AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>>;
+  chat(
+    params: LLMChatParamsNonStreaming<
+      OpenAIResponsesChatOptions,
+      ToolCallLLMMessageOptions
+    >,
+  ): Promise<ChatResponse<ToolCallLLMMessageOptions>>;
+  async chat(
+    params:
+      | LLMChatParamsNonStreaming<
+          OpenAIResponsesChatOptions,
+          ToolCallLLMMessageOptions
+        >
+      | LLMChatParamsStreaming<
+          OpenAIResponsesChatOptions,
+          ToolCallLLMMessageOptions
+        >,
+  ): Promise<
+    | ChatResponse<ToolCallLLMMessageOptions>
+    | AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>
+  > {
+    const { messages, stream, tools, responseFormat, additionalChatOptions } =
+      params;
+    const baseRequestParams = this.createBaseRequestParams(
+      messages,
+      tools,
+      additionalChatOptions,
+    );
+
+    if (
+      Array.isArray(baseRequestParams.tools) &&
+      baseRequestParams.tools.length === 0
+    ) {
+      // remove empty tools array to avoid OpenAI error
+      delete baseRequestParams.tools;
+    }
+
+    if (!isTemperatureSupported(baseRequestParams.model))
+      delete baseRequestParams.temperature;
+
+    if (stream) {
+      return this.streamChat(baseRequestParams);
+    }
+
+    const response = await (
+      await this.session
+    ).responses.create({
+      ...baseRequestParams,
+      stream: false,
+    });
+
+    const message = this.parseResponseOutput(response.output);
+    return {
+      raw: response,
+      message,
+    };
+  }
+
+  private initalizeStreamState(): StreamState {
+    return {
+      delta: "",
+      currentToolCall: null,
+      shouldEmitToolCall: null,
+      options: this.createInitialOptions(),
+      previousResponseId: this.previousResponseId,
+    };
+  }
+
+  private createResponseChunk(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+    state: StreamState,
+  ): ChatResponseChunk<ToolCallLLMMessageOptions> {
+    return {
+      raw: event,
+      delta: state.delta,
+      options: state.shouldEmitToolCall
+        ? { toolCall: [state.shouldEmitToolCall] }
+        : state.currentToolCall
+          ? { toolCall: [state.currentToolCall] }
+          : {},
+    };
+  }
+
+  @wrapEventCaller
+  protected async *streamChat(
+    baseRequestParams: OpenAILLM.Responses.ResponseCreateParams,
+  ): AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>> {
+    const streamState = this.initalizeStreamState();
+
+    const stream = await (
+      await this.session
+    ).responses.create({
+      ...baseRequestParams,
+      stream: true,
+    });
+
+    for await (const event of stream) {
+      this.processStreamEvent(event, streamState);
+      this.handlePreviousResponseId(streamState);
+      yield this.createResponseChunk(event, streamState);
+    }
+  }
+
+  toOpenAIResponsesRole(messageType: MessageType): OpenAIResponsesRole {
+    switch (messageType) {
+      case "user":
+        return "user";
+      case "assistant":
+        return "assistant";
+      case "system":
+        return "system";
+      case "developer":
+        return "developer";
+      default:
+        return "user";
+    }
+  }
+
+  private isToolResultPresent(
+    options: ToolCallLLMMessageOptions,
+  ): options is ToolResultOptions {
+    return "toolResult" in options;
+  }
+
+  private isToolCallPresent(
+    options: ToolCallLLMMessageOptions,
+  ): options is ToolCallOptions {
+    return "toolCall" in options;
+  }
+
+  private isUserMessage(message: ChatMessage<ToolCallLLMMessageOptions>) {
+    return message.role === "user";
+  }
+
+  private handlePreviousResponseId(streamState: StreamState) {
+    if (this.trackPreviousResponses) {
+      if (streamState.previousResponseId != this.previousResponseId) {
+        this.previousResponseId = streamState.previousResponseId;
+      }
+    }
+  }
+
+  private convertToOpenAIToolCallResult(
+    options: ToolResultOptions,
+    content: MessageContent,
+  ) {
+    return {
+      type: "function_call_output",
+      call_id: options.toolResult.id,
+      output: extractText(content),
+    } satisfies OpenAILLM.Responses.ResponseInputItem.FunctionCallOutput;
+  }
+
+  private convertToOpenAIToolCalls(options: ToolCallOptions) {
+    return options.toolCall.map((toolCall) => {
+      return {
+        type: "function_call",
+        call_id: toolCall.id,
+        name: toolCall.name,
+        arguments:
+          typeof toolCall.input === "string"
+            ? toolCall.input
+            : JSON.stringify(toolCall.input),
+      };
+    }) satisfies OpenAILLM.Responses.ResponseFunctionToolCall[];
+  }
+
+  private processMessageContent(
+    content: MessageContent,
+  ): ResponseMessageContent {
+    if (!Array.isArray(content)) {
+      return content;
+    }
+
+    return content.map((item) => {
+      if (item.type === "text") {
+        return {
+          type: "input_text",
+          text: item.text,
+        };
+      }
+      if (item.type === "image_url") {
+        return {
+          type: "input_image",
+          image_url: item.image_url.url,
+          detail: item.detail || "auto",
+        };
+      }
+      throw new Error("Unsupported content type");
+    });
+  }
+
+  private convertToOpenAIUserMessage(
+    message: ChatMessage<ToolCallLLMMessageOptions>,
+  ) {
+    const messageContent = this.processMessageContent(message.content);
+    return {
+      role: "user",
+      content: messageContent,
+    } satisfies OpenAILLM.Responses.EasyInputMessage;
+  }
+
+  private defaultOpenAIResponseMessage(
+    message: ChatMessage<ToolCallLLMMessageOptions>,
+  ) {
+    const response: OpenAILLM.Responses.ResponseInputItem = {
+      role: this.toOpenAIResponsesRole(message.role),
+      content: extractText(message.content),
+    };
+    return response;
+  }
+
+  toOpenAIResponseMessage(
+    message: ChatMessage<ToolCallLLMMessageOptions>,
+  ):
+    | OpenAILLM.Responses.ResponseInputItem
+    | OpenAILLM.Responses.ResponseInputItem[] {
+    const options = message.options ?? {};
+
+    if (this.isToolResultPresent(options)) {
+      return this.convertToOpenAIToolCallResult(options, message.content);
+    } else if (this.isToolCallPresent(options)) {
+      return this.convertToOpenAIToolCalls(options);
+    } else if (this.isUserMessage(message)) {
+      return this.convertToOpenAIUserMessage(message);
+    }
+
+    return this.defaultOpenAIResponseMessage(message);
+  }
+
+  toOpenAIResponseMessages(
+    messages: ChatMessage<ToolCallLLMMessageOptions>[],
+  ): OpenAILLM.Responses.ResponseInput {
+    const finalMessages: OpenAILLM.Responses.ResponseInputItem[] = [];
+    for (const message of messages) {
+      const processedMessage = this.toOpenAIResponseMessage(message);
+      if (Array.isArray(processedMessage)) {
+        finalMessages.push(...processedMessage);
+      } else {
+        finalMessages.push(processedMessage);
+      }
+    }
+    return finalMessages;
+  }
+
+  toResponsesTool(tool: BaseTool): OpenAILLM.Responses.Tool {
+    return {
+      type: "function",
+      name: tool.metadata.name,
+      description: tool.metadata.description,
+      parameters: tool.metadata.parameters ?? {},
+      strict: this.strict,
+    };
+  }
+}
+
+/**
+ * Convenience function to create a new OpenAI instance.
+ * @param init - Optional initialization parameters for the OpenAI instance.
+ * @returns A new OpenAI instance.
+ */
+export const openaiResponses = (
+  init?: ConstructorParameters<typeof OpenAIResponses>[0],
+) => new OpenAIResponses(init);

packages/providers/openai/src/utils.ts

@@ -0,0 +1,213 @@
+import type { LLM, PartialToolCall } from "@llamaindex/core/llms";
+import { AzureOpenAI as AzureOpenAILLM, OpenAI as OpenAILLM } from "openai";
+import type { ChatModel } from "openai/resources.mjs";
+import { OpenAI } from "./llm";
+
+export const GPT4_MODELS = {
+  "chatgpt-4o-latest": {
+    contextWindow: 128000,
+  },
+  "gpt-4.5-preview": { contextWindow: 128000 },
+  "gpt-4.5-preview-2025-02-27": { contextWindow: 128000 },
+  "gpt-4": { contextWindow: 8192 },
+  "gpt-4-32k": { contextWindow: 32768 },
+  "gpt-4-32k-0613": { contextWindow: 32768 },
+  "gpt-4-turbo": { contextWindow: 128000 },
+  "gpt-4-turbo-preview": { contextWindow: 128000 },
+  "gpt-4-1106-preview": { contextWindow: 128000 },
+  "gpt-4-0125-preview": { contextWindow: 128000 },
+  "gpt-4-vision-preview": { contextWindow: 128000 },
+  "gpt-4o": { contextWindow: 128000 },
+  "gpt-4o-2024-05-13": { contextWindow: 128000 },
+  "gpt-4o-mini": { contextWindow: 128000 },
+  "gpt-4o-mini-2024-07-18": { contextWindow: 128000 },
+  "gpt-4o-2024-08-06": { contextWindow: 128000 },
+  "gpt-4o-2024-09-14": { contextWindow: 128000 },
+  "gpt-4o-2024-10-14": { contextWindow: 128000 },
+  "gpt-4-0613": { contextWindow: 128000 },
+  "gpt-4-turbo-2024-04-09": { contextWindow: 128000 },
+  "gpt-4-0314": { contextWindow: 128000 },
+  "gpt-4-32k-0314": { contextWindow: 32768 },
+  "gpt-4o-realtime-preview": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-realtime-preview-2024-10-01": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-audio-preview": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-audio-preview-2024-10-01": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-2024-11-20": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-audio-preview-2024-12-17": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-mini-audio-preview": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-mini-audio-preview-2024-12-17": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-search-preview": { contextWindow: 128000 },
+  "gpt-4o-mini-search-preview": { contextWindow: 128000 },
+  "gpt-4o-search-preview-2025-03-11": { contextWindow: 128000 },
+  "gpt-4o-mini-search-preview-2025-03-11": { contextWindow: 128000 },
+};
+
+// NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
+export const GPT35_MODELS = {
+  "gpt-3.5-turbo": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0613": { contextWindow: 4096 },
+  "gpt-3.5-turbo-16k": { contextWindow: 16385 },
+  "gpt-3.5-turbo-16k-0613": { contextWindow: 16385 },
+  "gpt-3.5-turbo-1106": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0125": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0301": { contextWindow: 16385 },
+};
+
+export const O1_MODELS = {
+  "o1-preview": {
+    contextWindow: 128000,
+  },
+  "o1-preview-2024-09-12": {
+    contextWindow: 128000,
+  },
+  "o1-mini": {
+    contextWindow: 128000,
+  },
+  "o1-mini-2024-09-12": {
+    contextWindow: 128000,
+  },
+  o1: {
+    contextWindow: 128000,
+  },
+  "o1-2024-12-17": {
+    contextWindow: 128000,
+  },
+};
+
+export const O3_MODELS = {
+  "o3-mini": {
+    contextWindow: 200000,
+  },
+  "o3-mini-2025-01-31": {
+    contextWindow: 200000,
+  },
+};
+
+/**
+ * We currently support GPT-3.5 and GPT-4 models
+ */
+export const ALL_AVAILABLE_OPENAI_MODELS = {
+  ...GPT4_MODELS,
+  ...GPT35_MODELS,
+  ...O1_MODELS,
+  ...O3_MODELS,
+} satisfies Record<ChatModel, { contextWindow: number }>;
+
+export function isFunctionCallingModel(llm: LLM): llm is OpenAI {
+  let model: string;
+  if (llm instanceof OpenAI) {
+    model = llm.model;
+  } else if ("model" in llm && typeof llm.model === "string") {
+    model = llm.model;
+  } else {
+    return false;
+  }
+  const isChatModel = Object.keys(ALL_AVAILABLE_OPENAI_MODELS).includes(model);
+  const isOld = model.includes("0314") || model.includes("0301");
+  const isO1 = model.startsWith("o1");
+  return isChatModel && !isOld && !isO1;
+}
+
+export function isReasoningModel(model: ChatModel | string): boolean {
+  const isO1 = model.startsWith("o1");
+  const isO3 = model.startsWith("o3");
+  return isO1 || isO3;
+}
+
+export function isTemperatureSupported(model: ChatModel | string): boolean {
+  return !model.startsWith("o3");
+}
+
+export type OpenAIAdditionalMetadata = object;
+
+export type OpenAIAdditionalChatOptions = Omit<
+  Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
+  | "max_tokens"
+  | "messages"
+  | "model"
+  | "temperature"
+  | "reasoning_effort"
+  | "top_p"
+  | "stream"
+  | "tools"
+  | "toolChoice"
+>;
+
+export type LLMInstance = Pick<
+  AzureOpenAILLM | OpenAILLM,
+  "chat" | "apiKey" | "baseURL" | "responses"
+>;
+
+export type OpenAIResponsesChatOptions = Omit<
+  Partial<OpenAILLM.Responses.ResponseCreateParams>,
+  | "model"
+  | "input"
+  | "stream"
+  | "tools"
+  | "toolChoice"
+  | "temperature"
+  | "reasoning_effort"
+  | "top_p"
+  | "max_output_tokens"
+  | "include"
+>;
+
+export type ResponsesAdditionalOptions = {
+  built_in_tool_calls: Array<
+    | OpenAILLM.Responses.ResponseFileSearchToolCall
+    | OpenAILLM.Responses.ResponseComputerToolCall
+    | OpenAILLM.Responses.ResponseFunctionWebSearch
+    | OpenAILLM.Responses.ResponseFileSearchCallCompletedEvent
+    | OpenAILLM.Responses.ResponseWebSearchCallCompletedEvent
+  >;
+  annotations?: Array<
+    | OpenAILLM.Responses.ResponseOutputText.FileCitation
+    | OpenAILLM.Responses.ResponseOutputText.URLCitation
+    | OpenAILLM.Responses.ResponseOutputText.FilePath
+  >;
+  refusal?: string;
+  reasoning?: OpenAILLM.Responses.ResponseReasoningItem;
+  usage?: OpenAILLM.Responses.ResponseUsage;
+};
+
+export type StreamState = {
+  delta: string;
+  currentToolCall: PartialToolCall | null;
+  shouldEmitToolCall: PartialToolCall | null;
+  options: ResponsesAdditionalOptions;
+  previousResponseId: string | null;
+};
+
+export type ResponsesMessageContentTextDetail = {
+  type: "input_text";
+  text: string;
+};
+
+export type ResponsesMessageContentImageDetail = {
+  type: "input_image";
+  image_url: string;
+  detail: "high" | "low" | "auto";
+};
+export type ResponsesMessageContentDetail =
+  | ResponsesMessageContentTextDetail
+  | ResponsesMessageContentImageDetail;
+
+export type ResponseMessageContent = string | ResponsesMessageContentDetail[];
+
+export type OpenAIResponsesRole = "user" | "assistant" | "system" | "developer";

packages/providers/openai/tests/responses.test.ts

@@ -0,0 +1,223 @@
+import type { BaseTool, ToolCallOptions } from "@llamaindex/core/llms";
+import { describe, expect, it } from "vitest";
+import { OpenAIResponses } from "../src/responses";
+
+const API_KEY = process.env.MY_OPENAI_API_KEY;
+
+describe("OpenAIResponses Integration Tests", () => {
+  if (!API_KEY) {
+    describe.skip("OpenAI API key not found skipping tests");
+    return;
+  }
+
+  const llm = new OpenAIResponses({
+    model: "gpt-4o",
+    apiKey: API_KEY,
+  });
+
+  it("should handle basic text chat", async () => {
+    const response = await llm.chat({
+      messages: [
+        {
+          role: "user",
+          content: "What is 2+2? Answer in one word.",
+        },
+      ],
+    });
+
+    expect(response.message.content).toBe("Four.");
+    expect(response.message.role).toBe("assistant");
+  });
+
+  it("should handle image analysis", async () => {
+    const response = await llm.chat({
+      messages: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "What's in this image? Describe in one sentence.",
+            },
+            {
+              type: "image_url",
+              image_url: {
+                url: "https://storage.googleapis.com/cloud-samples-data/vision/face/faces.jpeg",
+              },
+            },
+          ],
+        },
+      ],
+    });
+    expect(response.raw).toHaveProperty("id");
+  });
+
+  it("should handle function calls", async () => {
+    const weatherTool: BaseTool = {
+      metadata: {
+        name: "get_weather",
+        description: "Get current weather for a location",
+        parameters: {
+          type: "object",
+          properties: {
+            location: {
+              type: "string",
+              description: "City name",
+            },
+          },
+          required: ["location"],
+        },
+      },
+    };
+
+    const response = await llm.chat({
+      messages: [
+        {
+          role: "user",
+          content: "What's the weather in London?",
+        },
+      ],
+      tools: [weatherTool],
+    });
+
+    expect(
+      (response.message.options as ToolCallOptions)?.toolCall,
+    ).toBeDefined();
+    const toolCall = (response.message.options as ToolCallOptions)
+      ?.toolCall?.[0];
+    expect(toolCall?.name).toBe("get_weather");
+    expect(
+      typeof toolCall?.input === "string" &&
+        JSON.parse(toolCall?.input || "{}"),
+    ).toHaveProperty("location", "London");
+  });
+});
+
+describe("OpenAIResponses Unit Tests", () => {
+  // Testing utility functions
+  describe("processMessageContent", () => {
+    const llm = new OpenAIResponses({
+      model: "gpt-4o",
+      apiKey: "test",
+    });
+
+    it("should handle non-array content (string)", () => {
+      const content = "Hello world";
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result).toBe(content);
+    });
+
+    it("should process text content", () => {
+      const content = [
+        {
+          type: "text",
+          text: "Hello world",
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result[0]).toEqual({
+        type: "input_text",
+        text: "Hello world",
+      });
+    });
+
+    it("should process image content with default detail", () => {
+      const content = [
+        {
+          type: "image_url",
+          image_url: { url: "https://example.com/image.jpg" },
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result[0]).toEqual({
+        type: "input_image",
+        image_url: "https://example.com/image.jpg",
+        detail: "auto",
+      });
+    });
+
+    it("should process image content with specified detail", () => {
+      const content = [
+        {
+          type: "image_url",
+          image_url: { url: "https://example.com/image.jpg" },
+          detail: "high",
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result[0]).toEqual({
+        type: "input_image",
+        image_url: "https://example.com/image.jpg",
+        detail: "high",
+      });
+    });
+
+    it("should process mixed content", () => {
+      const content = [
+        {
+          type: "text",
+          text: "What's in this image?",
+        },
+        {
+          type: "image_url",
+          image_url: { url: "https://example.com/image.jpg" },
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result).toEqual([
+        {
+          type: "input_text",
+          text: "What's in this image?",
+        },
+        {
+          type: "input_image",
+          image_url: "https://example.com/image.jpg",
+          detail: "auto",
+        },
+      ]);
+    });
+  });
+
+  describe("isResponseCreatedEvent", () => {
+    const llm = new OpenAIResponses({
+      model: "gpt-4o",
+      apiKey: "test",
+    });
+
+    it("should identify response created events", () => {
+      const event = { type: "response.created", response_id: "123" };
+      // @ts-expect-error accessing private method
+      expect(llm.isResponseCreatedEvent(event)).toBe(true);
+    });
+
+    it("should reject non-created events", () => {
+      const event = { type: "response.other", response_id: "123" };
+      // @ts-expect-error accessing private method
+      expect(llm.isResponseCreatedEvent(event)).toBe(false);
+    });
+  });
+
+  describe("isFunctionCall", () => {
+    const llm = new OpenAIResponses({
+      model: "gpt-4o",
+      apiKey: "test",
+    });
+
+    it("should identify function calls", () => {
+      const item = { type: "function_call", name: "test", arguments: "{}" };
+      // @ts-expect-error accessing private method
+      expect(llm.isFunctionCall(item)).toBe(true);
+    });
+
+    it("should reject non-function calls", () => {
+      const item = { type: "message", content: "test" };
+      // @ts-expect-error accessing private method
+      expect(llm.isFunctionCall(item)).toBe(false);
+    });
+  });
+});

packages/providers/perplexity/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/perplexity
 
+## 0.0.5
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
 ## 0.0.4
 
 ### Patch Changes

packages/providers/perplexity/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/perplexity",
   "description": "Perplexity Adapter for LlamaIndex",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/portkey-ai/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/portkey-ai
 
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.0.43
 
 ### Patch Changes

packages/providers/portkey-ai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/portkey-ai",
   "description": "Portkey Adapter for LlamaIndex",
-  "version": "0.0.43",
+  "version": "0.0.44",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/replicate/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/replicate
 
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.0.43
 
 ### Patch Changes

packages/providers/replicate/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/replicate",
   "description": "Replicate Adapter for LlamaIndex",
-  "version": "0.0.43",
+  "version": "0.0.44",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/storage/astra/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/astra
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.0.15
 
 ### Patch Changes

packages/providers/storage/astra/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/astra",
   "description": "Astra Storage for LlamaIndex",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/storage/azure/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/azure
 
+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.1.10
 
 ### Patch Changes

packages/providers/storage/azure/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/azure",
   "description": "Azure Storage for LlamaIndex",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/storage/chroma/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/chroma
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.0.15
 
 ### Patch Changes

packages/providers/storage/chroma/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/chroma",
   "description": "Chroma Storage for LlamaIndex",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/storage/elastic-search/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/elastic-search
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.1.1
 
 ### Patch Changes

packages/providers/storage/elastic-search/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/elastic-search",
   "description": "Elastic Search Storage for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/storage/firestore/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/firestore
 
+## 1.0.9
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 1.0.8
 
 ### Patch Changes

packages/providers/storage/firestore/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/firestore",
   "description": "Firestore Storage for LlamaIndex",
-  "version": "1.0.8",
+  "version": "1.0.9",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/storage/milvus/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/milvus
 
+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.1.10
 
 ### Patch Changes

packages/providers/storage/milvus/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/milvus",
   "description": "Milvus Storage for LlamaIndex",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/storage/mongodb/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/mongodb
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.0.15
 
 ### Patch Changes

packages/providers/storage/mongodb/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/mongodb",
   "description": "MongoDB Storage for LlamaIndex",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/storage/pinecone/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/pinecone
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
 ## 0.1.1
 
 ### Patch Changes