Release 0.9.15 (#1807)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>

apps/next/CHANGELOG.md

@@ -1,5 +1,50 @@
 # @llamaindex/doc
 
+## 0.2.4
+
+### Patch Changes
+
+- 9c63f3f: Add support for openai responses api
+- Updated dependencies [9c63f3f]
+- Updated dependencies [c515a32]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+  - @llamaindex/workflow@1.0.2
+  - llamaindex@0.9.15
+  - @llamaindex/cloud@4.0.2
+  - @llamaindex/node-parser@2.0.2
+  - @llamaindex/readers@3.0.2
+
+## 0.2.3
+
+### Patch Changes
+
+- 648cfb5: Add support for supabase vector store
+  Added doc for the supbase vector store
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+- Updated dependencies [9d951b2]
+  - @llamaindex/core@0.6.1
+  - llamaindex@0.9.14
+  - @llamaindex/cloud@4.0.1
+  - @llamaindex/node-parser@2.0.1
+  - @llamaindex/openai@0.2.1
+  - @llamaindex/readers@3.0.1
+  - @llamaindex/workflow@1.0.1
+
+## 0.2.2
+
+### Patch Changes
+
+- e98033e: docs: correct the number of indexes
+
+## 0.2.1
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+
 ## 0.2.0
 
 ### Minor Changes

apps/next/next.config.mjs

@@ -4,6 +4,8 @@ const withMDX = createMDX();
 
 /** @type {import('next').NextConfig} */
 const config = {
+  // default timeout for static generation is 60s, but we need to increase it to 10 minutes due to the large number of document pages
+  staticPageGenerationTimeout: 600,
   reactStrictMode: true,
   eslint: {
     ignoreDuringBuilds: true,

apps/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/doc",
-  "version": "0.2.0",
+  "version": "0.2.4",
   "private": true,
   "scripts": {
     "postinstall": "fumadocs-mdx",

apps/next/src/content/docs/llamaindex/modules/data_index.mdx

@@ -2,7 +2,7 @@
 title: Index
 ---
 
-An index is the basic container and organization for your data. LlamaIndex.TS supports two indexes:
+An index is the basic container and organization for your data. LlamaIndex.TS supports three indexes:
 
 - `VectorStoreIndex` - will send the top-k `Node`s to the LLM when generating a response. The default top-k is 2.
 - `SummaryIndex` - will send every `Node` in the index to the LLM in order to generate a response

apps/next/src/content/docs/llamaindex/modules/data_stores/vector_stores/supabase.mdx

@@ -0,0 +1,166 @@
+---
+title: Supabase Vector Store
+---
+
+[supabase.com](https://supabase.com/)
+
+To use this vector store, you need a Supabase project. You can create one at [supabase.com](https://supabase.com/).
+
+## Installation
+
+import { Tab, Tabs } from "fumadocs-ui/components/tabs";
+
+<Tabs groupId="install" items={["npm", "yarn", "pnpm"]} persist>
+  ```shell tab="npm"
+  npm install llamaindex @llamaindex/supabase
+  ```
+
+  ```shell tab="yarn"
+  yarn add llamaindex @llamaindex/supabase
+  ```
+
+  ```shell tab="pnpm"
+  pnpm add llamaindex @llamaindex/supabase
+  ```
+</Tabs>
+
+## Database Setup
+
+Before using the vector store, you need to:
+1. Enable the `pgvector` extension
+2. Create a table for storing vectors
+3. Create a vector similarity search function
+
+```sql
+create table documents (
+id uuid primary key,
+content text,
+metadata jsonb,
+embedding vector(1536)
+);
+```
+
+-- Create a function for similarity search
+```sql
+create function match_documents (
+query_embedding vector(1536),
+match_count int
+) returns table (
+id uuid,
+content text,
+metadata jsonb,
+embedding vector(1536),
+similarity float
+)
+language plpgsql
+as $$
+begin
+return query
+select
+id,
+content,
+metadata,
+embedding,
+1 - (embedding <=> query_embedding) as similarity
+from documents
+order by embedding <=> query_embedding
+limit match_count;
+end;
+$$;
+```
+
+## Importing the modules
+
+```ts
+import { Document, VectorStoreIndex } from "llamaindex";
+import { SupabaseVectorStore } from "@llamaindex/supabase";
+```
+
+## Setup Supabase
+
+```ts
+const vectorStore = new SupabaseVectorStore({
+  supabaseUrl: process.env.SUPABASE_URL,
+  supabaseKey: process.env.SUPABASE_KEY,
+  table: "documents",
+});
+```
+
+## Setup the index
+
+```ts
+const documents = [
+  new Document({ 
+    text: "Sample document text",
+    metadata: { source: "example" }
+  })
+];
+
+const storageContext = await storageContextFromDefaults({ vectorStore });
+const index = await VectorStoreIndex.fromDocuments(documents, {
+  storageContext,
+});
+```
+
+## Query the index
+
+```ts
+const queryEngine = index.asQueryEngine();
+
+const response = await queryEngine.query({
+  query: "What is in the document?",
+});
+
+// Output response
+console.log(response.toString());
+```
+
+## Full code
+
+```ts
+import { Document, VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
+import { SupabaseVectorStore } from "@llamaindex/supabase";
+
+async function main() {
+  // Initialize the vector store
+  const vectorStore = new SupabaseVectorStore({
+    supabaseUrl: process.env.SUPABASE_URL,
+    supabaseKey: process.env.SUPABASE_KEY,
+    table: "documents",
+  });
+
+  // Create sample documents
+  const documents = [
+    new Document({
+      text: "Vector search enables semantic similarity search",
+      metadata: {
+        source: "research_paper",
+        author: "Jane Smith",
+      },
+    }),
+  ];
+
+  // Create storage context
+  const storageContext = await storageContextFromDefaults({ vectorStore });
+
+  // Create and store embeddings
+  const index = await VectorStoreIndex.fromDocuments(documents, {
+    storageContext,
+  });
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query({
+    query: "What is vector search?",
+  });
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);
+```
+
+## API Reference
+
+- [SupabaseVectorStore](/docs/api/classes/SupabaseVectorStore)

apps/next/src/content/docs/llamaindex/modules/llms/openai.mdx

@@ -46,6 +46,156 @@ or
 Settings.llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0, apiKey: <YOUR_API_KEY>, baseURL: "https://api.scaleway.ai/v1" });
 ```
 
+## Using OpenAI Responses API
+
+The OpenAI Responses API provides enhanced functionality for handling complex interactions, including built-in tools, annotations, and streaming responses. Here's how to use it:
+
+### Basic Setup
+
+```ts
+import { openaiResponses } from "@llamaindex/openai";
+
+const llm = openaiResponses({
+  model: "gpt-4o",
+  temperature: 0.1,
+  maxOutputTokens: 1000
+});
+```
+
+### Message Content Types
+
+The API supports different types of message content, including text and images:
+
+```ts
+const response = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: [
+        {
+          type: "input_text",
+          text: "What's in this image?"
+        },
+        {
+          type: "input_image",
+          image_url: "https://example.com/image.jpg",
+          detail: "auto" // Optional: can be "auto", "low", or "high"
+        }
+      ]
+    }
+  ]
+});
+```
+
+### Advanced Features
+
+#### Built-in Tools
+
+```ts
+const llm = openaiResponses({
+  model: "gpt-4o",
+  builtInTools: [
+    {
+      type: "function",
+      name: "search_files",
+      description: "Search through available files"
+    }
+  ],
+  strict: true // Enable strict mode for tool calls
+});
+```
+
+#### Response Tracking and Storage
+
+```ts
+const llm = openaiResponses({
+  trackPreviousResponses: true, // Enable response tracking
+  store: true, // Store responses for future reference
+  user: "user-123", // Associate responses with a user
+  callMetadata: { // Add custom metadata
+    sessionId: "session-123",
+    context: "customer-support"
+  }
+});
+```
+
+#### Streaming Responses
+
+```ts
+const response = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: "Generate a long response"
+    }
+  ],
+  stream: true // Enable streaming
+});
+
+for await (const chunk of response) {
+  console.log(chunk.delta); // Process each chunk of the response
+}
+```
+
+### Configuration Options
+
+The OpenAI Responses API supports various configuration options:
+
+```ts
+const llm = openaiResponses({
+  // Model and basic settings
+  model: "gpt-4o",
+  temperature: 0.1,
+  topP: 1,
+  maxOutputTokens: 1000,
+  
+  // API configuration
+  apiKey: "your-api-key",
+  baseURL: "custom-endpoint",
+  maxRetries: 10,
+  timeout: 60000,
+  
+  // Response handling
+  trackPreviousResponses: false,
+  store: false,
+  strict: false,
+  
+  // Additional options
+  instructions: "Custom instructions for the model",
+  truncation: "auto", // Can be "auto", "disabled", or null
+  include: ["citations", "reasoning"] // Specify what to include in responses
+});
+```
+
+### Response Structure
+
+The API returns responses with rich metadata and optional annotations:
+
+```ts
+interface ResponseStructure {
+  message: {
+    content: string;
+    role: "assistant";
+    options: {
+      built_in_tool_calls: Array<ToolCall>;
+      annotations?: Array<Citation | URLCitation | FilePath>;
+      refusal?: string;
+      reasoning?: ReasoningItem;
+      usage?: ResponseUsage;
+      toolCall?: Array<PartialToolCall>;
+    }
+  }
+}
+```
+
+### Best Practices
+
+1. Use `trackPreviousResponses` when you need conversation continuity
+2. Enable `strict` mode when using tools to ensure accurate function calls
+3. Set appropriate `maxOutputTokens` to control response length
+4. Use `annotations` to track citations and references in responses
+5. Implement error handling for potential API failures and retries
+
 ## Using JSON Response Format
 
 You can configure OpenAI to return responses in JSON format:
@@ -73,6 +223,112 @@ Settings.llm = new OpenAI({
 });
 ```
 
+## Response Formats
+
+The OpenAI LLM supports different response formats to structure the output in specific ways. There are two main approaches to formatting responses:
+
+### 1. JSON Object Format
+
+The simplest way to get structured JSON responses is using the `json_object` response format:
+
+```ts
+Settings.llm = new OpenAI({ 
+  model: "gpt-4o", 
+  temperature: 0,
+  responseFormat: { type: "json_object" }  
+});
+
+const response = await llm.chat({
+  messages: [
+    {
+      role: "system",
+      content: "You are a helpful assistant that outputs JSON."
+    },
+    {
+      role: "user", 
+      content: "Summarize this meeting transcript"
+    }
+  ]
+});
+
+// Response will be valid JSON
+console.log(response.message.content);
+```
+
+### 2. Schema Validation with Zod
+
+For more robust type safety and validation, you can use Zod schemas to define the expected response structure:
+
+```ts
+import { z } from "zod";
+
+// Define the response schema
+const meetingSchema = z.object({
+  summary: z.string(),
+  participants: z.array(z.string()),
+  actionItems: z.array(z.string()),
+  nextSteps: z.string()
+});
+
+// Configure the LLM with the schema
+Settings.llm = new OpenAI({ 
+  model: "gpt-4o", 
+  temperature: 0,
+  responseFormat: meetingSchema
+});
+
+const response = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: "Summarize this meeting transcript" 
+    }
+  ]
+});
+
+// Response will be typed and validated according to the schema
+const result = response.message.content;
+console.log(result.summary);
+console.log(result.actionItems);
+```
+
+### Response Format Options
+
+The response format can be configured in two ways:
+
+1. At LLM initialization:
+```ts
+const llm = new OpenAI({
+  model: "gpt-4o",
+  responseFormat: { type: "json_object" } // or a Zod schema
+});
+```
+
+2. Per request:
+```ts
+const response = await llm.chat({
+  messages: [...],
+  responseFormat: { type: "json_object" } // or a Zod schema
+});
+```
+
+The response format options are:
+
+- `{ type: "json_object" }` - Returns responses as JSON objects
+- `zodSchema` - A Zod schema that defines and validates the response structure
+
+### Best Practices
+
+1. Use JSON object format for simple structured responses
+2. Use Zod schemas when you need:
+   - Type safety
+   - Response validation
+   - Complex nested structures
+   - Specific field constraints
+3. Set a low temperature (e.g. 0) when using structured outputs for more reliable formatting
+4. Include clear instructions in system or user messages about the expected response format
+5. Handle potential parsing errors when working with JSON responses
+
 ## Load and index documents
 
 For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.

apps/next/src/content/docs/llamaindex/modules/tool/index.mdx

@@ -7,9 +7,59 @@ A tool can be called to perform custom actions, or retrieve extra information ba
 A result from a tool call can be used by subsequent steps in a workflow, or to compute a final answer.
 For example, a "weather tool" could fetch some live weather information from a geographical location.
 
+## Tool Function
+
+The `tool` function is a utility provided to define a tool that can be used by an agent. It takes a function and a configuration object as arguments. The configuration object includes the tool's name, description, and parameters.
+
+### Parameters with Zod
+
+The `parameters` field in the tool configuration is defined using `zod`, a TypeScript-first schema declaration and validation library. `zod` allows you to specify the expected structure and types of the input parameters, ensuring that the data passed to the tool is valid.
+
+Example:
+```ts
+import { agent, tool } from "llamaindex";
+import { z } from "zod";
+
+// first arg is LLM input, second is bound arg
+const queryKnowledgeBase = async ({ question }, { userToken }) => {
+  const response = await fetch(`https://knowledge-base.com?token=${userToken}&query=${question}`);
+  // ...
+};
+
+// define tool with zod validation
+const kbTool = tool(queryKnowledgeBase, {
+  name: 'queryKnowledgeBase',
+  description: 'Query knowledge base',
+  parameters: z.object({
+    question: z.string({
+      description: 'The user question',
+    }),
+  }),
+});
+
+```
+In this example, `z.object` is used to define a schema for the `parameters` where `question` is expected to be a string. This ensures that any input to the tool adheres to the specified structure, providing a layer of type safety and validation.
+
+
+## Built-in tools
+
+You can import built-in tools from the `@llamaindex/tools` package.
+
+```ts
+import { agent } from "llamaindex";
+import { wiki } from "@llamaindex/tools";
+
+const researchAgent = agent({
+  name: "WikiAgent",
+  description: "Gathering information from the internet",
+  systemPrompt: `You are a research agent. Your role is to gather information from the internet using the provided tools.`,
+  tools: [wiki()],
+});
+```
+
 ## Function tool
 
-Function tools are implemented with the `FunctionTool` class.
+You can still use the `FunctionTool` class to define a tool.
 A `FunctionTool` is constructed from a function with signature
 ```ts
 (input: T, additionalArg?: AdditionalToolArgument) => R

e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.149
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
+## 0.0.148
+
+### Patch Changes
+
+- Updated dependencies [9d951b2]
+  - llamaindex@0.9.14
+
+## 0.0.147
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+
 ## 0.0.146
 
 ### Patch Changes

e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.146",
+  "version": "0.0.149",
   "type": "module",
   "private": true,
   "scripts": {

e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.57
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.2
+
+## 0.0.56
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.1
+
 ## 0.0.55
 
 ### Patch Changes

e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.55",
+  "version": "0.0.57",
   "type": "module",
   "scripts": {
     "dev": "vite",
@@ -10,7 +10,7 @@
   },
   "devDependencies": {
     "typescript": "^5.7.3",
-    "vite": "^5.4.12",
+    "vite": "^5.4.16",
     "vite-plugin-wasm": "^3.3.0"
   },
   "dependencies": {

e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @llamaindex/next-agent-test
 
+## 0.1.149
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
+## 0.1.148
+
+### Patch Changes
+
+- Updated dependencies [9d951b2]
+  - llamaindex@0.9.14
+
+## 0.1.147
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+
 ## 0.1.146
 
 ### Patch Changes

e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.146",
+  "version": "0.1.149",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,25 @@
 # test-edge-runtime
 
+## 0.1.148
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
+## 0.1.147
+
+### Patch Changes
+
+- Updated dependencies [9d951b2]
+  - llamaindex@0.9.14
+
+## 0.1.146
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+
 ## 0.1.145
 
 ### Patch Changes

e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.145",
+  "version": "0.1.148",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,29 @@
 # @llamaindex/next-node-runtime
 
+## 0.1.15
+
+### Patch Changes
+
+- llamaindex@0.9.15
+- @llamaindex/huggingface@0.1.2
+- @llamaindex/readers@3.0.2
+
+## 0.1.14
+
+### Patch Changes
+
+- Updated dependencies [9d951b2]
+  - llamaindex@0.9.14
+  - @llamaindex/huggingface@0.1.1
+  - @llamaindex/readers@3.0.1
+
+## 0.1.13
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+
 ## 0.1.12
 
 ### Patch Changes

e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.12",
+  "version": "0.1.15",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/vite-import-llamaindex/CHANGELOG.md

@@ -1,5 +1,25 @@
 # vite-import-llamaindex
 
+## 0.0.15
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [9d951b2]
+  - llamaindex@0.9.14
+
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+
 ## 0.0.12
 
 ### Patch Changes

e2e/examples/vite-import-llamaindex/package.json

@@ -1,7 +1,7 @@
 {
   "name": "vite-import-llamaindex",
   "private": true,
-  "version": "0.0.12",
+  "version": "0.0.15",
   "type": "module",
   "scripts": {
     "build": "vite build",
@@ -16,7 +16,7 @@
     "@size-limit/preset-big-lib": "^11.1.6",
     "size-limit": "^11.1.6",
     "typescript": "^5.7.3",
-    "vite": "^6.1.0"
+    "vite": "^5.4.16"
   },
   "dependencies": {
     "llamaindex": "workspace:*"

e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.149
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
+## 0.0.148
+
+### Patch Changes
+
+- Updated dependencies [9d951b2]
+  - llamaindex@0.9.14
+
+## 0.0.147
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+
 ## 0.0.146
 
 ### Patch Changes

e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.146",
+  "version": "0.0.149",
   "type": "module",
   "private": true,
   "scripts": {

examples/.nvmrc

@@ -0,0 +1 @@
+20

examples/CHANGELOG.md

@@ -1,5 +1,106 @@
 # examples
 
+## 0.3.2
+
+### Patch Changes
+
+- 9c63f3f: Add support for openai responses api
+- Updated dependencies [9c63f3f]
+- Updated dependencies [c515a32]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/google@0.2.2
+  - @llamaindex/core@0.6.2
+  - @llamaindex/workflow@1.0.2
+  - llamaindex@0.9.15
+  - @llamaindex/clip@0.0.48
+  - @llamaindex/deepinfra@0.0.48
+  - @llamaindex/deepseek@0.0.8
+  - @llamaindex/fireworks@0.0.8
+  - @llamaindex/groq@0.0.63
+  - @llamaindex/huggingface@0.1.2
+  - @llamaindex/jinaai@0.0.8
+  - @llamaindex/perplexity@0.0.5
+  - @llamaindex/azure@0.1.11
+  - @llamaindex/elastic-search@0.1.2
+  - @llamaindex/milvus@0.1.11
+  - @llamaindex/qdrant@0.1.11
+  - @llamaindex/supabase@0.1.1
+  - @llamaindex/together@0.0.8
+  - @llamaindex/vllm@0.0.34
+  - @llamaindex/cloud@4.0.2
+  - @llamaindex/node-parser@2.0.2
+  - @llamaindex/anthropic@0.3.2
+  - @llamaindex/cohere@0.0.16
+  - @llamaindex/mistral@0.1.2
+  - @llamaindex/mixedbread@0.0.16
+  - @llamaindex/ollama@0.1.2
+  - @llamaindex/portkey-ai@0.0.44
+  - @llamaindex/replicate@0.0.44
+  - @llamaindex/astra@0.0.16
+  - @llamaindex/chroma@0.0.16
+  - @llamaindex/firestore@1.0.9
+  - @llamaindex/mongodb@0.0.16
+  - @llamaindex/pinecone@0.1.2
+  - @llamaindex/postgres@0.0.44
+  - @llamaindex/upstash@0.0.16
+  - @llamaindex/weaviate@0.0.16
+  - @llamaindex/vercel@0.1.2
+  - @llamaindex/voyage-ai@1.0.8
+  - @llamaindex/readers@3.0.2
+  - @llamaindex/tools@0.0.4
+
+## 0.3.1
+
+### Patch Changes
+
+- 648cfb5: Add support for supabase vector store
+  Added doc for the supbase vector store
+- Updated dependencies [648cfb5]
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+- Updated dependencies [9d951b2]
+  - @llamaindex/supabase@0.1.0
+  - @llamaindex/core@0.6.1
+  - llamaindex@0.9.14
+  - @llamaindex/tools@0.0.3
+  - @llamaindex/cloud@4.0.1
+  - @llamaindex/node-parser@2.0.1
+  - @llamaindex/anthropic@0.3.1
+  - @llamaindex/clip@0.0.47
+  - @llamaindex/cohere@0.0.15
+  - @llamaindex/deepinfra@0.0.47
+  - @llamaindex/google@0.2.1
+  - @llamaindex/huggingface@0.1.1
+  - @llamaindex/jinaai@0.0.7
+  - @llamaindex/mistral@0.1.1
+  - @llamaindex/mixedbread@0.0.15
+  - @llamaindex/ollama@0.1.1
+  - @llamaindex/openai@0.2.1
+  - @llamaindex/perplexity@0.0.4
+  - @llamaindex/portkey-ai@0.0.43
+  - @llamaindex/replicate@0.0.43
+  - @llamaindex/astra@0.0.15
+  - @llamaindex/azure@0.1.10
+  - @llamaindex/chroma@0.0.15
+  - @llamaindex/elastic-search@0.1.1
+  - @llamaindex/firestore@1.0.8
+  - @llamaindex/milvus@0.1.10
+  - @llamaindex/mongodb@0.0.15
+  - @llamaindex/pinecone@0.1.1
+  - @llamaindex/postgres@0.0.43
+  - @llamaindex/qdrant@0.1.10
+  - @llamaindex/upstash@0.0.15
+  - @llamaindex/weaviate@0.0.15
+  - @llamaindex/vercel@0.1.1
+  - @llamaindex/voyage-ai@1.0.7
+  - @llamaindex/readers@3.0.1
+  - @llamaindex/workflow@1.0.1
+  - @llamaindex/deepseek@0.0.7
+  - @llamaindex/fireworks@0.0.7
+  - @llamaindex/groq@0.0.62
+  - @llamaindex/together@0.0.7
+  - @llamaindex/vllm@0.0.33
+
 ## 0.3.0
 
 ### Minor Changes

examples/agent/query_tool.ts

@@ -0,0 +1,51 @@
+import {
+  AgentStream,
+  AgentToolCallResult,
+  Document,
+  VectorStoreIndex,
+  agent,
+  openai,
+} from "llamaindex";
+
+async function main() {
+  const index = await VectorStoreIndex.fromDocuments([
+    new Document({
+      text: "Cats have a specialized collarbone that allows them to always land on their feet when they fall.",
+    }),
+    new Document({
+      text: "Dogs have a sense of smell that is 10,000 to 100,000 times more acute than humans.",
+    }),
+    new Document({
+      text: "Cats are known for their agility and ability to jump high.",
+    }),
+  ]);
+
+  const myAgent = agent({
+    llm: openai({ model: "gpt-4o" }),
+    tools: [
+      index.queryTool({
+        options: { similarityTopK: 2 },
+        includeSourceNodes: true,
+      }),
+    ],
+  });
+
+  const context = myAgent.run("The fact about cats");
+
+  for await (const event of context) {
+    if (event instanceof AgentToolCallResult) {
+      console.log(
+        "Using these retrieved information to answer the question:\n",
+        event.data.toolOutput.result,
+      );
+    } else if (event instanceof AgentStream) {
+      for (const chunk of event.data.delta) {
+        process.stdout.write(chunk);
+      }
+    }
+  }
+}
+
+void main().then(() => {
+  console.log("Done");
+});

examples/openai/responses/agent.ts

@@ -0,0 +1,30 @@
+import { openaiResponses } from "@llamaindex/openai";
+import { wiki } from "@llamaindex/tools";
+import { agent, tool } from "llamaindex";
+import { z } from "zod";
+
+const workflow = agent({
+  tools: [
+    tool({
+      name: "weather",
+      description: "Get the weather",
+      parameters: z.object({
+        location: z.string().describe("The location to get the weather for"),
+      }),
+      execute: ({ location }) => `The weather in ${location} is sunny`,
+    }),
+    wiki(),
+  ],
+  llm: openaiResponses({
+    model: "gpt-4o-mini",
+  }),
+});
+
+async function main() {
+  const result = await workflow.run(
+    "What is the weather in New York? What's the history of New York from Wikipedia in 3 sentences?",
+  );
+  console.log(result.data);
+}
+
+void main();

examples/openai/responses/image.ts

@@ -0,0 +1,33 @@
+import { openaiResponses } from "@llamaindex/openai";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4o",
+    maxOutputTokens: 1000,
+    apiKey: process.env.MY_OPENAI_API_KEY,
+  });
+
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this image? Describe it in detail.",
+          },
+          {
+            type: "image_url",
+            image_url: {
+              url: "https://storage.googleapis.com/cloud-samples-data/vision/face/faces.jpeg",
+            },
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log("Single Image Analysis:", response.message.content);
+}
+
+main().catch(console.error);

examples/openai/responses/openai-responses.ts

@@ -0,0 +1,22 @@
+import { openaiResponses } from "@llamaindex/openai";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4o-mini",
+    temperature: 0.1,
+  });
+
+  // Basic chat example
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: "What is the capital of France?",
+      },
+    ],
+  });
+
+  console.log(response.message.content);
+}
+
+main().catch(console.error);

examples/openai/responses/stream.ts

@@ -0,0 +1,26 @@
+import { openaiResponses } from "@llamaindex/openai";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4o-mini",
+    temperature: 0.1,
+  });
+
+  const stream = await llm.chat({
+    messages: [
+      { content: "You want to talk in rhymes.", role: "system" },
+      {
+        content:
+          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
+        role: "user",
+      },
+    ],
+    stream: true,
+  });
+
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.delta);
+  }
+}
+
+main().catch(console.error);

examples/openai/responses/tool-call.ts

@@ -0,0 +1,37 @@
+import { openaiResponses } from "@llamaindex/openai";
+import { tool } from "llamaindex";
+
+import { z } from "zod";
+async function main() {
+  const weatherTool = tool({
+    name: "weather",
+    description: "Get the weather",
+    parameters: z.object({
+      location: z.string({
+        description: "The location to get the weather for",
+      }),
+    }),
+    execute: ({ location }) => {
+      return `The weather in ${location} is sunny`;
+    },
+  });
+
+  const llm = openaiResponses({
+    model: "gpt-4o-mini",
+    temperature: 0.1,
+  });
+
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: "What is the weather in New York?",
+      },
+    ],
+    tools: [weatherTool],
+  });
+
+  console.log(response.message.options);
+}
+
+main().catch(console.error);

examples/openai/responses/web-search.ts

@@ -0,0 +1,23 @@
+import { openaiResponses } from "@llamaindex/openai";
+
+async function main() {
+  const llm = openaiResponses({
+    model: "gpt-4o",
+    temperature: 0.1,
+    builtInTools: [{ type: "web_search_preview" }],
+  });
+
+  // Streaming chat example
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: "What are the latest developments in AI?",
+      },
+    ],
+  });
+
+  console.log(response.message.content);
+}
+
+main().catch(console.error);

examples/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/examples",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "private": true,
   "scripts": {
     "lint": "eslint .",
@@ -11,46 +11,47 @@
     "@azure/cosmos": "^4.1.1",
     "@azure/identity": "^4.4.1",
     "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.0",
-    "@llamaindex/astra": "^0.0.14",
-    "@llamaindex/azure": "^0.1.9",
-    "@llamaindex/chroma": "^0.0.14",
-    "@llamaindex/clip": "^0.0.46",
-    "@llamaindex/cloud": "^4.0.0",
-    "@llamaindex/cohere": "^0.0.14",
-    "@llamaindex/core": "^0.6.0",
-    "@llamaindex/deepinfra": "^0.0.46",
+    "@llamaindex/anthropic": "^0.3.2",
+    "@llamaindex/astra": "^0.0.16",
+    "@llamaindex/azure": "^0.1.11",
+    "@llamaindex/chroma": "^0.0.16",
+    "@llamaindex/clip": "^0.0.48",
+    "@llamaindex/cloud": "^4.0.2",
+    "@llamaindex/cohere": "^0.0.16",
+    "@llamaindex/core": "^0.6.2",
+    "@llamaindex/deepinfra": "^0.0.48",
     "@llamaindex/env": "^0.1.29",
-    "@llamaindex/firestore": "^1.0.7",
-    "@llamaindex/google": "^0.2.0",
-    "@llamaindex/groq": "^0.0.61",
-    "@llamaindex/huggingface": "^0.1.0",
-    "@llamaindex/milvus": "^0.1.9",
-    "@llamaindex/mistral": "^0.1.0",
-    "@llamaindex/mixedbread": "^0.0.14",
-    "@llamaindex/mongodb": "^0.0.14",
-    "@llamaindex/elastic-search": "^0.1.0",
-    "@llamaindex/node-parser": "^2.0.0",
-    "@llamaindex/ollama": "^0.1.0",
-    "@llamaindex/openai": "^0.2.0",
-    "@llamaindex/pinecone": "^0.1.0",
-    "@llamaindex/portkey-ai": "^0.0.42",
-    "@llamaindex/postgres": "^0.0.42",
-    "@llamaindex/qdrant": "^0.1.9",
-    "@llamaindex/readers": "^3.0.0",
-    "@llamaindex/replicate": "^0.0.42",
-    "@llamaindex/upstash": "^0.0.14",
-    "@llamaindex/vercel": "^0.1.0",
-    "@llamaindex/vllm": "^0.0.32",
-    "@llamaindex/voyage-ai": "^1.0.6",
-    "@llamaindex/weaviate": "^0.0.14",
-    "@llamaindex/workflow": "^1.0.0",
-    "@llamaindex/deepseek": "^0.0.6",
-    "@llamaindex/fireworks": "^0.0.6",
-    "@llamaindex/together": "^0.0.6",
-    "@llamaindex/jinaai": "^0.0.6",
-    "@llamaindex/perplexity": "^0.0.3",
-    "@llamaindex/tools": "^0.0.2",
+    "@llamaindex/firestore": "^1.0.9",
+    "@llamaindex/google": "^0.2.2",
+    "@llamaindex/groq": "^0.0.63",
+    "@llamaindex/huggingface": "^0.1.2",
+    "@llamaindex/milvus": "^0.1.11",
+    "@llamaindex/mistral": "^0.1.2",
+    "@llamaindex/mixedbread": "^0.0.16",
+    "@llamaindex/mongodb": "^0.0.16",
+    "@llamaindex/elastic-search": "^0.1.2",
+    "@llamaindex/node-parser": "^2.0.2",
+    "@llamaindex/ollama": "^0.1.2",
+    "@llamaindex/openai": "^0.3.0",
+    "@llamaindex/pinecone": "^0.1.2",
+    "@llamaindex/portkey-ai": "^0.0.44",
+    "@llamaindex/postgres": "^0.0.44",
+    "@llamaindex/qdrant": "^0.1.11",
+    "@llamaindex/readers": "^3.0.2",
+    "@llamaindex/replicate": "^0.0.44",
+    "@llamaindex/upstash": "^0.0.16",
+    "@llamaindex/vercel": "^0.1.2",
+    "@llamaindex/vllm": "^0.0.34",
+    "@llamaindex/voyage-ai": "^1.0.8",
+    "@llamaindex/weaviate": "^0.0.16",
+    "@llamaindex/workflow": "^1.0.2",
+    "@llamaindex/deepseek": "^0.0.8",
+    "@llamaindex/fireworks": "^0.0.8",
+    "@llamaindex/together": "^0.0.8",
+    "@llamaindex/jinaai": "^0.0.8",
+    "@llamaindex/perplexity": "^0.0.5",
+    "@llamaindex/supabase": "^0.1.1",
+    "@llamaindex/tools": "^0.0.4",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^4.0.0",
     "@vercel/postgres": "^0.10.0",
@@ -59,7 +60,7 @@
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.9.12",
+    "llamaindex": "^0.9.15",
     "mongodb": "6.7.0",
     "postgres": "^3.4.4",
     "wikipedia": "^2.1.2",

examples/vector-store/supbases/index.ts

@@ -0,0 +1,75 @@
+import {
+  gemini,
+  GEMINI_EMBEDDING_MODEL,
+  GEMINI_MODEL,
+  GeminiEmbedding,
+} from "@llamaindex/google";
+import { SupabaseVectorStore } from "@llamaindex/supabase";
+import {
+  Document,
+  Settings,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+async function main() {
+  Settings.embedModel = new GeminiEmbedding({
+    model: GEMINI_EMBEDDING_MODEL.TEXT_EMBEDDING_004,
+  });
+  Settings.llm = gemini({
+    model: GEMINI_MODEL.GEMINI_PRO_1_5_FLASH,
+  });
+  // Create sample documents
+  const documents = [
+    new Document({
+      text: "Supbase is a powerful Database engine",
+      metadata: {
+        source: "tech_docs",
+        author: "John Doe",
+      },
+    }),
+    new Document({
+      text: "Vector search enables semantic similarity search",
+      metadata: {
+        source: "research_paper",
+        author: "Jane Smith",
+      },
+    }),
+    new Document({
+      text: "Supbase vector store supports various distance metrics for vector search",
+      metadata: {
+        source: "tech_docs",
+        author: "Bob Wilson",
+      },
+    }),
+  ];
+
+  // Initialize ElasticSearch Vector Store
+  const vectorStore = new SupabaseVectorStore({
+    supabaseUrl: process.env.SUPABASE_URL,
+    supabaseKey: process.env.SUPABASE_KEY,
+    table: "document",
+  });
+
+  // await vectorStore.delete("fc079c38-2af4-4782-96e4-955c28608fcf");
+
+  // Create storage context with the vector store
+  const storageContext = await storageContextFromDefaults({
+    vectorStore,
+  });
+
+  // Create and store embeddings in ElasticSearch
+  const index = await VectorStoreIndex.fromDocuments(documents, {
+    storageContext,
+  });
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+
+  // Simple query
+  const response = await queryEngine.query({
+    query: "What is vector search?",
+  });
+  console.log("Basic Query Response:", response.toString());
+}
+
+main().catch(console.error);

examples/vector-store/supbases/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "vector-store-supabase",
+  "type": "module",
+  "private": true,
+  "scripts": {
+    "start": "npx tsx index.ts"
+  }
+}

package.json

@@ -35,7 +35,8 @@
     "prettier-plugin-tailwindcss": "^0.6.11",
     "turbo": "^2.4.4",
     "typescript": "^5.7.3",
-    "typescript-eslint": "^8.18.0"
+    "typescript-eslint": "^8.18.0",
+    "vitest": "^3.1.1"
   },
   "packageManager": "pnpm@9.12.3",
   "lint-staged": {

packages/autotool/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @llamaindex/autotool
 
+## 6.0.15
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
+## 6.0.14
+
+### Patch Changes
+
+- Updated dependencies [9d951b2]
+  - llamaindex@0.9.14
+
+## 6.0.13
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+
 ## 6.0.12
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.96
+
+### Patch Changes
+
+- llamaindex@0.9.15
+- @llamaindex/autotool@6.0.15
+
+## 0.0.95
+
+### Patch Changes
+
+- Updated dependencies [9d951b2]
+  - llamaindex@0.9.14
+  - @llamaindex/autotool@6.0.14
+
+## 0.0.94
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+  - @llamaindex/autotool@6.0.13
+
 ## 0.0.93
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.93"
+  "version": "0.0.96"
 }

packages/autotool/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
     "directory": "packages/autotool"
   },
-  "version": "6.0.12",
+  "version": "6.0.15",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/cloud
 
+## 4.0.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 4.0.1
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 4.0.0
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "4.0.0",
+  "version": "4.0.2",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @llamaindex/community
 
+## 0.0.94
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.0.93
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
+## 0.0.92
+
+### Patch Changes
+
+- 1325178: fix: stringify all tool results for anthropic on bedrock
+
 ## 0.0.91
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.91",
+  "version": "0.0.94",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/community/src/llm/bedrock/anthropic/utils.ts

@@ -111,7 +111,7 @@ export const mapChatMessagesToAnthropicMessages = <
               {
                 type: "tool_result",
                 tool_use_id: msg.options.toolResult.id,
-                content: msg.options.toolResult.result,
+                content: JSON.stringify(msg.options.toolResult.result),
               },
             ],
           },

packages/core/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/core
 
+## 0.6.2
+
+### Patch Changes
+
+- 9c63f3f: Add support for openai responses api
+
+## 0.6.1
+
+### Patch Changes
+
+- 1b6f368: Support loading from URLs for all readers extending FileReader
+- eaf326e: Fix passing right llm setting from SimpleChatEngine to ChatMemoryBuffer
+
 ## 0.6.0
 
 ### Minor Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./agent": {

packages/core/src/chat-engine/simple-chat-engine.ts

@@ -24,8 +24,12 @@ export class SimpleChatEngine implements BaseChatEngine {
   }
 
   constructor(init?: Partial<SimpleChatEngine>) {
-    this.memory = init?.memory ?? new ChatMemoryBuffer();
     this.llm = init?.llm ?? Settings.llm;
+    this.memory =
+      init?.memory ??
+      new ChatMemoryBuffer({
+        llm: this.llm,
+      });
   }
 
   chat(params: NonStreamingChatEngineParams): Promise<EngineResponse>;
@@ -40,6 +44,7 @@ export class SimpleChatEngine implements BaseChatEngine {
 
     const chatHistory = params.chatHistory
       ? new ChatMemoryBuffer({
+          llm: this.llm,
           chatHistory:
             params.chatHistory instanceof BaseMemory
               ? await params.chatHistory.getMessages()

packages/core/src/llms/type.ts

@@ -2,7 +2,6 @@ import type { Tokenizers } from "@llamaindex/env/tokenizers";
 import type { JSONSchemaType } from "ajv";
 import { z } from "zod";
 import type { JSONObject, JSONValue } from "../global";
-
 /**
  * @internal
  */
@@ -55,7 +54,12 @@ export interface LLM<
   ): Promise<CompletionResponse>;
 }
 
-export type MessageType = "user" | "assistant" | "system" | "memory";
+export type MessageType =
+  | "user"
+  | "assistant"
+  | "system"
+  | "memory"
+  | "developer";
 
 export type TextChatMessage<AdditionalMessageOptions extends object = object> =
   {
@@ -156,6 +160,7 @@ export type MessageContentTextDetail = {
 export type MessageContentImageDetail = {
   type: "image_url";
   image_url: { url: string };
+  detail?: "high" | "low" | "auto";
 };
 
 export type MessageContentDetail =

packages/core/src/schema/type.ts

@@ -63,8 +63,29 @@ export abstract class FileReader<T extends BaseNode = Document>
   ): Promise<T[]>;
 
   async loadData(filePath: string): Promise<T[]> {
-    const fileContent = await fs.readFile(filePath);
-    const filename = path.basename(filePath);
+    let fileContent: Uint8Array;
+    let filename: string;
+
+    // Check if filePath is a URL
+    if (filePath.startsWith("http://") || filePath.startsWith("https://")) {
+      // Handle URL
+      const response = await fetch(filePath);
+      if (!response.ok) {
+        throw new Error(
+          `Failed to fetch URL: ${filePath}, status: ${response.status}`,
+        );
+      }
+      const buffer = await response.arrayBuffer();
+      fileContent = new Uint8Array(buffer);
+      // Extract filename from URL
+      const url = new URL(filePath);
+      filename = path.basename(url.pathname) || "url_document";
+    } else {
+      // Handle local file
+      fileContent = await fs.readFile(filePath);
+      filename = path.basename(filePath);
+    }
+
     const docs = await this.loadDataAsContent(fileContent, filename);
     docs.forEach(FileReader.addMetaData(filePath));
     return docs;

packages/core/tests/file-reader.test.ts

@@ -0,0 +1,91 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { Document, FileReader } from "@llamaindex/core/schema";
+import { fs, path } from "@llamaindex/env";
+import { TextDecoder, TextEncoder } from "util";
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+
+// Mock implementation of FileReader for testing
+class TestFileReader extends FileReader<Document> {
+  async loadDataAsContent(
+    fileContent: Uint8Array,
+    filename?: string,
+  ): Promise<Document[]> {
+    const text = new TextDecoder().decode(fileContent);
+    return [new Document({ text, metadata: { filename } })];
+  }
+}
+
+describe("FileReader", () => {
+  let reader: TestFileReader;
+  let mockFetch: any;
+  let mockFsReadFile: any;
+
+  beforeEach(() => {
+    reader = new TestFileReader();
+
+    // Mock fetch for URL tests
+    mockFetch = vi.fn();
+    global.fetch = mockFetch;
+
+    // Mock fs.readFile for local file tests
+    mockFsReadFile = vi.spyOn(fs, "readFile");
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  test("loadData should load content from a local file", async () => {
+    const testFilePath = "/path/to/test.txt";
+    const testContent = "Test file content";
+    const testContentBuffer = new TextEncoder().encode(testContent);
+
+    mockFsReadFile.mockResolvedValue(testContentBuffer);
+
+    const result = await reader.loadData(testFilePath);
+
+    expect(mockFsReadFile).toHaveBeenCalledWith(testFilePath);
+    expect(result).toHaveLength(1);
+    expect(result[0]).toBeInstanceOf(Document);
+    expect(result[0]?.getText()).toBe(testContent);
+    expect(result[0]?.metadata.file_path).toBe(path.resolve(testFilePath));
+    expect(result[0]?.metadata.file_name).toBe("test.txt");
+  });
+
+  test("loadData should load content from a URL", async () => {
+    const testUrl = "https://example.com/test.txt";
+    const testContent = "Test URL content";
+    const testContentBuffer = new TextEncoder().encode(testContent);
+
+    // Mock fetch response
+    mockFetch.mockResolvedValue({
+      ok: true,
+      arrayBuffer: () => Promise.resolve(testContentBuffer.buffer),
+    });
+
+    const result = await reader.loadData(testUrl);
+
+    expect(mockFetch).toHaveBeenCalledWith(testUrl);
+    expect(result).toHaveLength(1);
+    expect(result[0]).toBeInstanceOf(Document);
+    expect(result[0]?.getText()).toBe(testContent);
+    expect(result[0]?.metadata.file_path).toBe(path.resolve(testUrl));
+    expect(result[0]?.metadata.file_name).toBe("test.txt");
+  });
+
+  test("loadData should throw an error for failed URL fetch", async () => {
+    const testUrl = "https://example.com/not-found.txt";
+
+    // Mock failed fetch response
+    mockFetch.mockResolvedValue({
+      ok: false,
+      status: 404,
+    });
+
+    await expect(reader.loadData(testUrl)).rejects.toThrow(
+      `Failed to fetch URL: ${testUrl}, status: 404`,
+    );
+
+    expect(mockFetch).toHaveBeenCalledWith(testUrl);
+  });
+});

packages/core/tests/simple-chat-engine.test.ts

@@ -0,0 +1,14 @@
+import { SimpleChatEngine } from "@llamaindex/core/chat-engine";
+import { ChatMemoryBuffer } from "@llamaindex/core/memory";
+import { MockLLM } from "@llamaindex/core/utils";
+import { describe, expect, test } from "vitest";
+
+describe("SimpleChatEngine", () => {
+  test("constructor initializes with provided LLM", () => {
+    const llm = new MockLLM();
+    const engine = new SimpleChatEngine({ llm });
+    expect(engine.llm).toBe(llm);
+    expect(engine.memory).toBeInstanceOf(ChatMemoryBuffer);
+    expect((engine.memory as ChatMemoryBuffer).tokenLimit).toBe(768);
+  });
+});

packages/experimental/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @llamaindex/experimental
 
+## 0.0.165
+
+### Patch Changes
+
+- llamaindex@0.9.15
+
+## 0.0.164
+
+### Patch Changes
+
+- Updated dependencies [9d951b2]
+  - llamaindex@0.9.14
+
+## 0.0.163
+
+### Patch Changes
+
+- Updated dependencies [75d6e29]
+  - llamaindex@0.9.13
+
 ## 0.0.162
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.162",
+  "version": "0.0.165",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,36 @@
 # llamaindex
 
+## 0.9.15
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+- Updated dependencies [c515a32]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+  - @llamaindex/workflow@1.0.2
+  - @llamaindex/cloud@4.0.2
+  - @llamaindex/node-parser@2.0.2
+
+## 0.9.14
+
+### Patch Changes
+
+- 9d951b2: feat: support llamacloud in @llamaindex/server
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+  - @llamaindex/cloud@4.0.1
+  - @llamaindex/node-parser@2.0.1
+  - @llamaindex/openai@0.2.1
+  - @llamaindex/workflow@1.0.1
+
+## 0.9.13
+
+### Patch Changes
+
+- 75d6e29: feat: response source nodes in query tool output
+
 ## 0.9.12
 
 ### Patch Changes

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.9.12",
+  "version": "0.9.15",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/cloud/LlamaCloudIndex.ts

@@ -25,7 +25,9 @@ import {
 } from "@llamaindex/cloud/api";
 import type { BaseRetriever } from "@llamaindex/core/retriever";
 import { getEnv } from "@llamaindex/env";
+import type { QueryToolParams } from "../indices/BaseIndex.js";
 import { Settings } from "../Settings.js";
+import { QueryEngineTool } from "../tools/QueryEngineTool.js";
 
 export class LlamaCloudIndex {
   params: CloudConstructorParams;
@@ -272,6 +274,22 @@ export class LlamaCloudIndex {
     );
   }
 
+  asQueryTool(params: QueryToolParams): QueryEngineTool {
+    if (params.options) {
+      params.retriever = this.asRetriever(params.options);
+    }
+
+    return new QueryEngineTool({
+      queryEngine: this.asQueryEngine(params),
+      metadata: params?.metadata,
+      includeSourceNodes: params?.includeSourceNodes ?? false,
+    });
+  }
+
+  queryTool(params: QueryToolParams): QueryEngineTool {
+    return this.asQueryTool(params);
+  }
+
   async insert(document: Document) {
     const pipelineId = await this.getPipelineId();
 

packages/llamaindex/src/indices/BaseIndex.ts

@@ -41,6 +41,7 @@ export type QueryToolParams = (
 ) & {
   responseSynthesizer?: BaseSynthesizer;
   metadata?: ToolMetadata<JSONSchemaType<QueryEngineParam>> | undefined;
+  includeSourceNodes?: boolean;
 };
 
 /**
@@ -98,6 +99,7 @@ export abstract class BaseIndex<T> {
     return new QueryEngineTool({
       queryEngine: this.asQueryEngine(params),
       metadata: params?.metadata,
+      includeSourceNodes: params?.includeSourceNodes ?? false,
     });
   }
 

packages/llamaindex/src/tools/QueryEngineTool.ts

@@ -1,3 +1,4 @@
+import type { JSONValue } from "@llamaindex/core/global";
 import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
 import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
 import type { JSONSchemaType } from "ajv";
@@ -20,6 +21,7 @@ const DEFAULT_PARAMETERS: JSONSchemaType<QueryEngineParam> = {
 export type QueryEngineToolParams = {
   queryEngine: BaseQueryEngine;
   metadata?: ToolMetadata<JSONSchemaType<QueryEngineParam>> | undefined;
+  includeSourceNodes?: boolean;
 };
 
 export type QueryEngineParam = {
@@ -29,19 +31,32 @@ export type QueryEngineParam = {
 export class QueryEngineTool implements BaseTool<QueryEngineParam> {
   private queryEngine: BaseQueryEngine;
   metadata: ToolMetadata<JSONSchemaType<QueryEngineParam>>;
+  includeSourceNodes: boolean;
 
-  constructor({ queryEngine, metadata }: QueryEngineToolParams) {
+  constructor({
+    queryEngine,
+    metadata,
+    includeSourceNodes,
+  }: QueryEngineToolParams) {
     this.queryEngine = queryEngine;
     this.metadata = {
       name: metadata?.name ?? DEFAULT_NAME,
       description: metadata?.description ?? DEFAULT_DESCRIPTION,
       parameters: metadata?.parameters ?? DEFAULT_PARAMETERS,
     };
+    this.includeSourceNodes = includeSourceNodes ?? false;
   }
 
   async call({ query }: QueryEngineParam) {
     const response = await this.queryEngine.query({ query });
 
-    return response.message.content;
+    if (!this.includeSourceNodes) {
+      return { content: response.message.content };
+    }
+
+    return {
+      content: response.message.content,
+      sourceNodes: response.sourceNodes,
+    } as unknown as JSONValue;
   }
 }

packages/node-parser/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/node-parser
 
+## 2.0.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 2.0.1
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 2.0.0
 
 ### Patch Changes

packages/node-parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/node-parser",
-  "version": "2.0.0",
+  "version": "2.0.2",
   "description": "Node parser for LlamaIndex",
   "type": "module",
   "exports": {

packages/providers/anthropic/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/anthropic
 
+## 0.3.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.3.1
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 0.3.0
 
 ### Minor Changes

packages/providers/anthropic/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/anthropic",
   "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/clip/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/clip
 
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
+## 0.0.47
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+  - @llamaindex/openai@0.2.1
+
 ## 0.0.46
 
 ### Patch Changes

packages/providers/clip/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/clip",
   "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.46",
+  "version": "0.0.48",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/cohere/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/cohere
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 0.0.14
 
 ### Patch Changes

packages/providers/cohere/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/cohere",
   "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.14",
+  "version": "0.0.16",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepinfra/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/deepinfra
 
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
+## 0.0.47
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+  - @llamaindex/openai@0.2.1
+
 ## 0.0.46
 
 ### Patch Changes

packages/providers/deepinfra/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepinfra",
   "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.46",
+  "version": "0.0.48",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepseek/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/deepseek
 
+## 0.0.8
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+
+## 0.0.7
+
+### Patch Changes
+
+- @llamaindex/openai@0.2.1
+
 ## 0.0.6
 
 ### Patch Changes

packages/providers/deepseek/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepseek",
   "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/fireworks/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/fireworks
 
+## 0.0.8
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+
+## 0.0.7
+
+### Patch Changes
+
+- @llamaindex/openai@0.2.1
+
 ## 0.0.6
 
 ### Patch Changes

packages/providers/fireworks/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/fireworks",
   "description": "Fireworks Adapter for LlamaIndex",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/google/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @llamaindex/google
 
+## 0.2.2
+
+### Patch Changes
+
+- 9c63f3f: Add support for openai responses api
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.2.1
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 0.2.0
 
 ### Minor Changes

packages/providers/google/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/google",
   "description": "Google Adapter for LlamaIndex",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/google/src/utils.ts

@@ -181,7 +181,7 @@ export const mapBaseToolToGeminiFunctionDeclaration = (
 export class GeminiHelper {
   // Gemini only has user and model roles. Put the rest in user role.
   public static readonly ROLES_TO_GEMINI: Record<
-    MessageType,
+    Exclude<MessageType, "developer">,
     GeminiMessageRole
   > = {
     user: "user",
@@ -285,7 +285,9 @@ export class GeminiHelper {
     if (message.options && "toolResult" in message.options) {
       return "function";
     }
-    return GeminiHelper.ROLES_TO_GEMINI[message.role];
+    return GeminiHelper.ROLES_TO_GEMINI[
+      message.role as Exclude<MessageType, "developer">
+    ];
   }
 
   public static chatMessageToGemini(

packages/providers/groq/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/groq
 
+## 0.0.63
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+
+## 0.0.62
+
+### Patch Changes
+
+- @llamaindex/openai@0.2.1
+
 ## 0.0.61
 
 ### Patch Changes

packages/providers/groq/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/groq",
   "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.61",
+  "version": "0.0.63",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/huggingface/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/huggingface
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
+## 0.1.1
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+  - @llamaindex/openai@0.2.1
+
 ## 0.1.0
 
 ### Minor Changes

packages/providers/huggingface/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/huggingface",
   "description": "Huggingface Adapter for LlamaIndex",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/jinaai/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/jinaai
 
+## 0.0.8
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
+## 0.0.7
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+  - @llamaindex/openai@0.2.1
+
 ## 0.0.6
 
 ### Patch Changes

packages/providers/jinaai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/jinaai",
   "description": "JinaAI Adapter for LlamaIndex",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/mistral/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/mistral
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.1.1
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 0.1.0
 
 ### Minor Changes

packages/providers/mistral/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/mistral",
   "description": "Mistral Adapter for LlamaIndex",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/mixedbread/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/mixedbread
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 0.0.14
 
 ### Patch Changes

packages/providers/mixedbread/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/mixedbread",
   "description": "Mixedbread Adapter for LlamaIndex",
-  "version": "0.0.14",
+  "version": "0.0.16",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/ollama/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/ollama
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.1.1
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 0.1.0
 
 ### Minor Changes

packages/providers/ollama/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/ollama",
   "description": "Ollama Adapter for LlamaIndex",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/openai/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/openai
 
+## 0.3.0
+
+### Minor Changes
+
+- 9c63f3f: Add support for openai responses api
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.2.1
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 0.2.0
 
 ### Minor Changes

packages/providers/openai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/openai",
   "description": "OpenAI Adapter for LlamaIndex",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -27,7 +27,8 @@
   },
   "scripts": {
     "build": "bunchee",
-    "dev": "bunchee --watch"
+    "dev": "bunchee --watch",
+    "test": "vitest"
   },
   "devDependencies": {
     "bunchee": "6.4.0"
@@ -35,7 +36,7 @@
   "dependencies": {
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*",
-    "openai": "^4.86.0",
+    "openai": "^4.90.0",
     "zod": "^3.24.2"
   }
 }

packages/providers/openai/src/agent.ts

@@ -5,7 +5,8 @@ import {
 } from "@llamaindex/core/agent";
 import { Settings } from "@llamaindex/core/global";
 import type { ToolCallLLMMessageOptions } from "@llamaindex/core/llms";
-import { OpenAI, type OpenAIAdditionalChatOptions } from "./llm";
+import { OpenAI } from "./llm";
+import type { OpenAIAdditionalChatOptions } from "./utils";
 
 // This is likely not necessary anymore but leaving it here just in case it's in use elsewhere
 

packages/providers/openai/src/index.ts

@@ -1,3 +1,4 @@
 export * from "./agent";
 export * from "./embedding";
 export * from "./llm";
+export * from "./responses";

packages/providers/openai/src/llm.ts

@@ -4,7 +4,6 @@ import {
   type ChatMessage,
   type ChatResponse,
   type ChatResponseChunk,
-  type LLM,
   type LLMChatParamsNonStreaming,
   type LLMChatParamsStreaming,
   type LLMMetadata,
@@ -18,7 +17,6 @@ import { getEnv } from "@llamaindex/env";
 import { Tokenizers } from "@llamaindex/env/tokenizers";
 import type {
   AzureClientOptions,
-  AzureOpenAI as AzureOpenAILLM,
   ClientOptions as OpenAIClientOptions,
   OpenAI as OpenAILLM,
 } from "openai";
@@ -45,153 +43,15 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure.js";
-
-export const GPT4_MODELS = {
-  "chatgpt-4o-latest": {
-    contextWindow: 128000,
-  },
-  "gpt-4.5-preview": { contextWindow: 128000 },
-  "gpt-4.5-preview-2025-02-27": { contextWindow: 128000 },
-  "gpt-4": { contextWindow: 8192 },
-  "gpt-4-32k": { contextWindow: 32768 },
-  "gpt-4-32k-0613": { contextWindow: 32768 },
-  "gpt-4-turbo": { contextWindow: 128000 },
-  "gpt-4-turbo-preview": { contextWindow: 128000 },
-  "gpt-4-1106-preview": { contextWindow: 128000 },
-  "gpt-4-0125-preview": { contextWindow: 128000 },
-  "gpt-4-vision-preview": { contextWindow: 128000 },
-  "gpt-4o": { contextWindow: 128000 },
-  "gpt-4o-2024-05-13": { contextWindow: 128000 },
-  "gpt-4o-mini": { contextWindow: 128000 },
-  "gpt-4o-mini-2024-07-18": { contextWindow: 128000 },
-  "gpt-4o-2024-08-06": { contextWindow: 128000 },
-  "gpt-4o-2024-09-14": { contextWindow: 128000 },
-  "gpt-4o-2024-10-14": { contextWindow: 128000 },
-  "gpt-4-0613": { contextWindow: 128000 },
-  "gpt-4-turbo-2024-04-09": { contextWindow: 128000 },
-  "gpt-4-0314": { contextWindow: 128000 },
-  "gpt-4-32k-0314": { contextWindow: 32768 },
-  "gpt-4o-realtime-preview": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-realtime-preview-2024-10-01": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-audio-preview": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-audio-preview-2024-10-01": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-2024-11-20": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-audio-preview-2024-12-17": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-mini-audio-preview": {
-    contextWindow: 128000,
-  },
-  "gpt-4o-mini-audio-preview-2024-12-17": {
-    contextWindow: 128000,
-  },
-};
-
-// NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
-export const GPT35_MODELS = {
-  "gpt-3.5-turbo": { contextWindow: 16385 },
-  "gpt-3.5-turbo-0613": { contextWindow: 4096 },
-  "gpt-3.5-turbo-16k": { contextWindow: 16385 },
-  "gpt-3.5-turbo-16k-0613": { contextWindow: 16385 },
-  "gpt-3.5-turbo-1106": { contextWindow: 16385 },
-  "gpt-3.5-turbo-0125": { contextWindow: 16385 },
-  "gpt-3.5-turbo-0301": { contextWindow: 16385 },
-};
-
-export const O1_MODELS = {
-  "o1-preview": {
-    contextWindow: 128000,
-  },
-  "o1-preview-2024-09-12": {
-    contextWindow: 128000,
-  },
-  "o1-mini": {
-    contextWindow: 128000,
-  },
-  "o1-mini-2024-09-12": {
-    contextWindow: 128000,
-  },
-  o1: {
-    contextWindow: 128000,
-  },
-  "o1-2024-12-17": {
-    contextWindow: 128000,
-  },
-};
-
-export const O3_MODELS = {
-  "o3-mini": {
-    contextWindow: 200000,
-  },
-  "o3-mini-2025-01-31": {
-    contextWindow: 200000,
-  },
-};
-
-/**
- * We currently support GPT-3.5 and GPT-4 models
- */
-export const ALL_AVAILABLE_OPENAI_MODELS = {
-  ...GPT4_MODELS,
-  ...GPT35_MODELS,
-  ...O1_MODELS,
-  ...O3_MODELS,
-} satisfies Record<ChatModel, { contextWindow: number }>;
-
-function isFunctionCallingModel(llm: LLM): llm is OpenAI {
-  let model: string;
-  if (llm instanceof OpenAI) {
-    model = llm.model;
-  } else if ("model" in llm && typeof llm.model === "string") {
-    model = llm.model;
-  } else {
-    return false;
-  }
-  const isChatModel = Object.keys(ALL_AVAILABLE_OPENAI_MODELS).includes(model);
-  const isOld = model.includes("0314") || model.includes("0301");
-  const isO1 = model.startsWith("o1");
-  return isChatModel && !isOld && !isO1;
-}
-
-function isReasoningModel(model: ChatModel | string): boolean {
-  const isO1 = model.startsWith("o1");
-  const isO3 = model.startsWith("o3");
-  return isO1 || isO3;
-}
-
-function isTemperatureSupported(model: ChatModel | string): boolean {
-  return !model.startsWith("o3");
-}
-
-export type OpenAIAdditionalMetadata = object;
-
-export type OpenAIAdditionalChatOptions = Omit<
-  Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
-  | "max_tokens"
-  | "messages"
-  | "model"
-  | "temperature"
-  | "reasoning_effort"
-  | "top_p"
-  | "stream"
-  | "tools"
-  | "toolChoice"
->;
-
-type LLMInstance = Pick<
-  AzureOpenAILLM | OpenAILLM,
-  "chat" | "apiKey" | "baseURL"
->;
+import {
+  ALL_AVAILABLE_OPENAI_MODELS,
+  isFunctionCallingModel,
+  isReasoningModel,
+  isTemperatureSupported,
+  type LLMInstance,
+  type OpenAIAdditionalChatOptions,
+  type OpenAIAdditionalMetadata,
+} from "./utils.js";
 
 export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
   model:
@@ -230,6 +90,7 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
     },
   ) {
     super();
+
     this.model = init?.model ?? "gpt-4o";
     this.temperature = init?.temperature ?? 0.1;
     this.reasoningEffort = isReasoningModel(this.model)

packages/providers/openai/src/responses.ts

@@ -0,0 +1,774 @@
+import {
+  ToolCallLLM,
+  type BaseTool,
+  type ChatMessage,
+  type ChatResponse,
+  type ChatResponseChunk,
+  type LLMChatParamsNonStreaming,
+  type LLMChatParamsStreaming,
+  type LLMMetadata,
+  type MessageContent,
+  type MessageType,
+  type PartialToolCall,
+  type ToolCallLLMMessageOptions,
+  type ToolCallOptions,
+  type ToolResultOptions,
+} from "@llamaindex/core/llms";
+import type { StoredValue } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
+import { getEnv } from "@llamaindex/env";
+import {
+  OpenAI as OpenAILLM,
+  type AzureClientOptions,
+  type ClientOptions as OpenAIClientOptions,
+} from "openai";
+
+import { wrapEventCaller } from "@llamaindex/core/decorator";
+import { Tokenizers } from "@llamaindex/env/tokenizers";
+import {
+  AzureOpenAIWithUserAgent,
+  getAzureConfigFromEnv,
+  getAzureModel,
+  shouldUseAzure,
+} from "./azure";
+import {
+  ALL_AVAILABLE_OPENAI_MODELS,
+  isFunctionCallingModel,
+  isReasoningModel,
+  isTemperatureSupported,
+  type LLMInstance,
+  type OpenAIAdditionalMetadata,
+  type OpenAIResponsesChatOptions,
+  type OpenAIResponsesRole,
+  type ResponseMessageContent,
+  type ResponsesAdditionalOptions,
+  type StreamState,
+} from "./utils";
+
+export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxOutputTokens?: number | undefined;
+  additionalChatOptions?: OpenAIResponsesChatOptions | undefined;
+  reasoningEffort?: "low" | "medium" | "high" | undefined;
+  apiKey?: string | undefined;
+  baseURL?: string | undefined;
+  maxRetries: number;
+  timeout?: number;
+  additionalSessionOptions?:
+    | undefined
+    | Omit<Partial<OpenAIClientOptions>, "apiKey" | "maxRetries" | "timeout">;
+  lazySession: () => Promise<LLMInstance>;
+  #session: Promise<LLMInstance> | null = null;
+
+  trackPreviousResponses: boolean;
+  store: boolean;
+  user: string;
+  callMetadata: StoredValue;
+  builtInTools: OpenAILLM.Responses.Tool[] | null;
+  strict: boolean;
+  include: OpenAILLM.Responses.ResponseIncludable[] | null;
+  instructions: string;
+  previousResponseId: string | null;
+  truncation: "auto" | "disabled" | null;
+
+  constructor(
+    init?: Omit<Partial<OpenAIResponses>, "session"> & {
+      session?: LLMInstance | undefined;
+      azure?: AzureClientOptions;
+    },
+  ) {
+    super();
+    this.model = init?.model ?? "gpt-4o";
+    this.temperature = init?.temperature ?? 0.1;
+
+    this.reasoningEffort = isReasoningModel(this.model)
+      ? init?.reasoningEffort
+      : undefined;
+
+    this.topP = init?.topP ?? 1;
+    this.maxOutputTokens = init?.maxOutputTokens ?? undefined;
+    this.maxRetries = init?.maxRetries ?? 10;
+
+    this.timeout = init?.timeout ?? 60 * 1000;
+
+    this.apiKey =
+      init?.session?.apiKey ?? init?.apiKey ?? getEnv("OPENAI_API_KEY");
+    this.baseURL =
+      init?.session?.baseURL ?? init?.baseURL ?? getEnv("OPENAI_BASE_URL");
+
+    this.additionalSessionOptions = init?.additionalSessionOptions;
+    this.additionalChatOptions = init?.additionalChatOptions;
+
+    this.trackPreviousResponses = init?.trackPreviousResponses ?? false;
+    this.builtInTools = init?.builtInTools ?? null;
+    this.store = init?.store ?? false;
+    this.user = init?.user ?? "";
+    this.callMetadata = init?.callMetadata ?? {};
+    this.strict = init?.strict ?? false;
+    this.include = init?.include ?? null;
+    this.instructions = init?.instructions ?? "";
+    this.previousResponseId = init?.previousResponseId ?? null;
+    this.truncation = init?.truncation ?? null;
+
+    if (init?.azure || shouldUseAzure()) {
+      const azureConfig = {
+        ...getAzureConfigFromEnv({
+          model: getAzureModel(this.model),
+        }),
+        ...init?.azure,
+      };
+
+      this.lazySession = async () =>
+        init?.session ??
+        import("openai").then(({ AzureOpenAI }) => {
+          AzureOpenAI = AzureOpenAIWithUserAgent(AzureOpenAI);
+
+          return new AzureOpenAI({
+            maxRetries: this.maxRetries,
+            timeout: this.timeout!,
+            ...this.additionalSessionOptions,
+            ...azureConfig,
+          });
+        });
+    } else {
+      this.lazySession = async () =>
+        init?.session ??
+        import("openai").then(({ OpenAI }) => {
+          return new OpenAI({
+            apiKey: this.apiKey,
+            baseURL: this.baseURL,
+            maxRetries: this.maxRetries,
+            timeout: this.timeout!,
+            ...this.additionalSessionOptions,
+          });
+        });
+    }
+  }
+
+  get session() {
+    if (!this.#session) {
+      this.#session = this.lazySession();
+    }
+    return this.#session;
+  }
+
+  get supportToolCall() {
+    return isFunctionCallingModel(this);
+  }
+
+  get metadata(): LLMMetadata & OpenAIAdditionalMetadata {
+    const contextWindow =
+      ALL_AVAILABLE_OPENAI_MODELS[
+        this.model as keyof typeof ALL_AVAILABLE_OPENAI_MODELS
+      ]?.contextWindow ?? 1024;
+
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxOutputTokens,
+      contextWindow,
+      tokenizer: Tokenizers.CL100K_BASE,
+      structuredOutput: true,
+    };
+  }
+
+  private createInitialMessage(): ChatMessage<ToolCallLLMMessageOptions> {
+    return {
+      role: "assistant",
+      content: "",
+    };
+  }
+
+  private createInitialOptions(): ResponsesAdditionalOptions {
+    return {
+      built_in_tool_calls: [],
+    };
+  }
+
+  private isBuiltInToolCall(
+    item: OpenAILLM.Responses.ResponseOutputItem,
+  ): item is
+    | OpenAILLM.Responses.ResponseFileSearchToolCall
+    | OpenAILLM.Responses.ResponseComputerToolCall
+    | OpenAILLM.Responses.ResponseFunctionWebSearch {
+    return ["file_search_call", "computer_call", "web_search_call"].includes(
+      item.type,
+    );
+  }
+
+  private isReasoning(item: OpenAILLM.Responses.ResponseOutputItem) {
+    return item.type === "reasoning";
+  }
+
+  private isMessageBlock(item: OpenAILLM.Responses.ResponseOutputItem) {
+    return item.type === "message";
+  }
+
+  private isFunctionCall(item: OpenAILLM.Responses.ResponseOutputItem) {
+    return item.type === "function_call";
+  }
+
+  private isResponseCreatedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseCreatedEvent {
+    return event.type === "response.created";
+  }
+
+  private isResponseOutputItemAddedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseOutputItemAddedEvent {
+    return event.type === "response.output_item.added";
+  }
+
+  private isToolCallEvent(
+    event: OpenAILLM.Responses.ResponseOutputItemAddedEvent,
+  ): event is OpenAILLM.Responses.ResponseOutputItemAddedEvent & {
+    item: OpenAILLM.Responses.ResponseFunctionToolCall;
+  } {
+    return event.item.type === "function_call";
+  }
+
+  private isResponseTextDeltaEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseTextDeltaEvent {
+    return event.type === "response.output_text.delta";
+  }
+
+  private isResponseFunctionCallArgumentsDeltaEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseFunctionCallArgumentsDeltaEvent {
+    return event.type === "response.function_call_arguments.delta";
+  }
+
+  private isResponseFunctionCallDoneEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseFunctionCallArgumentsDoneEvent {
+    return event.type === "response.function_call_arguments.done";
+  }
+
+  private isResponseOutputTextAnnotationAddedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseTextAnnotationDeltaEvent {
+    return event.type === "response.output_text.annotation.added";
+  }
+
+  private isResponseFileSearchCallCompletedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseFileSearchCallCompletedEvent {
+    return event.type === "response.file_search_call.completed";
+  }
+
+  private isResponseWebSearchCallCompletedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseWebSearchCallCompletedEvent {
+    return event.type === "response.web_search_call.completed";
+  }
+
+  private isResponseCompletedEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+  ): event is OpenAILLM.Responses.ResponseCompletedEvent {
+    return event.type === "response.completed";
+  }
+
+  private isTextPresent(
+    part:
+      | OpenAILLM.Responses.ResponseOutputText
+      | OpenAILLM.Responses.ResponseOutputRefusal,
+  ) {
+    return "text" in part;
+  }
+
+  private isRefusalPresent(
+    part:
+      | OpenAILLM.Responses.ResponseOutputText
+      | OpenAILLM.Responses.ResponseOutputRefusal,
+  ) {
+    return "refusal" in part;
+  }
+
+  private isAnnotationPresent(
+    part:
+      | OpenAILLM.Responses.ResponseOutputText
+      | OpenAILLM.Responses.ResponseOutputRefusal,
+  ) {
+    return "annotations" in part;
+  }
+
+  private handleResponseOutputMessage(
+    item: OpenAILLM.Responses.ResponseOutputMessage,
+    options: ResponsesAdditionalOptions,
+  ): string {
+    let outputContent = "";
+    for (const part of item.content) {
+      if (this.isTextPresent(part)) {
+        outputContent += part.text;
+      }
+      if (this.isAnnotationPresent(part)) {
+        options.annotations = part.annotations;
+      }
+      if (this.isRefusalPresent(part)) {
+        options.refusal = part.refusal;
+      }
+    }
+    return outputContent;
+  }
+
+  private extractToolCalls(
+    response: OpenAILLM.Responses.ResponseOutputItem[],
+  ): PartialToolCall[] {
+    return response
+      .filter((item): item is OpenAILLM.Responses.ResponseFunctionToolCall =>
+        this.isFunctionCall(item),
+      )
+      .map((item) => {
+        return {
+          name: item.name,
+          id: item.call_id,
+          input: item.arguments,
+        };
+      });
+  }
+
+  private parseResponseOutput(
+    response: OpenAILLM.Responses.ResponseOutputItem[],
+  ): ChatMessage<ToolCallLLMMessageOptions> {
+    const message = this.createInitialMessage();
+    const options = this.createInitialOptions();
+    const toolCall = this.extractToolCalls(response);
+
+    for (const item of response) {
+      if (this.isMessageBlock(item)) {
+        const outputContent = this.handleResponseOutputMessage(item, options);
+        message.content = outputContent;
+      } else if (this.isBuiltInToolCall(item)) {
+        options.built_in_tool_calls.push(item);
+      } else if (this.isReasoning(item)) {
+        options.reasoning = item;
+      }
+    }
+    message.options = {
+      ...options,
+      toolCall: toolCall,
+    };
+
+    return message;
+  }
+
+  private processStreamEvent(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+    streamState: StreamState,
+  ) {
+    switch (true) {
+      case this.isResponseCreatedEvent(event):
+        this.handleResponseCreatedEvent(event, streamState);
+        break;
+      case this.isResponseOutputItemAddedEvent(event):
+        this.handleOutputItemAddedEvent(event, streamState);
+        break;
+      case this.isResponseTextDeltaEvent(event):
+        this.handleTextDeltaEvent(event, streamState);
+        break;
+      case this.isResponseFunctionCallArgumentsDeltaEvent(event):
+        this.handleFunctionCallArgumentsDeltaEvent(event, streamState);
+        break;
+      case this.isResponseFunctionCallDoneEvent(event):
+        this.handleFunctionCallArgumentsDoneEvent(event, streamState);
+        break;
+      case this.isResponseOutputTextAnnotationAddedEvent(event):
+        this.handleOutputTextAnnotationAddedEvent(event, streamState);
+        break;
+      case this.isResponseFileSearchCallCompletedEvent(event):
+      case this.isResponseWebSearchCallCompletedEvent(event):
+        this.handleBuiltInToolCallCompletedEvent(event, streamState);
+        break;
+      case this.isResponseCompletedEvent(event):
+        this.handleCompletedEvent(event, streamState);
+        break;
+    }
+  }
+
+  private handleResponseCreatedEvent(
+    event: OpenAILLM.Responses.ResponseCreatedEvent,
+    streamState: StreamState,
+  ) {
+    if (this.trackPreviousResponses) {
+      streamState.previousResponseId = event.response.id;
+    }
+  }
+
+  private handleOutputItemAddedEvent(
+    event: OpenAILLM.Responses.ResponseOutputItemAddedEvent,
+    streamState: StreamState,
+  ) {
+    if (this.isToolCallEvent(event)) {
+      streamState.currentToolCall = {
+        name: event.item.name,
+        id: event.item.call_id,
+        input: event.item.arguments,
+      };
+    }
+  }
+
+  private handleTextDeltaEvent(
+    event: OpenAILLM.Responses.ResponseTextDeltaEvent,
+    streamState: StreamState,
+  ) {
+    streamState.delta = event.delta;
+  }
+
+  private handleFunctionCallArgumentsDeltaEvent(
+    event: OpenAILLM.Responses.ResponseFunctionCallArgumentsDeltaEvent,
+    streamState: StreamState,
+  ) {
+    if (streamState.currentToolCall) {
+      streamState.currentToolCall!.input += event.delta;
+    }
+  }
+
+  private handleFunctionCallArgumentsDoneEvent(
+    event: OpenAILLM.Responses.ResponseFunctionCallArgumentsDoneEvent,
+    streamState: StreamState,
+  ) {
+    if (streamState.currentToolCall) {
+      streamState.currentToolCall.input = event.arguments;
+    }
+
+    streamState.shouldEmitToolCall = {
+      ...streamState.currentToolCall!,
+      input: JSON.parse(streamState.currentToolCall!.input),
+    };
+  }
+
+  private handleOutputTextAnnotationAddedEvent(
+    event: OpenAILLM.Responses.ResponseTextAnnotationDeltaEvent,
+    streamState: StreamState,
+  ) {
+    if (!streamState.options.annotations) {
+      streamState.options.annotations = [];
+    }
+    streamState.options.annotations.push(event.annotation);
+  }
+
+  private handleBuiltInToolCallCompletedEvent(
+    event:
+      | OpenAILLM.Responses.ResponseFileSearchCallCompletedEvent
+      | OpenAILLM.Responses.ResponseWebSearchCallCompletedEvent,
+    streamState: StreamState,
+  ) {
+    streamState.options.built_in_tool_calls.push(event);
+  }
+
+  private handleCompletedEvent(
+    event: OpenAILLM.Responses.ResponseCompletedEvent,
+    streamState: StreamState,
+  ) {
+    if (event.response.usage) {
+      streamState.options.usage = event.response.usage;
+    }
+  }
+
+  private createBaseRequestParams(
+    messages: ChatMessage<ToolCallLLMMessageOptions>[],
+    tools: BaseTool[] | undefined,
+    additionalChatOptions: OpenAIResponsesChatOptions | undefined,
+  ) {
+    const baseRequestParams = <OpenAILLM.Responses.ResponseCreateParams>{
+      model: this.model,
+      include: this.include,
+      input: this.toOpenAIResponseMessages(messages),
+      tools: this.builtInTools ? [...this.builtInTools] : [],
+      instructions: this.instructions,
+      max_output_tokens: this.maxOutputTokens,
+      previous_response_id: this.previousResponseId,
+      store: this.store,
+      metadata: this.callMetadata,
+      top_p: this.topP,
+      truncation: this.truncation,
+      user: this.user,
+      ...Object.assign({}, this.additionalChatOptions, additionalChatOptions),
+    };
+
+    if (tools?.length) {
+      if (!baseRequestParams.tools) {
+        baseRequestParams.tools = [];
+      }
+      baseRequestParams.tools.push(
+        ...tools.map(this.toResponsesTool.bind(this)),
+      );
+    }
+
+    return baseRequestParams;
+  }
+
+  chat(
+    params: LLMChatParamsStreaming<
+      OpenAIResponsesChatOptions,
+      ToolCallLLMMessageOptions
+    >,
+  ): Promise<AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>>;
+  chat(
+    params: LLMChatParamsNonStreaming<
+      OpenAIResponsesChatOptions,
+      ToolCallLLMMessageOptions
+    >,
+  ): Promise<ChatResponse<ToolCallLLMMessageOptions>>;
+  async chat(
+    params:
+      | LLMChatParamsNonStreaming<
+          OpenAIResponsesChatOptions,
+          ToolCallLLMMessageOptions
+        >
+      | LLMChatParamsStreaming<
+          OpenAIResponsesChatOptions,
+          ToolCallLLMMessageOptions
+        >,
+  ): Promise<
+    | ChatResponse<ToolCallLLMMessageOptions>
+    | AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>
+  > {
+    const { messages, stream, tools, responseFormat, additionalChatOptions } =
+      params;
+    const baseRequestParams = this.createBaseRequestParams(
+      messages,
+      tools,
+      additionalChatOptions,
+    );
+
+    if (
+      Array.isArray(baseRequestParams.tools) &&
+      baseRequestParams.tools.length === 0
+    ) {
+      // remove empty tools array to avoid OpenAI error
+      delete baseRequestParams.tools;
+    }
+
+    if (!isTemperatureSupported(baseRequestParams.model))
+      delete baseRequestParams.temperature;
+
+    if (stream) {
+      return this.streamChat(baseRequestParams);
+    }
+
+    const response = await (
+      await this.session
+    ).responses.create({
+      ...baseRequestParams,
+      stream: false,
+    });
+
+    const message = this.parseResponseOutput(response.output);
+    return {
+      raw: response,
+      message,
+    };
+  }
+
+  private initalizeStreamState(): StreamState {
+    return {
+      delta: "",
+      currentToolCall: null,
+      shouldEmitToolCall: null,
+      options: this.createInitialOptions(),
+      previousResponseId: this.previousResponseId,
+    };
+  }
+
+  private createResponseChunk(
+    event: OpenAILLM.Responses.ResponseStreamEvent,
+    state: StreamState,
+  ): ChatResponseChunk<ToolCallLLMMessageOptions> {
+    return {
+      raw: event,
+      delta: state.delta,
+      options: state.shouldEmitToolCall
+        ? { toolCall: [state.shouldEmitToolCall] }
+        : state.currentToolCall
+          ? { toolCall: [state.currentToolCall] }
+          : {},
+    };
+  }
+
+  @wrapEventCaller
+  protected async *streamChat(
+    baseRequestParams: OpenAILLM.Responses.ResponseCreateParams,
+  ): AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>> {
+    const streamState = this.initalizeStreamState();
+
+    const stream = await (
+      await this.session
+    ).responses.create({
+      ...baseRequestParams,
+      stream: true,
+    });
+
+    for await (const event of stream) {
+      this.processStreamEvent(event, streamState);
+      this.handlePreviousResponseId(streamState);
+      yield this.createResponseChunk(event, streamState);
+    }
+  }
+
+  toOpenAIResponsesRole(messageType: MessageType): OpenAIResponsesRole {
+    switch (messageType) {
+      case "user":
+        return "user";
+      case "assistant":
+        return "assistant";
+      case "system":
+        return "system";
+      case "developer":
+        return "developer";
+      default:
+        return "user";
+    }
+  }
+
+  private isToolResultPresent(
+    options: ToolCallLLMMessageOptions,
+  ): options is ToolResultOptions {
+    return "toolResult" in options;
+  }
+
+  private isToolCallPresent(
+    options: ToolCallLLMMessageOptions,
+  ): options is ToolCallOptions {
+    return "toolCall" in options;
+  }
+
+  private isUserMessage(message: ChatMessage<ToolCallLLMMessageOptions>) {
+    return message.role === "user";
+  }
+
+  private handlePreviousResponseId(streamState: StreamState) {
+    if (this.trackPreviousResponses) {
+      if (streamState.previousResponseId != this.previousResponseId) {
+        this.previousResponseId = streamState.previousResponseId;
+      }
+    }
+  }
+
+  private convertToOpenAIToolCallResult(
+    options: ToolResultOptions,
+    content: MessageContent,
+  ) {
+    return {
+      type: "function_call_output",
+      call_id: options.toolResult.id,
+      output: extractText(content),
+    } satisfies OpenAILLM.Responses.ResponseInputItem.FunctionCallOutput;
+  }
+
+  private convertToOpenAIToolCalls(options: ToolCallOptions) {
+    return options.toolCall.map((toolCall) => {
+      return {
+        type: "function_call",
+        call_id: toolCall.id,
+        name: toolCall.name,
+        arguments:
+          typeof toolCall.input === "string"
+            ? toolCall.input
+            : JSON.stringify(toolCall.input),
+      };
+    }) satisfies OpenAILLM.Responses.ResponseFunctionToolCall[];
+  }
+
+  private processMessageContent(
+    content: MessageContent,
+  ): ResponseMessageContent {
+    if (!Array.isArray(content)) {
+      return content;
+    }
+
+    return content.map((item) => {
+      if (item.type === "text") {
+        return {
+          type: "input_text",
+          text: item.text,
+        };
+      }
+      if (item.type === "image_url") {
+        return {
+          type: "input_image",
+          image_url: item.image_url.url,
+          detail: item.detail || "auto",
+        };
+      }
+      throw new Error("Unsupported content type");
+    });
+  }
+
+  private convertToOpenAIUserMessage(
+    message: ChatMessage<ToolCallLLMMessageOptions>,
+  ) {
+    const messageContent = this.processMessageContent(message.content);
+    return {
+      role: "user",
+      content: messageContent,
+    } satisfies OpenAILLM.Responses.EasyInputMessage;
+  }
+
+  private defaultOpenAIResponseMessage(
+    message: ChatMessage<ToolCallLLMMessageOptions>,
+  ) {
+    const response: OpenAILLM.Responses.ResponseInputItem = {
+      role: this.toOpenAIResponsesRole(message.role),
+      content: extractText(message.content),
+    };
+    return response;
+  }
+
+  toOpenAIResponseMessage(
+    message: ChatMessage<ToolCallLLMMessageOptions>,
+  ):
+    | OpenAILLM.Responses.ResponseInputItem
+    | OpenAILLM.Responses.ResponseInputItem[] {
+    const options = message.options ?? {};
+
+    if (this.isToolResultPresent(options)) {
+      return this.convertToOpenAIToolCallResult(options, message.content);
+    } else if (this.isToolCallPresent(options)) {
+      return this.convertToOpenAIToolCalls(options);
+    } else if (this.isUserMessage(message)) {
+      return this.convertToOpenAIUserMessage(message);
+    }
+
+    return this.defaultOpenAIResponseMessage(message);
+  }
+
+  toOpenAIResponseMessages(
+    messages: ChatMessage<ToolCallLLMMessageOptions>[],
+  ): OpenAILLM.Responses.ResponseInput {
+    const finalMessages: OpenAILLM.Responses.ResponseInputItem[] = [];
+    for (const message of messages) {
+      const processedMessage = this.toOpenAIResponseMessage(message);
+      if (Array.isArray(processedMessage)) {
+        finalMessages.push(...processedMessage);
+      } else {
+        finalMessages.push(processedMessage);
+      }
+    }
+    return finalMessages;
+  }
+
+  toResponsesTool(tool: BaseTool): OpenAILLM.Responses.Tool {
+    return {
+      type: "function",
+      name: tool.metadata.name,
+      description: tool.metadata.description,
+      parameters: tool.metadata.parameters ?? {},
+      strict: this.strict,
+    };
+  }
+}
+
+/**
+ * Convenience function to create a new OpenAI instance.
+ * @param init - Optional initialization parameters for the OpenAI instance.
+ * @returns A new OpenAI instance.
+ */
+export const openaiResponses = (
+  init?: ConstructorParameters<typeof OpenAIResponses>[0],
+) => new OpenAIResponses(init);

packages/providers/openai/src/utils.ts

@@ -0,0 +1,213 @@
+import type { LLM, PartialToolCall } from "@llamaindex/core/llms";
+import { AzureOpenAI as AzureOpenAILLM, OpenAI as OpenAILLM } from "openai";
+import type { ChatModel } from "openai/resources.mjs";
+import { OpenAI } from "./llm";
+
+export const GPT4_MODELS = {
+  "chatgpt-4o-latest": {
+    contextWindow: 128000,
+  },
+  "gpt-4.5-preview": { contextWindow: 128000 },
+  "gpt-4.5-preview-2025-02-27": { contextWindow: 128000 },
+  "gpt-4": { contextWindow: 8192 },
+  "gpt-4-32k": { contextWindow: 32768 },
+  "gpt-4-32k-0613": { contextWindow: 32768 },
+  "gpt-4-turbo": { contextWindow: 128000 },
+  "gpt-4-turbo-preview": { contextWindow: 128000 },
+  "gpt-4-1106-preview": { contextWindow: 128000 },
+  "gpt-4-0125-preview": { contextWindow: 128000 },
+  "gpt-4-vision-preview": { contextWindow: 128000 },
+  "gpt-4o": { contextWindow: 128000 },
+  "gpt-4o-2024-05-13": { contextWindow: 128000 },
+  "gpt-4o-mini": { contextWindow: 128000 },
+  "gpt-4o-mini-2024-07-18": { contextWindow: 128000 },
+  "gpt-4o-2024-08-06": { contextWindow: 128000 },
+  "gpt-4o-2024-09-14": { contextWindow: 128000 },
+  "gpt-4o-2024-10-14": { contextWindow: 128000 },
+  "gpt-4-0613": { contextWindow: 128000 },
+  "gpt-4-turbo-2024-04-09": { contextWindow: 128000 },
+  "gpt-4-0314": { contextWindow: 128000 },
+  "gpt-4-32k-0314": { contextWindow: 32768 },
+  "gpt-4o-realtime-preview": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-realtime-preview-2024-10-01": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-audio-preview": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-audio-preview-2024-10-01": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-2024-11-20": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-audio-preview-2024-12-17": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-mini-audio-preview": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-mini-audio-preview-2024-12-17": {
+    contextWindow: 128000,
+  },
+  "gpt-4o-search-preview": { contextWindow: 128000 },
+  "gpt-4o-mini-search-preview": { contextWindow: 128000 },
+  "gpt-4o-search-preview-2025-03-11": { contextWindow: 128000 },
+  "gpt-4o-mini-search-preview-2025-03-11": { contextWindow: 128000 },
+};
+
+// NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
+export const GPT35_MODELS = {
+  "gpt-3.5-turbo": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0613": { contextWindow: 4096 },
+  "gpt-3.5-turbo-16k": { contextWindow: 16385 },
+  "gpt-3.5-turbo-16k-0613": { contextWindow: 16385 },
+  "gpt-3.5-turbo-1106": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0125": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0301": { contextWindow: 16385 },
+};
+
+export const O1_MODELS = {
+  "o1-preview": {
+    contextWindow: 128000,
+  },
+  "o1-preview-2024-09-12": {
+    contextWindow: 128000,
+  },
+  "o1-mini": {
+    contextWindow: 128000,
+  },
+  "o1-mini-2024-09-12": {
+    contextWindow: 128000,
+  },
+  o1: {
+    contextWindow: 128000,
+  },
+  "o1-2024-12-17": {
+    contextWindow: 128000,
+  },
+};
+
+export const O3_MODELS = {
+  "o3-mini": {
+    contextWindow: 200000,
+  },
+  "o3-mini-2025-01-31": {
+    contextWindow: 200000,
+  },
+};
+
+/**
+ * We currently support GPT-3.5 and GPT-4 models
+ */
+export const ALL_AVAILABLE_OPENAI_MODELS = {
+  ...GPT4_MODELS,
+  ...GPT35_MODELS,
+  ...O1_MODELS,
+  ...O3_MODELS,
+} satisfies Record<ChatModel, { contextWindow: number }>;
+
+export function isFunctionCallingModel(llm: LLM): llm is OpenAI {
+  let model: string;
+  if (llm instanceof OpenAI) {
+    model = llm.model;
+  } else if ("model" in llm && typeof llm.model === "string") {
+    model = llm.model;
+  } else {
+    return false;
+  }
+  const isChatModel = Object.keys(ALL_AVAILABLE_OPENAI_MODELS).includes(model);
+  const isOld = model.includes("0314") || model.includes("0301");
+  const isO1 = model.startsWith("o1");
+  return isChatModel && !isOld && !isO1;
+}
+
+export function isReasoningModel(model: ChatModel | string): boolean {
+  const isO1 = model.startsWith("o1");
+  const isO3 = model.startsWith("o3");
+  return isO1 || isO3;
+}
+
+export function isTemperatureSupported(model: ChatModel | string): boolean {
+  return !model.startsWith("o3");
+}
+
+export type OpenAIAdditionalMetadata = object;
+
+export type OpenAIAdditionalChatOptions = Omit<
+  Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
+  | "max_tokens"
+  | "messages"
+  | "model"
+  | "temperature"
+  | "reasoning_effort"
+  | "top_p"
+  | "stream"
+  | "tools"
+  | "toolChoice"
+>;
+
+export type LLMInstance = Pick<
+  AzureOpenAILLM | OpenAILLM,
+  "chat" | "apiKey" | "baseURL" | "responses"
+>;
+
+export type OpenAIResponsesChatOptions = Omit<
+  Partial<OpenAILLM.Responses.ResponseCreateParams>,
+  | "model"
+  | "input"
+  | "stream"
+  | "tools"
+  | "toolChoice"
+  | "temperature"
+  | "reasoning_effort"
+  | "top_p"
+  | "max_output_tokens"
+  | "include"
+>;
+
+export type ResponsesAdditionalOptions = {
+  built_in_tool_calls: Array<
+    | OpenAILLM.Responses.ResponseFileSearchToolCall
+    | OpenAILLM.Responses.ResponseComputerToolCall
+    | OpenAILLM.Responses.ResponseFunctionWebSearch
+    | OpenAILLM.Responses.ResponseFileSearchCallCompletedEvent
+    | OpenAILLM.Responses.ResponseWebSearchCallCompletedEvent
+  >;
+  annotations?: Array<
+    | OpenAILLM.Responses.ResponseOutputText.FileCitation
+    | OpenAILLM.Responses.ResponseOutputText.URLCitation
+    | OpenAILLM.Responses.ResponseOutputText.FilePath
+  >;
+  refusal?: string;
+  reasoning?: OpenAILLM.Responses.ResponseReasoningItem;
+  usage?: OpenAILLM.Responses.ResponseUsage;
+};
+
+export type StreamState = {
+  delta: string;
+  currentToolCall: PartialToolCall | null;
+  shouldEmitToolCall: PartialToolCall | null;
+  options: ResponsesAdditionalOptions;
+  previousResponseId: string | null;
+};
+
+export type ResponsesMessageContentTextDetail = {
+  type: "input_text";
+  text: string;
+};
+
+export type ResponsesMessageContentImageDetail = {
+  type: "input_image";
+  image_url: string;
+  detail: "high" | "low" | "auto";
+};
+export type ResponsesMessageContentDetail =
+  | ResponsesMessageContentTextDetail
+  | ResponsesMessageContentImageDetail;
+
+export type ResponseMessageContent = string | ResponsesMessageContentDetail[];
+
+export type OpenAIResponsesRole = "user" | "assistant" | "system" | "developer";

packages/providers/openai/tests/responses.test.ts

@@ -0,0 +1,223 @@
+import type { BaseTool, ToolCallOptions } from "@llamaindex/core/llms";
+import { describe, expect, it } from "vitest";
+import { OpenAIResponses } from "../src/responses";
+
+const API_KEY = process.env.MY_OPENAI_API_KEY;
+
+describe("OpenAIResponses Integration Tests", () => {
+  if (!API_KEY) {
+    describe.skip("OpenAI API key not found skipping tests");
+    return;
+  }
+
+  const llm = new OpenAIResponses({
+    model: "gpt-4o",
+    apiKey: API_KEY,
+  });
+
+  it("should handle basic text chat", async () => {
+    const response = await llm.chat({
+      messages: [
+        {
+          role: "user",
+          content: "What is 2+2? Answer in one word.",
+        },
+      ],
+    });
+
+    expect(response.message.content).toBe("Four.");
+    expect(response.message.role).toBe("assistant");
+  });
+
+  it("should handle image analysis", async () => {
+    const response = await llm.chat({
+      messages: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "What's in this image? Describe in one sentence.",
+            },
+            {
+              type: "image_url",
+              image_url: {
+                url: "https://storage.googleapis.com/cloud-samples-data/vision/face/faces.jpeg",
+              },
+            },
+          ],
+        },
+      ],
+    });
+    expect(response.raw).toHaveProperty("id");
+  });
+
+  it("should handle function calls", async () => {
+    const weatherTool: BaseTool = {
+      metadata: {
+        name: "get_weather",
+        description: "Get current weather for a location",
+        parameters: {
+          type: "object",
+          properties: {
+            location: {
+              type: "string",
+              description: "City name",
+            },
+          },
+          required: ["location"],
+        },
+      },
+    };
+
+    const response = await llm.chat({
+      messages: [
+        {
+          role: "user",
+          content: "What's the weather in London?",
+        },
+      ],
+      tools: [weatherTool],
+    });
+
+    expect(
+      (response.message.options as ToolCallOptions)?.toolCall,
+    ).toBeDefined();
+    const toolCall = (response.message.options as ToolCallOptions)
+      ?.toolCall?.[0];
+    expect(toolCall?.name).toBe("get_weather");
+    expect(
+      typeof toolCall?.input === "string" &&
+        JSON.parse(toolCall?.input || "{}"),
+    ).toHaveProperty("location", "London");
+  });
+});
+
+describe("OpenAIResponses Unit Tests", () => {
+  // Testing utility functions
+  describe("processMessageContent", () => {
+    const llm = new OpenAIResponses({
+      model: "gpt-4o",
+      apiKey: "test",
+    });
+
+    it("should handle non-array content (string)", () => {
+      const content = "Hello world";
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result).toBe(content);
+    });
+
+    it("should process text content", () => {
+      const content = [
+        {
+          type: "text",
+          text: "Hello world",
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result[0]).toEqual({
+        type: "input_text",
+        text: "Hello world",
+      });
+    });
+
+    it("should process image content with default detail", () => {
+      const content = [
+        {
+          type: "image_url",
+          image_url: { url: "https://example.com/image.jpg" },
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result[0]).toEqual({
+        type: "input_image",
+        image_url: "https://example.com/image.jpg",
+        detail: "auto",
+      });
+    });
+
+    it("should process image content with specified detail", () => {
+      const content = [
+        {
+          type: "image_url",
+          image_url: { url: "https://example.com/image.jpg" },
+          detail: "high",
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result[0]).toEqual({
+        type: "input_image",
+        image_url: "https://example.com/image.jpg",
+        detail: "high",
+      });
+    });
+
+    it("should process mixed content", () => {
+      const content = [
+        {
+          type: "text",
+          text: "What's in this image?",
+        },
+        {
+          type: "image_url",
+          image_url: { url: "https://example.com/image.jpg" },
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result).toEqual([
+        {
+          type: "input_text",
+          text: "What's in this image?",
+        },
+        {
+          type: "input_image",
+          image_url: "https://example.com/image.jpg",
+          detail: "auto",
+        },
+      ]);
+    });
+  });
+
+  describe("isResponseCreatedEvent", () => {
+    const llm = new OpenAIResponses({
+      model: "gpt-4o",
+      apiKey: "test",
+    });
+
+    it("should identify response created events", () => {
+      const event = { type: "response.created", response_id: "123" };
+      // @ts-expect-error accessing private method
+      expect(llm.isResponseCreatedEvent(event)).toBe(true);
+    });
+
+    it("should reject non-created events", () => {
+      const event = { type: "response.other", response_id: "123" };
+      // @ts-expect-error accessing private method
+      expect(llm.isResponseCreatedEvent(event)).toBe(false);
+    });
+  });
+
+  describe("isFunctionCall", () => {
+    const llm = new OpenAIResponses({
+      model: "gpt-4o",
+      apiKey: "test",
+    });
+
+    it("should identify function calls", () => {
+      const item = { type: "function_call", name: "test", arguments: "{}" };
+      // @ts-expect-error accessing private method
+      expect(llm.isFunctionCall(item)).toBe(true);
+    });
+
+    it("should reject non-function calls", () => {
+      const item = { type: "message", content: "test" };
+      // @ts-expect-error accessing private method
+      expect(llm.isFunctionCall(item)).toBe(false);
+    });
+  });
+});

packages/providers/perplexity/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/perplexity
 
+## 0.0.5
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/openai@0.3.0
+  - @llamaindex/core@0.6.2
+
+## 0.0.4
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+  - @llamaindex/openai@0.2.1
+
 ## 0.0.3
 
 ### Patch Changes

packages/providers/perplexity/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/perplexity",
   "description": "Perplexity Adapter for LlamaIndex",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/portkey-ai/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/portkey-ai
 
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.0.43
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 0.0.42
 
 ### Patch Changes

packages/providers/portkey-ai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/portkey-ai",
   "description": "Portkey Adapter for LlamaIndex",
-  "version": "0.0.42",
+  "version": "0.0.44",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/replicate/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/replicate
 
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [9c63f3f]
+  - @llamaindex/core@0.6.2
+
+## 0.0.43
+
+### Patch Changes
+
+- Updated dependencies [1b6f368]
+- Updated dependencies [eaf326e]
+  - @llamaindex/core@0.6.1
+
 ## 0.0.42
 
 ### Patch Changes