Release 0.6.15 (#1295)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

apps/docs/CHANGELOG.md

@@ -1,5 +1,33 @@
 # docs
 
+## 0.0.84
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 0.0.83
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 0.0.82
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 0.0.81
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
 ## 0.0.80
 
 ### Patch Changes

apps/docs/docs/modules/data_loaders/llama_parse/index.mdx

@@ -13,7 +13,7 @@ Official documentation for LlamaParse can be found [here](https://docs.cloud.lla
 ## Usage
 
 You can then use the `LlamaParseReader` class to load local files and convert them into a parsed document that can be used by LlamaIndex.
-See [LlamaParseReader.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/llamaindex/src/readers/LlamaParseReader.ts) for a list of supported file types:
+See [reader.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/cloud/src/reader.ts) for a list of supported file types:
 
 <CodeBlock language="ts">{CodeSource}</CodeBlock>
 

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.80",
+  "version": "0.0.84",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/agent/large_toolcall_with_gpt4o.ts

@@ -13,7 +13,7 @@ import { FunctionTool, OpenAI, ToolCallOptions } from "llamaindex";
   }
 })();
 
-async function callLLM(init: Partial<OpenAI>) {
+async function callLLM(init: { model: string }) {
   const csvData =
     "Country,Average Height (cm)\nNetherlands,156\nDenmark,158\nNorway,160";
 

packages/autotool/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/autotool
 
+## 3.0.15
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 3.0.14
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 3.0.13
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 3.0.12
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
 ## 3.0.11
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.24
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+  - @llamaindex/autotool@3.0.15
+
+## 0.0.23
+
+### Patch Changes
+
+- llamaindex@0.6.14
+- @llamaindex/autotool@3.0.14
+
+## 0.0.22
+
+### Patch Changes
+
+- llamaindex@0.6.13
+- @llamaindex/autotool@3.0.13
+
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+  - @llamaindex/autotool@3.0.12
+
 ## 0.0.20
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.20"
+  "version": "0.0.24"
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.68
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+  - @llamaindex/autotool@3.0.15
+
+## 0.1.67
+
+### Patch Changes
+
+- llamaindex@0.6.14
+- @llamaindex/autotool@3.0.14
+
+## 0.1.66
+
+### Patch Changes
+
+- llamaindex@0.6.13
+- @llamaindex/autotool@3.0.13
+
+## 0.1.65
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+  - @llamaindex/autotool@3.0.12
+
 ## 0.1.64
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.64",
+  "version": "0.1.68",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "3.0.11",
+  "version": "3.0.15",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @llamaindex/cloud
 
+## 0.2.11
+
+### Patch Changes
+
+- 0b20ff9: fix: package.json format
+
+## 0.2.10
+
+### Patch Changes
+
+- 981811e: fix(cloud): llama parse reader save image incorrectly
+
 ## 0.2.9
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.2.9",
+  "version": "0.2.11",
   "type": "module",
   "license": "MIT",
   "scripts": {
@@ -51,13 +51,13 @@
   "devDependencies": {
     "@hey-api/client-fetch": "^0.2.4",
     "@hey-api/openapi-ts": "^0.53.0",
-    "@llamaindex/core": "workspace:^0.2.8",
-    "@llamaindex/env": "workspace:^0.1.13",
+    "@llamaindex/core": "workspace:*",
+    "@llamaindex/env": "workspace:*",
     "bunchee": "5.3.2"
   },
   "peerDependencies": {
-    "@llamaindex/core": "workspace:^0.2.8",
-    "@llamaindex/env": "workspace:^0.1.13"
+    "@llamaindex/core": "workspace:*",
+    "@llamaindex/env": "workspace:*"
   },
   "dependencies": {
     "magic-bytes.js": "^1.10.0"

packages/cloud/src/reader.ts

@@ -1,11 +1,11 @@
-import { createClient, createConfig, type Client } from "@hey-api/client-fetch";
+import { type Client, createClient, createConfig } from "@hey-api/client-fetch";
 import { Document, FileReader } from "@llamaindex/core/schema";
-import { fs, getEnv } from "@llamaindex/env";
+import { fs, getEnv, path } from "@llamaindex/env";
 import { filetypeinfo } from "magic-bytes.js";
 import {
-  ParsingService,
   type Body_upload_file_api_v1_parsing_upload_post,
   type ParserLanguages,
+  ParsingService,
 } from "./api";
 import { sleep } from "./utils";
 
@@ -510,14 +510,7 @@ export class LlamaParseReader extends FileReader {
     jobId: string,
     imageName: string,
   ): Promise<string> {
-    // Get the full path
-    let imagePath = `${downloadPath}/${jobId}-${imageName}`;
-    // Get a valid image path
-    if (!imagePath.endsWith(".png") && !imagePath.endsWith(".jpg")) {
-      imagePath += ".png";
-    }
-
-    return imagePath;
+    return path.join(downloadPath, `${jobId}-${imageName}`);
   }
 
   private async fetchAndSaveImage(
@@ -538,10 +531,9 @@ export class LlamaParseReader extends FileReader {
     if (response.error) {
       throw new Error(`Failed to download image: ${response.error.detail}`);
     }
-    const arrayBuffer = (await response.data) as ArrayBuffer;
-    const buffer = new Uint8Array(arrayBuffer);
+    const blob = (await response.data) as Blob;
     // Write the image buffer to the specified imagePath
-    await fs.writeFile(imagePath, buffer);
+    await fs.writeFile(imagePath, new Uint8Array(await blob.arrayBuffer()));
   }
 
   // Filters out invalid values (null, undefined, empty string) of specific params.

packages/community/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/community
 
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [b17d439]
+  - @llamaindex/core@0.2.9
+
+## 0.0.43
+
+### Patch Changes
+
+- 2774e80: feat: added meta3.2 support via Bedrock including vision, tool call and inference region support
+
 ## 0.0.42
 
 ### Patch Changes

packages/community/README.md

@@ -5,9 +5,11 @@
 ## Current Features:
 
 - Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
-- Bedrock support for the Meta LLama 2, 3 and 3.1 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
-- Meta LLama3.1 405b tool call support
+- Bedrock support for the Meta LLama 2, 3, 3.1 and 3.2 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
+- Meta LLama3.1 405b and Llama3.2 tool call support
+- Meta 3.2 11B and 90B vision support
 - Bedrock support for querying Knowledge Base
+- Bedrock: [Supported Regions and models for cross-region inference](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference-support.html)
 
 ## LICENSE
 

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.42",
+  "version": "0.0.44",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/community/src/index.ts

@@ -2,5 +2,7 @@ export {
   BEDROCK_MODELS,
   BEDROCK_MODEL_MAX_TOKENS,
   Bedrock,
+  INFERENCE_BEDROCK_MODELS,
+  INFERENCE_TO_BEDROCK_MAP,
 } from "./llm/bedrock/index.js";
 export { AmazonKnowledgeBaseRetriever } from "./retrievers/bedrock.js";

packages/community/src/llm/bedrock/anthropic/utils.ts

@@ -6,7 +6,10 @@ import type {
   MessageContentDetail,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
-import { mapMessageContentToMessageContentDetails } from "../utils";
+import {
+  extractDataUrlComponents,
+  mapMessageContentToMessageContentDetails,
+} from "../utils";
 import type {
   AnthropicContent,
   AnthropicImageContent,
@@ -143,27 +146,6 @@ export const mapTextContent = (text: string): AnthropicTextContent => {
   return { type: "text", text };
 };
 
-export const extractDataUrlComponents = (
-  dataUrl: string,
-): {
-  mimeType: string;
-  base64: string;
-} => {
-  const parts = dataUrl.split(";base64,");
-
-  if (parts.length !== 2 || !parts[0]!.startsWith("data:")) {
-    throw new Error("Invalid data URL");
-  }
-
-  const mimeType = parts[0]!.slice(5);
-  const base64 = parts[1]!;
-
-  return {
-    mimeType,
-    base64,
-  };
-};
-
 export const mapImageContent = (imageUrl: string): AnthropicImageContent => {
   if (!imageUrl.startsWith("data:"))
     throw new Error(

packages/community/src/llm/bedrock/index.ts

@@ -47,35 +47,96 @@ export type BedrockChatParamsNonStreaming = LLMChatParamsNonStreaming<
 export type BedrockChatNonStreamResponse =
   ChatResponse<ToolCallLLMMessageOptions>;
 
-export enum BEDROCK_MODELS {
-  AMAZON_TITAN_TG1_LARGE = "amazon.titan-tg1-large",
-  AMAZON_TITAN_TEXT_EXPRESS_V1 = "amazon.titan-text-express-v1",
-  AI21_J2_GRANDE_INSTRUCT = "ai21.j2-grande-instruct",
-  AI21_J2_JUMBO_INSTRUCT = "ai21.j2-jumbo-instruct",
-  AI21_J2_MID = "ai21.j2-mid",
-  AI21_J2_MID_V1 = "ai21.j2-mid-v1",
-  AI21_J2_ULTRA = "ai21.j2-ultra",
-  AI21_J2_ULTRA_V1 = "ai21.j2-ultra-v1",
-  COHERE_COMMAND_TEXT_V14 = "cohere.command-text-v14",
-  ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1",
-  ANTHROPIC_CLAUDE_1 = "anthropic.claude-v1", // EOF: No longer supported
-  ANTHROPIC_CLAUDE_2 = "anthropic.claude-v2",
-  ANTHROPIC_CLAUDE_2_1 = "anthropic.claude-v2:1",
-  ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0",
-  ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0",
-  ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0",
-  ANTHROPIC_CLAUDE_3_5_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0",
-  META_LLAMA2_13B_CHAT = "meta.llama2-13b-chat-v1",
-  META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1",
-  META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0",
-  META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0",
-  META_LLAMA3_1_8B_INSTRUCT = "meta.llama3-1-8b-instruct-v1:0",
-  META_LLAMA3_1_70B_INSTRUCT = "meta.llama3-1-70b-instruct-v1:0",
-  META_LLAMA3_1_405B_INSTRUCT = "meta.llama3-1-405b-instruct-v1:0",
-  MISTRAL_7B_INSTRUCT = "mistral.mistral-7b-instruct-v0:2",
-  MISTRAL_MIXTRAL_7B_INSTRUCT = "mistral.mixtral-8x7b-instruct-v0:1",
-  MISTRAL_MIXTRAL_LARGE_2402 = "mistral.mistral-large-2402-v1:0",
-}
+export const BEDROCK_MODELS = {
+  AMAZON_TITAN_TG1_LARGE: "amazon.titan-tg1-large",
+  AMAZON_TITAN_TEXT_EXPRESS_V1: "amazon.titan-text-express-v1",
+  AI21_J2_GRANDE_INSTRUCT: "ai21.j2-grande-instruct",
+  AI21_J2_JUMBO_INSTRUCT: "ai21.j2-jumbo-instruct",
+  AI21_J2_MID: "ai21.j2-mid",
+  AI21_J2_MID_V1: "ai21.j2-mid-v1",
+  AI21_J2_ULTRA: "ai21.j2-ultra",
+  AI21_J2_ULTRA_V1: "ai21.j2-ultra-v1",
+  COHERE_COMMAND_TEXT_V14: "cohere.command-text-v14",
+  ANTHROPIC_CLAUDE_INSTANT_1: "anthropic.claude-instant-v1",
+  ANTHROPIC_CLAUDE_1: "anthropic.claude-v1", // EOF: No longer supported
+  ANTHROPIC_CLAUDE_2: "anthropic.claude-v2",
+  ANTHROPIC_CLAUDE_2_1: "anthropic.claude-v2:1",
+  ANTHROPIC_CLAUDE_3_SONNET: "anthropic.claude-3-sonnet-20240229-v1:0",
+  ANTHROPIC_CLAUDE_3_HAIKU: "anthropic.claude-3-haiku-20240307-v1:0",
+  ANTHROPIC_CLAUDE_3_OPUS: "anthropic.claude-3-opus-20240229-v1:0",
+  ANTHROPIC_CLAUDE_3_5_SONNET: "anthropic.claude-3-5-sonnet-20240620-v1:0",
+  META_LLAMA2_13B_CHAT: "meta.llama2-13b-chat-v1",
+  META_LLAMA2_70B_CHAT: "meta.llama2-70b-chat-v1",
+  META_LLAMA3_8B_INSTRUCT: "meta.llama3-8b-instruct-v1:0",
+  META_LLAMA3_70B_INSTRUCT: "meta.llama3-70b-instruct-v1:0",
+  META_LLAMA3_1_8B_INSTRUCT: "meta.llama3-1-8b-instruct-v1:0",
+  META_LLAMA3_1_70B_INSTRUCT: "meta.llama3-1-70b-instruct-v1:0",
+  META_LLAMA3_1_405B_INSTRUCT: "meta.llama3-1-405b-instruct-v1:0",
+  META_LLAMA3_2_1B_INSTRUCT: "meta.llama3-2-1b-instruct-v1:0",
+  META_LLAMA3_2_3B_INSTRUCT: "meta.llama3-2-3b-instruct-v1:0",
+  META_LLAMA3_2_11B_INSTRUCT: "meta.llama3-2-11b-instruct-v1:0",
+  META_LLAMA3_2_90B_INSTRUCT: "meta.llama3-2-90b-instruct-v1:0",
+  MISTRAL_7B_INSTRUCT: "mistral.mistral-7b-instruct-v0:2",
+  MISTRAL_MIXTRAL_7B_INSTRUCT: "mistral.mixtral-8x7b-instruct-v0:1",
+  MISTRAL_MIXTRAL_LARGE_2402: "mistral.mistral-large-2402-v1:0",
+};
+export type BEDROCK_MODELS =
+  (typeof BEDROCK_MODELS)[keyof typeof BEDROCK_MODELS];
+
+export const INFERENCE_BEDROCK_MODELS = {
+  US_ANTHROPIC_CLAUDE_3_HAIKU: "us.anthropic.claude-3-haiku-20240307-v1:0",
+  US_ANTHROPIC_CLAUDE_3_OPUS: "us.anthropic.claude-3-opus-20240229-v1:0",
+  US_ANTHROPIC_CLAUDE_3_SONNET: "us.anthropic.claude-3-sonnet-20240229-v1:0",
+  US_ANTHROPIC_CLAUDE_3_5_SONNET:
+    "us.anthropic.claude-3-5-sonnet-20240620-v1:0",
+  US_META_LLAMA_3_2_1B_INSTRUCT: "us.meta.llama3-2-1b-instruct-v1:0",
+  US_META_LLAMA_3_2_3B_INSTRUCT: "us.meta.llama3-2-3b-instruct-v1:0",
+  US_META_LLAMA_3_2_11B_INSTRUCT: "us.meta.llama3-2-11b-instruct-v1:0",
+  US_META_LLAMA_3_2_90B_INSTRUCT: "us.meta.llama3-2-90b-instruct-v1:0",
+
+  EU_ANTHROPIC_CLAUDE_3_HAIKU: "eu.anthropic.claude-3-haiku-20240307-v1:0",
+  EU_ANTHROPIC_CLAUDE_3_SONNET: "eu.anthropic.claude-3-sonnet-20240229-v1:0",
+  EU_ANTHROPIC_CLAUDE_3_5_SONNET:
+    "eu.anthropic.claude-3-5-sonnet-20240620-v1:0",
+  EU_META_LLAMA_3_2_1B_INSTRUCT: "eu.meta.llama3-2-1b-instruct-v1:0",
+  EU_META_LLAMA_3_2_3B_INSTRUCT: "eu.meta.llama3-2-3b-instruct-v1:0",
+};
+
+export type INFERENCE_BEDROCK_MODELS =
+  (typeof INFERENCE_BEDROCK_MODELS)[keyof typeof INFERENCE_BEDROCK_MODELS];
+
+export const INFERENCE_TO_BEDROCK_MAP: Record<
+  INFERENCE_BEDROCK_MODELS,
+  BEDROCK_MODELS
+> = {
+  [INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_HAIKU]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+  [INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_OPUS]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
+  [INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_SONNET]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
+  [INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_5_SONNET]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
+  [INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_1B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
+  [INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_3B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT,
+  [INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_11B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT,
+  [INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_90B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT,
+
+  [INFERENCE_BEDROCK_MODELS.EU_ANTHROPIC_CLAUDE_3_HAIKU]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+  [INFERENCE_BEDROCK_MODELS.EU_ANTHROPIC_CLAUDE_3_SONNET]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
+  [INFERENCE_BEDROCK_MODELS.EU_ANTHROPIC_CLAUDE_3_5_SONNET]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
+  [INFERENCE_BEDROCK_MODELS.EU_META_LLAMA_3_2_1B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
+  [INFERENCE_BEDROCK_MODELS.EU_META_LLAMA_3_2_3B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT,
+};
 
 /*
  * Values taken from https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html#model-parameters-claude
@@ -109,6 +170,10 @@ const CHAT_ONLY_MODELS = {
   [BEDROCK_MODELS.META_LLAMA3_1_8B_INSTRUCT]: 128000,
   [BEDROCK_MODELS.META_LLAMA3_1_70B_INSTRUCT]: 128000,
   [BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT]: 128000,
+  [BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT]: 131000,
+  [BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT]: 131000,
+  [BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT]: 128000,
+  [BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT]: 128000,
   [BEDROCK_MODELS.MISTRAL_7B_INSTRUCT]: 32000,
   [BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT]: 32000,
   [BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402]: 32000,
@@ -139,17 +204,25 @@ export const STREAMING_MODELS = new Set([
   BEDROCK_MODELS.META_LLAMA3_1_8B_INSTRUCT,
   BEDROCK_MODELS.META_LLAMA3_1_70B_INSTRUCT,
   BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT,
   BEDROCK_MODELS.MISTRAL_7B_INSTRUCT,
   BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT,
   BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402,
 ]);
 
-export const TOOL_CALL_MODELS = [
+export const TOOL_CALL_MODELS: BEDROCK_MODELS[] = [
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
   BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT,
 ];
 
 const getProvider = (model: string): Provider => {
@@ -166,7 +239,7 @@ const getProvider = (model: string): Provider => {
 };
 
 export type BedrockModelParams = {
-  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
+  model: BEDROCK_MODELS | INFERENCE_BEDROCK_MODELS;
   temperature?: number;
   topP?: number;
   maxTokens?: number;
@@ -185,6 +258,10 @@ export const BEDROCK_MODEL_MAX_TOKENS: Partial<Record<BEDROCK_MODELS, number>> =
     [BEDROCK_MODELS.META_LLAMA3_1_8B_INSTRUCT]: 2048,
     [BEDROCK_MODELS.META_LLAMA3_1_70B_INSTRUCT]: 2048,
     [BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT]: 2048,
   };
 
 const DEFAULT_BEDROCK_PARAMS = {
@@ -193,14 +270,15 @@ const DEFAULT_BEDROCK_PARAMS = {
   maxTokens: 1024, // required by anthropic
 };
 
-export type BedrockParams = BedrockModelParams & BedrockRuntimeClientConfig;
+export type BedrockParams = BedrockRuntimeClientConfig & BedrockModelParams;
 
 /**
  * ToolCallLLM for Bedrock
  */
 export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
   private client: BedrockRuntimeClient;
-  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
+  protected actualModel: BEDROCK_MODELS | INFERENCE_BEDROCK_MODELS;
+  model: BEDROCK_MODELS;
   temperature: number;
   topP: number;
   maxTokens?: number;
@@ -217,8 +295,8 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
     ...params
   }: BedrockParams) {
     super();
-
-    this.model = model;
+    this.actualModel = model;
+    this.model = INFERENCE_TO_BEDROCK_MAP[model] ?? model;
     this.provider = getProvider(this.model);
     this.maxTokens = maxTokens ?? DEFAULT_BEDROCK_PARAMS.maxTokens;
     this.temperature = temperature ?? DEFAULT_BEDROCK_PARAMS.temperature;
@@ -241,7 +319,7 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
       temperature: this.temperature,
       topP: this.topP,
       maxTokens: this.maxTokens,
-      contextWindow: BEDROCK_FOUNDATION_LLMS[this.model],
+      contextWindow: BEDROCK_FOUNDATION_LLMS[this.model] ?? 128000,
       tokenizer: undefined,
     };
   }
@@ -256,6 +334,8 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
       params.additionalChatOptions,
     );
     const command = new InvokeModelCommand(input);
+    command.input.modelId = this.actualModel;
+
     const response = await this.client.send(command);
     let options: ToolCallLLMMessageOptions = {};
     if (this.supportToolCall) {
@@ -287,6 +367,8 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
       params.additionalChatOptions,
     );
     const command = new InvokeModelWithResponseStreamCommand(input);
+    command.input.modelId = this.actualModel;
+
     const response = await this.client.send(command);
 
     if (response.body) yield* this.provider.reduceStream(response.body);

packages/community/src/llm/bedrock/meta/provider.ts

@@ -67,21 +67,26 @@ export class MetaProvider extends Provider<MetaStreamEvent> {
     for await (const response of stream) {
       const event = this.getStreamingEventResponse(response);
       const delta = this.getTextFromStreamResponse(response);
+
       // odd quirk of llama3.1, start token is \n\n
       if (
+        !toolId &&
         !event?.generation.trim() &&
         event?.generation_token_count === 1 &&
-        event.prompt_token_count !== null
+        event?.prompt_token_count !== null
       )
         continue;
 
-      if (delta === TOKENS.TOOL_CALL) {
+      if (delta.startsWith(TOKENS.TOOL_CALL)) {
         toolId = randomUUID();
+        const parts = delta.split(TOKENS.TOOL_CALL).filter((part) => part);
+        collecting.push(...parts);
         continue;
       }
 
       let options: undefined | ToolCallLLMMessageOptions = undefined;
       if (toolId && event?.stop_reason === "stop") {
+        if (delta) collecting.push(delta);
         const tool = JSON.parse(collecting.join(""));
         options = {
           toolCall: [
@@ -110,11 +115,18 @@ export class MetaProvider extends Provider<MetaStreamEvent> {
   getRequestBody<T extends ChatMessage>(
     metadata: LLMMetadata,
     messages: T[],
-    tools?: BaseTool[],
+    tools: BaseTool[] = [],
   ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
     let prompt: string = "";
+    let images: string[] = [];
     if (metadata.model.startsWith("meta.llama3")) {
-      prompt = mapChatMessagesToMetaLlama3Messages(messages, tools);
+      const mapped = mapChatMessagesToMetaLlama3Messages({
+        messages,
+        tools,
+        model: metadata.model,
+      });
+      prompt = mapped.prompt;
+      images = mapped.images;
     } else if (metadata.model.startsWith("meta.llama2")) {
       prompt = mapChatMessagesToMetaLlama2Messages(messages);
     } else {
@@ -127,6 +139,7 @@ export class MetaProvider extends Provider<MetaStreamEvent> {
       accept: "application/json",
       body: JSON.stringify({
         prompt,
+        images: images.length ? images : undefined,
         max_gen_len: metadata.maxTokens,
         temperature: metadata.temperature,
         top_p: metadata.topP,

packages/community/src/llm/bedrock/meta/utils.ts

@@ -1,9 +1,12 @@
 import type {
   BaseTool,
   ChatMessage,
+  LLMMetadata,
   MessageContentTextDetail,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
+import { extractDataUrlComponents } from "../utils";
+import { TOKENS } from "./constants";
 import type { MetaMessage } from "./types";
 
 const getToolCallInstructionString = (tool: BaseTool): string => {
@@ -24,7 +27,7 @@ const getToolCallParametersString = (tool: BaseTool): string => {
 
 // ported from https://github.com/meta-llama/llama-agentic-system/blob/main/llama_agentic_system/system_prompt.py
 // NOTE: using json instead of the above xml style tool calling works more reliability
-export const getToolsPrompt = (tools?: BaseTool[]) => {
+export const getToolsPrompt_3_1 = (tools?: BaseTool[]) => {
   if (!tools?.length) return "";
 
   const customToolParams = tools.map((tool) => {
@@ -77,6 +80,46 @@ Reminder:
   `;
 };
 
+export const getToolsPrompt_3_2 = (tools?: BaseTool[]) => {
+  if (!tools?.length) return "";
+  return `
+You are an expert in composing functions. You are given a question and a set of possible functions.
+Based on the question, you will need to make one or more function/tool calls to achieve the purpose.
+If none of the function can be used, point it out. If the given question lacks the parameters required by the function,
+also point it out. You should only return the function call in tools call sections.
+
+If you decide to invoke any of the function(s), you MUST put it in the format of and start with the token: ${TOKENS.TOOL_CALL}:
+{
+  "name": function_name,
+  "parameters": parameters,
+}
+where
+
+{
+  "name": function_name,
+  "parameters": parameters, => a JSON dict with the function argument name as key and function argument value as value.
+}
+
+Here is an example,
+
+{
+  "name": "example_function_name",
+  "parameters": {"example_name": "example_value"}
+}
+
+Reminder:
+- Function calls MUST follow the specified format
+- Required parameters MUST be specified
+- Only call one function at a time
+- You SHOULD NOT include any other text in the response
+- Put the entire function call reply on one line
+
+Here is a list of functions in JSON format that you can invoke.
+
+${JSON.stringify(tools)}
+`;
+};
+
 export const mapChatRoleToMetaRole = (
   role: ChatMessage["role"],
 ): MetaMessage["role"] => {
@@ -125,16 +168,46 @@ export const mapChatMessagesToMetaMessages = <
 /**
  * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
  */
-export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>(
-  messages: T[],
-  tools?: BaseTool[],
-): string => {
+export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>({
+  messages,
+  model,
+  tools,
+}: {
+  messages: T[];
+  model: LLMMetadata["model"];
+  tools?: BaseTool[];
+}): { prompt: string; images: string[] } => {
+  const images: string[] = [];
+  const textMessages: T[] = [];
+
+  messages.forEach((message) => {
+    if (Array.isArray(message.content)) {
+      message.content.forEach((content) => {
+        if (content.type === "image_url") {
+          const { base64 } = extractDataUrlComponents(content.image_url.url);
+          images.push(base64);
+        } else {
+          textMessages.push(message);
+        }
+      });
+    } else {
+      textMessages.push(message);
+    }
+  });
+
   const parts: string[] = [];
-  if (tools?.length) {
+
+  let toolsPrompt = "";
+  if (model.startsWith("meta.llama3-2")) {
+    toolsPrompt = getToolsPrompt_3_2(tools);
+  } else if (model.startsWith("meta.llama3-1")) {
+    toolsPrompt = getToolsPrompt_3_1(tools);
+  }
+  if (toolsPrompt) {
     parts.push(
       "<|begin_of_text|>",
       "<|start_header_id|>system<|end_header_id|>",
-      getToolsPrompt(tools),
+      toolsPrompt,
       "<|eot_id|>",
     );
   }
@@ -154,7 +227,9 @@ export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>(
     ...mapped,
     "<|start_header_id|>assistant<|end_header_id|>",
   );
-  return parts.join("\n");
+
+  const prompt = parts.join("\n");
+  return { prompt, images };
 };
 
 /**

packages/community/src/llm/bedrock/utils.ts

@@ -11,3 +11,24 @@ export const mapMessageContentToMessageContentDetails = (
 
 export const toUtf8 = (input: Uint8Array): string =>
   new TextDecoder("utf-8").decode(input);
+
+export const extractDataUrlComponents = (
+  dataUrl: string,
+): {
+  mimeType: string;
+  base64: string;
+} => {
+  const parts = dataUrl.split(";base64,");
+
+  if (parts.length !== 2 || !parts[0]!.startsWith("data:")) {
+    throw new Error("Invalid data URL");
+  }
+
+  const mimeType = parts[0]!.slice(5);
+  const base64 = parts[1]!;
+
+  return {
+    mimeType,
+    base64,
+  };
+};

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/core
 
+## 0.2.9
+
+### Patch Changes
+
+- b17d439: Fix #1278: resolved issue where the id\_ was not correctly passed as the id when creating a TextNode. As a result, the upsert operation to the vector database was using a generated ID instead of the provided document ID, if available.
+
 ## 0.2.8
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./agent": {

packages/core/src/schema/node.ts

@@ -479,7 +479,7 @@ export function buildNodeFromSplits(
     ) {
       const imageDoc = doc as ImageNode;
       const imageNode = new ImageNode({
-        id_: idGenerator(i, imageDoc),
+        id_: imageDoc.id_ ?? idGenerator(i, imageDoc),
         text: textChunk,
         image: imageDoc.image,
         embedding: imageDoc.embedding,
@@ -496,7 +496,7 @@ export function buildNodeFromSplits(
     ) {
       const textDoc = doc as TextNode;
       const node = new TextNode({
-        id_: idGenerator(i, textDoc),
+        id_: textDoc.id_ ?? idGenerator(i, textDoc),
         text: textChunk,
         embedding: textDoc.embedding,
         excludedEmbedMetadataKeys: [...textDoc.excludedEmbedMetadataKeys],

packages/core/tests/workflow.test.ts

@@ -166,4 +166,68 @@ describe("Workflow", () => {
       greeting: "Hello Alice, you are 30 years old!",
     });
   });
+
+  test("workflow with two concurrent steps", async () => {
+    const concurrentFlow = new Workflow({ verbose: true });
+
+    const step1 = vi.fn(async (_context, _ev: StartEvent) => {
+      await new Promise((resolve) => setTimeout(resolve, 200));
+      return new StopEvent({ result: "Step 1 completed" });
+    });
+
+    const step2 = vi.fn(async (_context, _ev: StartEvent) => {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      return new StopEvent({ result: "Step 2 completed" });
+    });
+
+    concurrentFlow.addStep(StartEvent, step1);
+    concurrentFlow.addStep(StartEvent, step2);
+
+    const startTime = new Date();
+    const result = await concurrentFlow.run("start");
+    const endTime = new Date();
+    const duration = endTime.getTime() - startTime.getTime();
+
+    expect(step1).toHaveBeenCalledTimes(1);
+    expect(step2).toHaveBeenCalledTimes(1);
+    expect(duration).toBeLessThan(200);
+    expect(result.data.result).toBe("Step 2 completed");
+  });
+
+  test("workflow with two concurrent cyclic steps", async () => {
+    const concurrentCyclicFlow = new Workflow({ verbose: true });
+
+    class Step1Event extends WorkflowEvent {}
+    class Step2Event extends WorkflowEvent {}
+
+    let step2Count = 0;
+
+    const step1 = vi.fn(async (_context, ev: StartEvent | Step1Event) => {
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+      return new Step1Event({ result: "Step 1 completed" });
+    });
+
+    const step2 = vi.fn(async (_context, ev: StartEvent | Step2Event) => {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      step2Count++;
+      if (step2Count >= 5) {
+        return new StopEvent({ result: "Step 2 completed 5 times" });
+      }
+      return new Step2Event({ result: "Step 2 completed" });
+    });
+
+    concurrentCyclicFlow.addStep([StartEvent, Step1Event], step1);
+    concurrentCyclicFlow.addStep([StartEvent, Step2Event], step2);
+
+    const startTime = new Date();
+    const result = await concurrentCyclicFlow.run("start");
+    const endTime = new Date();
+    const duration = endTime.getTime() - startTime.getTime();
+
+    expect(step1).toHaveBeenCalledTimes(1);
+    expect(step2).toHaveBeenCalledTimes(5);
+    expect(duration).toBeGreaterThan(500); // At least 5 * 100ms for step2
+    expect(duration).toBeLessThan(1000); // Less than 1 second
+    expect(result.data.result).toBe("Step 2 completed 5 times");
+  });
 });

packages/experimental/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/experimental
 
+## 0.0.93
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 0.0.92
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 0.0.91
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 0.0.90
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
 ## 0.0.89
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.89",
+  "version": "0.0.93",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,41 @@
 # llamaindex
 
+## 0.6.15
+
+### Patch Changes
+
+- 2a82413: fix(core): set `Settings.llm` to OpenAI by default and support lazy load openai
+- Updated dependencies [2a82413]
+- Updated dependencies [0b20ff9]
+  - @llamaindex/groq@0.0.11
+  - @llamaindex/openai@0.1.12
+  - @llamaindex/cloud@0.2.11
+
+## 0.6.14
+
+### Patch Changes
+
+- Updated dependencies [b17d439]
+  - @llamaindex/core@0.2.9
+  - @llamaindex/ollama@0.0.4
+  - @llamaindex/openai@0.1.11
+  - @llamaindex/groq@0.0.10
+
+## 0.6.13
+
+### Patch Changes
+
+- Updated dependencies [981811e]
+  - @llamaindex/cloud@0.2.10
+
+## 0.6.12
+
+### Patch Changes
+
+- f7b4e94: feat: add filters for pinecone
+- 78037a6: fix: bypass service context embed model
+- 1d9e3b1: fix: export llama reader in non-nodejs runtime
+
 ## 0.6.11
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.77
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 0.0.76
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 0.0.75
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 0.0.74
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
 ## 0.0.73
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.73",
+  "version": "0.0.77",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.7
+
+### Patch Changes
+
+- Updated dependencies [0b20ff9]
+  - @llamaindex/cloud@0.2.11
+
+## 0.0.6
+
+### Patch Changes
+
+- Updated dependencies [981811e]
+  - @llamaindex/cloud@0.2.10
+
 ## 0.0.5
 
 ### Patch Changes

packages/llamaindex/e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.5",
+  "version": "0.0.7",
   "type": "module",
   "scripts": {
     "dev": "vite",

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/next-agent-test
 
+## 0.1.77
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 0.1.76
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 0.1.75
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 0.1.74
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
 ## 0.1.73
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.73",
+  "version": "0.1.77",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,33 @@
 # test-edge-runtime
 
+## 0.1.76
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 0.1.75
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 0.1.74
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 0.1.73
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
 ## 0.1.72
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.72",
+  "version": "0.1.76",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.58
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 0.0.57
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 0.0.56
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 0.0.55
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
 ## 0.0.54
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.54",
+  "version": "0.0.58",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.77
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 0.0.76
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 0.0.75
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 0.0.74
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
 ## 0.0.73
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.73",
+  "version": "0.0.77",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.6.11",
+  "version": "0.6.15",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/Settings.ts

@@ -2,7 +2,6 @@ import {
   type CallbackManager,
   Settings as CoreSettings,
 } from "@llamaindex/core/global";
-import { OpenAI } from "@llamaindex/openai";
 
 import { PromptHelper } from "@llamaindex/core/indices";
 
@@ -61,12 +60,6 @@ class GlobalSettings implements Config {
   }
 
   get llm(): LLM {
-    // fixme: we might need check internal error instead of try-catch here
-    try {
-      CoreSettings.llm;
-    } catch (error) {
-      CoreSettings.llm = new OpenAI();
-    }
     return CoreSettings.llm;
   }
 

packages/llamaindex/src/embeddings/fireworks.ts

@@ -2,7 +2,7 @@ import { getEnv } from "@llamaindex/env";
 import { OpenAIEmbedding } from "@llamaindex/openai";
 
 export class FireworksEmbedding extends OpenAIEmbedding {
-  constructor(init?: Partial<OpenAIEmbedding>) {
+  constructor(init?: Omit<Partial<OpenAIEmbedding>, "session">) {
     const {
       apiKey = getEnv("FIREWORKS_API_KEY"),
       additionalSessionOptions = {},

packages/llamaindex/src/embeddings/together.ts

@@ -2,7 +2,7 @@ import { getEnv } from "@llamaindex/env";
 import { OpenAIEmbedding } from "@llamaindex/openai";
 
 export class TogetherEmbedding extends OpenAIEmbedding {
-  constructor(init?: Partial<OpenAIEmbedding>) {
+  constructor(init?: Omit<Partial<OpenAIEmbedding>, "session">) {
     const {
       apiKey = getEnv("TOGETHER_API_KEY"),
       additionalSessionOptions = {},

packages/llamaindex/src/index.edge.ts

@@ -1,3 +1,20 @@
+//#region initial setup for OpenAI
+import { OpenAI } from "@llamaindex/openai";
+import { Settings } from "./Settings.js";
+
+try {
+  Settings.llm;
+} catch {
+  Settings.llm = new OpenAI();
+}
+
+//#endregion
+
+export {
+  LlamaParseReader,
+  type Language,
+  type ResultType,
+} from "@llamaindex/cloud/reader";
 export * from "@llamaindex/core/agent";
 export * from "@llamaindex/core/chat-engine";
 export {
@@ -23,12 +40,12 @@ export type {
   JSONArray,
   JSONObject,
   JSONValue,
+  LlamaIndexEventMaps,
   LLMEndEvent,
   LLMStartEvent,
   LLMStreamEvent,
   LLMToolCallEvent,
   LLMToolResultEvent,
-  LlamaIndexEventMaps,
 } from "@llamaindex/core/global";
 export * from "@llamaindex/core/indices";
 export * from "@llamaindex/core/llms";
@@ -56,7 +73,7 @@ export * from "./postprocessors/index.js";
 export * from "./QuestionGenerator.js";
 export * from "./selectors/index.js";
 export * from "./ServiceContext.js";
-export { Settings } from "./Settings.js";
 export * from "./storage/StorageContext.js";
 export * from "./tools/index.js";
 export * from "./types.js";
+export { Settings };

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -203,7 +203,10 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
     } = {},
   ): Promise<VectorStoreIndex> {
     args.storageContext =
-      args.storageContext ?? (await storageContextFromDefaults({}));
+      args.storageContext ??
+      (await storageContextFromDefaults({
+        serviceContext: args.serviceContext,
+      }));
     args.vectorStores = args.vectorStores ?? args.storageContext.vectorStores;
     args.docStoreStrategy =
       args.docStoreStrategy ??

packages/llamaindex/src/llm/deepinfra.ts

@@ -6,7 +6,7 @@ const DEFAULT_MODEL = "mistralai/Mixtral-8x22B-Instruct-v0.1";
 const BASE_URL = "https://api.deepinfra.com/v1/openai";
 
 export class DeepInfra extends OpenAI {
-  constructor(init?: Partial<OpenAI>) {
+  constructor(init?: Omit<Partial<OpenAI>, "session">) {
     const {
       apiKey = getEnv(ENV_VARIABLE_NAME),
       additionalSessionOptions = {},

packages/llamaindex/src/llm/deepseek.ts

@@ -10,7 +10,9 @@ type DeepSeekModelName = keyof typeof DEEPSEEK_MODELS;
 const DEFAULT_MODEL: DeepSeekModelName = "deepseek-coder";
 
 export class DeepSeekLLM extends OpenAI {
-  constructor(init?: Partial<OpenAI> & { model?: DeepSeekModelName }) {
+  constructor(
+    init?: Omit<Partial<OpenAI>, "session"> & { model?: DeepSeekModelName },
+  ) {
     const {
       apiKey = getEnv("DEEPSEEK_API_KEY"),
       additionalSessionOptions = {},

packages/llamaindex/src/llm/fireworks.ts

@@ -2,7 +2,7 @@ import { getEnv } from "@llamaindex/env";
 import { OpenAI } from "@llamaindex/openai";
 
 export class FireworksLLM extends OpenAI {
-  constructor(init?: Partial<OpenAI>) {
+  constructor(init?: Omit<Partial<OpenAI>, "session">) {
     const {
       apiKey = getEnv("FIREWORKS_API_KEY"),
       additionalSessionOptions = {},

packages/llamaindex/src/llm/together.ts

@@ -2,7 +2,7 @@ import { getEnv } from "@llamaindex/env";
 import { OpenAI } from "@llamaindex/openai";
 
 export class TogetherLLM extends OpenAI {
-  constructor(init?: Partial<OpenAI>) {
+  constructor(init?: Omit<Partial<OpenAI>, "session">) {
     const {
       apiKey = getEnv("TOGETHER_API_KEY"),
       additionalSessionOptions = {},

packages/llamaindex/src/storage/StorageContext.ts

@@ -5,6 +5,7 @@ import {
 import { ModalityType, ObjectType } from "@llamaindex/core/schema";
 import { path } from "@llamaindex/env";
 import { getImageEmbedModel } from "../internal/settings/image-embed-model.js";
+import type { ServiceContext } from "../ServiceContext.js";
 import { SimpleVectorStore } from "../vector-store/SimpleVectorStore.js";
 import type { VectorStore, VectorStoreByType } from "../vector-store/types.js";
 import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js";
@@ -25,6 +26,10 @@ type BuilderParams = {
   vectorStores: VectorStoreByType;
   storeImages: boolean;
   persistDir: string;
+  /**
+   * @deprecated Please use `Settings` instead
+   */
+  serviceContext?: ServiceContext | undefined;
 };
 
 export async function storageContextFromDefaults({
@@ -34,6 +39,7 @@ export async function storageContextFromDefaults({
   vectorStores,
   storeImages,
   persistDir,
+  serviceContext,
 }: Partial<BuilderParams>): Promise<StorageContext> {
   vectorStores = vectorStores ?? {};
   if (!persistDir) {
@@ -48,6 +54,7 @@ export async function storageContextFromDefaults({
       });
     }
   } else {
+    const embedModel = serviceContext?.embedModel;
     docStore =
       docStore ||
       (await SimpleDocumentStore.fromPersistDir(persistDir, DEFAULT_NAMESPACE));
@@ -55,7 +62,8 @@ export async function storageContextFromDefaults({
       indexStore || (await SimpleIndexStore.fromPersistDir(persistDir));
     if (!(ObjectType.TEXT in vectorStores)) {
       vectorStores[ModalityType.TEXT] =
-        vectorStore ?? (await SimpleVectorStore.fromPersistDir(persistDir));
+        vectorStore ??
+        (await SimpleVectorStore.fromPersistDir(persistDir, embedModel));
     }
     if (storeImages && !(ObjectType.IMAGE in vectorStores)) {
       vectorStores[ModalityType.IMAGE] = await SimpleVectorStore.fromPersistDir(

packages/llamaindex/src/vector-store/PineconeVectorStore.ts

@@ -1,4 +1,6 @@
 import {
+  FilterCondition,
+  FilterOperator,
   VectorStoreBase,
   type IEmbedModel,
   type MetadataFilter,
@@ -198,14 +200,60 @@ export class PineconeVectorStore
   }
 
   toPineconeFilter(stdFilters?: MetadataFilters) {
-    return stdFilters?.filters?.reduce((carry: any, item: MetadataFilter) => {
-      // Use MetadataFilter with EQ operator to replace ExactMatchFilter
-      // TODO: support filter with other operators
-      if (item.operator === "==") {
-        carry[item.key] = item.value;
+    if (!stdFilters) return undefined;
+
+    const transformCondition = (
+      condition: `${FilterCondition}` = "and",
+    ): string => {
+      if (condition === "and") return "$and";
+      if (condition === "or") return "$or";
+      throw new Error(`Filter condition ${condition} not supported`);
+    };
+
+    const transformOperator = (operator: `${FilterOperator}`): string => {
+      switch (operator) {
+        case "!=":
+          return "$ne";
+        case "==":
+          return "$eq";
+        case ">":
+          return "$gt";
+        case "<":
+          return "$lt";
+        case ">=":
+          return "$gte";
+        case "<=":
+          return "$lte";
+        case "in":
+          return "$in";
+        case "nin":
+          return "$nin";
+        default:
+          throw new Error(`Filter operator ${operator} not supported`);
       }
-      return carry;
-    }, {});
+    };
+
+    const convertFilterItem = (filter: MetadataFilter) => {
+      return {
+        [filter.key]: {
+          [transformOperator(filter.operator)]: filter.value,
+        },
+      };
+    };
+
+    const convertFilter = (filter: MetadataFilters) => {
+      const filtersList = filter.filters
+        .map((f) => convertFilterItem(f))
+        .filter((f) => Object.keys(f).length > 0);
+
+      if (filtersList.length === 0) return undefined;
+      if (filtersList.length === 1) return filtersList[0];
+
+      const condition = transformCondition(filter.condition);
+      return { [condition]: filtersList };
+    };
+
+    return convertFilter(stdFilters);
   }
 
   textFromResultRow(row: ScoredPineconeRecord<Metadata>): string {

packages/llamaindex/tests/indices/VectorStoreIndex.test.ts

@@ -27,7 +27,7 @@ describe("VectorStoreIndex", () => {
       runs: number = 2,
     ): Promise<Array<number>> => {
       const documents = [new Document({ text: "lorem ipsem", id_: "1" })];
-      const entries = [];
+      const entries: number[] = [];
       for (let i = 0; i < runs; i++) {
         await VectorStoreIndex.fromDocuments(documents, {
           serviceContext,
@@ -43,7 +43,7 @@ describe("VectorStoreIndex", () => {
 
   test("fromDocuments stores duplicates without a doc store strategy", async () => {
     const entries = await testStrategy(DocStoreStrategy.NONE);
-    expect(entries[0]! + 1).toBe(entries[1]);
+    expect(entries[0]).toBe(entries[1]);
   });
 
   test("fromDocuments ignores duplicates with upserts doc store strategy", async () => {

packages/llamaindex/tests/init.test.ts

@@ -0,0 +1,7 @@
+import { expect, test, vi } from "vitest";
+
+test("init without error", async () => {
+  vi.stubEnv("OPENAI_API_KEY", undefined);
+  const { Settings } = await import("llamaindex");
+  expect(Settings.llm).toBeDefined();
+});

packages/llm/groq/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @llamaindex/groq
 
+## 0.0.11
+
+### Patch Changes
+
+- 2a82413: fix(core): set `Settings.llm` to OpenAI by default and support lazy load openai
+- Updated dependencies [2a82413]
+  - @llamaindex/openai@0.1.12
+
+## 0.0.10
+
+### Patch Changes
+
+- @llamaindex/openai@0.1.11
+
 ## 0.0.9
 
 ### Patch Changes

packages/llm/groq/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/groq",
   "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/llm/groq/src/llm.ts

@@ -4,7 +4,7 @@ import GroqSDK, { type ClientOptions } from "groq-sdk";
 
 export class Groq extends OpenAI {
   constructor(
-    init?: Partial<OpenAI> & {
+    init?: Omit<Partial<OpenAI>, "session"> & {
       additionalSessionOptions?: ClientOptions;
     },
   ) {
@@ -22,9 +22,10 @@ export class Groq extends OpenAI {
       ...rest,
     });
 
-    this.session.openai = new GroqSDK({
-      apiKey,
-      ...init?.additionalSessionOptions,
-    }) as any;
+    this.lazySession = async () =>
+      new GroqSDK({
+        apiKey,
+        ...init?.additionalSessionOptions,
+      }) as any;
   }
 }

packages/llm/ollama/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/ollama
 
+## 0.0.4
+
+### Patch Changes
+
+- Updated dependencies [b17d439]
+  - @llamaindex/core@0.2.9
+
 ## 0.0.3
 
 ### Patch Changes

packages/llm/ollama/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/ollama",
   "description": "Ollama Adapter for LlamaIndex",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/llm/openai/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/openai
 
+## 0.1.12
+
+### Patch Changes
+
+- 2a82413: fix(core): set `Settings.llm` to OpenAI by default and support lazy load openai
+
+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [b17d439]
+  - @llamaindex/core@0.2.9
+
 ## 0.1.10
 
 ### Patch Changes

packages/llm/openai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/openai",
   "description": "OpenAI Adapter for LlamaIndex",
-  "version": "0.1.10",
+  "version": "0.1.12",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/llm/openai/src/azure.ts

@@ -2,11 +2,6 @@ import { getEnv } from "@llamaindex/env";
 
 import type { AzureClientOptions } from "openai";
 
-export interface AzureOpenAIConfig extends AzureClientOptions {
-  /** @deprecated use "deployment" instead */
-  deploymentName?: string | undefined;
-}
-
 // NOTE we're not supporting the legacy models as they're not available for new deployments
 // https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/legacy-models
 // If you have a need for them, please open an issue on GitHub
@@ -85,14 +80,15 @@ const DEFAULT_API_VERSION = "2023-05-15";
 //^ NOTE: this will change over time, if you want to pin it, use a specific version
 
 export function getAzureConfigFromEnv(
-  init?: Partial<AzureOpenAIConfig> & { model?: string },
-): AzureOpenAIConfig {
+  init?: Partial<AzureClientOptions> & { model?: string },
+): AzureClientOptions {
   const deployment =
-    init?.deploymentName ??
-    init?.deployment ??
-    getEnv("AZURE_OPENAI_DEPLOYMENT") ?? // From Azure docs
-    getEnv("AZURE_OPENAI_API_DEPLOYMENT_NAME") ?? // LCJS compatible
-    init?.model; // Fall back to model name, Python compatible
+    init && "deploymentName" in init && typeof init.deploymentName === "string"
+      ? init?.deploymentName
+      : (init?.deployment ??
+        getEnv("AZURE_OPENAI_DEPLOYMENT") ?? // From Azure docs
+        getEnv("AZURE_OPENAI_API_DEPLOYMENT_NAME") ?? // LCJS compatible
+        init?.model); // Fall back to model name, Python compatible
   return {
     apiKey:
       init?.apiKey ??
@@ -110,15 +106,10 @@ export function getAzureConfigFromEnv(
       getEnv("OPENAI_API_VERSION") ?? // Python compatible
       getEnv("AZURE_OPENAI_API_VERSION") ?? // LCJS compatible
       DEFAULT_API_VERSION,
-    deploymentName: deployment, // LCJS compatible
     deployment, // For Azure OpenAI
   };
 }
 
-export function getAzureBaseUrl(config: AzureOpenAIConfig): string {
-  return `${config.endpoint}/openai/deployments/${config.deploymentName}`;
-}
-
 export function getAzureModel(openAIModel: string) {
   for (const [key, value] of Object.entries(
     ALL_AZURE_OPENAI_EMBEDDING_MODELS,

packages/llm/openai/src/embedding.ts

@@ -1,14 +1,16 @@
 import { BaseEmbedding } from "@llamaindex/core/embeddings";
-import { Tokenizers } from "@llamaindex/env";
-import type { ClientOptions as OpenAIClientOptions } from "openai";
-import type { AzureOpenAIConfig } from "./azure.js";
+import { getEnv, Tokenizers } from "@llamaindex/env";
+import type {
+  AzureClientOptions,
+  AzureOpenAI as AzureOpenAILLM,
+  ClientOptions as OpenAIClientOptions,
+  OpenAI as OpenAILLM,
+} from "openai";
 import {
   getAzureConfigFromEnv,
   getAzureModel,
   shouldUseAzure,
 } from "./azure.js";
-import type { OpenAISession } from "./llm.js";
-import { getOpenAISession } from "./llm.js";
 
 export const ALL_OPENAI_EMBEDDING_MODELS = {
   "text-embedding-ada-002": {
@@ -32,6 +34,8 @@ export const ALL_OPENAI_EMBEDDING_MODELS = {
 
 type ModelKeys = keyof typeof ALL_OPENAI_EMBEDDING_MODELS;
 
+type LLMInstance = Pick<AzureOpenAILLM | OpenAILLM, "embeddings" | "apiKey">;
+
 export class OpenAIEmbedding extends BaseEmbedding {
   /** embeddding model. defaults to "text-embedding-ada-002" */
   model: string;
@@ -51,14 +55,26 @@ export class OpenAIEmbedding extends BaseEmbedding {
     | Omit<Partial<OpenAIClientOptions>, "apiKey" | "maxRetries" | "timeout">
     | undefined;
 
-  /** session object */
-  session: OpenAISession;
+  // use lazy here to avoid check OPENAI_API_KEY immediately
+  lazySession: () => Promise<LLMInstance>;
+  #session: Promise<LLMInstance> | null = null;
+  get session() {
+    if (!this.#session) {
+      this.#session = this.lazySession();
+    }
+    return this.#session;
+  }
 
   /**
    * OpenAI Embedding
    * @param init - initial parameters
    */
-  constructor(init?: Partial<OpenAIEmbedding> & { azure?: AzureOpenAIConfig }) {
+  constructor(
+    init?: Omit<Partial<OpenAIEmbedding>, "lazySession"> & {
+      session?: LLMInstance | undefined;
+      azure?: AzureClientOptions;
+    },
+  ) {
     super();
 
     this.model = init?.model ?? "text-embedding-ada-002";
@@ -77,7 +93,6 @@ export class OpenAIEmbedding extends BaseEmbedding {
     if (key) {
       this.embedInfo = ALL_OPENAI_EMBEDDING_MODELS[key];
     }
-
     if (init?.azure || shouldUseAzure()) {
       const azureConfig = {
         ...getAzureConfigFromEnv({
@@ -85,26 +100,32 @@ export class OpenAIEmbedding extends BaseEmbedding {
         }),
         ...init?.azure,
       };
-
-      this.apiKey = azureConfig.apiKey;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          azure: true,
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          ...this.additionalSessionOptions,
-          ...azureConfig,
-        });
+      this.apiKey =
+        init?.session?.apiKey ?? azureConfig.apiKey ?? getEnv("OPENAI_API_KEY");
+      this.lazySession = async () =>
+        import("openai").then(
+          async ({ AzureOpenAI }) =>
+            init?.session ??
+            new AzureOpenAI({
+              maxRetries: this.maxRetries,
+              timeout: this.timeout!,
+              ...this.additionalSessionOptions,
+              ...azureConfig,
+            }),
+        );
     } else {
-      this.apiKey = init?.apiKey ?? undefined;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          apiKey: this.apiKey,
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          ...this.additionalSessionOptions,
+      this.apiKey = init?.session?.apiKey ?? getEnv("OPENAI_API_KEY");
+      this.lazySession = async () =>
+        import("openai").then(({ OpenAI }) => {
+          return (
+            init?.session ??
+            new OpenAI({
+              apiKey: this.apiKey,
+              maxRetries: this.maxRetries,
+              timeout: this.timeout!,
+              ...this.additionalSessionOptions,
+            })
+          );
         });
     }
   }
@@ -118,7 +139,9 @@ export class OpenAIEmbedding extends BaseEmbedding {
     // TODO: ensure this for every sub class by calling it in the base class
     input = this.truncateMaxTokens(input);
 
-    const { data } = await this.session.openai.embeddings.create(
+    const { data } = await (
+      await this.session
+    ).embeddings.create(
       this.dimensions
         ? {
             model: this.model,

packages/llm/openai/src/index.ts

@@ -10,9 +10,6 @@ export {
   GPT4_MODELS,
   O1_MODELS,
   OpenAI,
-  OpenAISession,
   type OpenAIAdditionalChatOptions,
   type OpenAIAdditionalMetadata,
 } from "./llm";
-
-export { type AzureOpenAIConfig } from "./azure";

packages/llm/openai/src/llm.ts

@@ -1,12 +1,11 @@
 import { getEnv } from "@llamaindex/env";
-import type OpenAILLM from "openai";
 import type {
-  ClientOptions,
+  AzureClientOptions,
+  AzureOpenAI as AzureOpenAILLM,
   ClientOptions as OpenAIClientOptions,
+  OpenAI as OpenAILLM,
 } from "openai";
-import { AzureOpenAI, OpenAI as OrigOpenAI } from "openai";
 import type { ChatModel } from "openai/resources/chat/chat";
-import { isDeepEqual } from "remeda";
 
 import { wrapEventCaller, wrapLLMEvent } from "@llamaindex/core/decorator";
 import {
@@ -35,64 +34,12 @@ import type {
   ChatCompletionUserMessageParam,
 } from "openai/resources/chat/completions";
 import type { ChatCompletionMessageParam } from "openai/resources/index.js";
-import type { AzureOpenAIConfig } from "./azure.js";
 import {
   getAzureConfigFromEnv,
   getAzureModel,
   shouldUseAzure,
 } from "./azure.js";
 
-export class OpenAISession {
-  openai: Pick<OrigOpenAI, "chat" | "embeddings">;
-
-  constructor(options: ClientOptions & { azure?: boolean } = {}) {
-    if (options.azure) {
-      this.openai = new AzureOpenAI(options as AzureOpenAIConfig);
-    } else {
-      if (!options.apiKey) {
-        options.apiKey = getEnv("OPENAI_API_KEY");
-      }
-
-      if (!options.apiKey) {
-        throw new Error("Set OpenAI Key in OPENAI_API_KEY env variable"); // Overriding OpenAI package's error message
-      }
-
-      this.openai = new OrigOpenAI({
-        ...options,
-      });
-    }
-  }
-}
-
-// I'm not 100% sure this is necessary vs. just starting a new session
-// every time we make a call. They say they try to reuse connections
-// so in theory this is more efficient, but we should test it in the future.
-const defaultOpenAISession: {
-  session: OpenAISession;
-  options: ClientOptions;
-}[] = [];
-
-/**
- * Get a session for the OpenAI API. If one already exists with the same options,
- * it will be returned. Otherwise, a new session will be created.
- * @param options
- * @returns
- */
-export function getOpenAISession(
-  options: ClientOptions & { azure?: boolean } = {},
-) {
-  let session = defaultOpenAISession.find((session) => {
-    return isDeepEqual(session.options, options);
-  })?.session;
-
-  if (!session) {
-    session = new OpenAISession(options);
-    defaultOpenAISession.push({ session, options });
-  }
-
-  return session;
-}
-
 export const GPT4_MODELS = {
   "chatgpt-4o-latest": {
     contextWindow: 128000,
@@ -182,6 +129,8 @@ export type OpenAIAdditionalChatOptions = Omit<
   | "toolChoice"
 >;
 
+type LLMInstance = Pick<AzureOpenAILLM | OpenAILLM, "chat" | "apiKey">;
+
 export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
   model:
     | ChatModel
@@ -196,14 +145,24 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
   apiKey?: string | undefined = undefined;
   maxRetries: number;
   timeout?: number;
-  session: OpenAISession;
   additionalSessionOptions?:
     | undefined
     | Omit<Partial<OpenAIClientOptions>, "apiKey" | "maxRetries" | "timeout">;
 
+  // use lazy here to avoid check OPENAI_API_KEY immediately
+  lazySession: () => Promise<LLMInstance>;
+  #session: Promise<LLMInstance> | null = null;
+  get session() {
+    if (!this.#session) {
+      this.#session = this.lazySession();
+    }
+    return this.#session;
+  }
+
   constructor(
-    init?: Partial<OpenAI> & {
-      azure?: AzureOpenAIConfig;
+    init?: Omit<Partial<OpenAI>, "session"> & {
+      session?: LLMInstance | undefined;
+      azure?: AzureClientOptions;
     },
   ) {
     super();
@@ -216,6 +175,8 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
     this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
     this.additionalChatOptions = init?.additionalChatOptions;
     this.additionalSessionOptions = init?.additionalSessionOptions;
+    this.apiKey =
+      init?.session?.apiKey ?? init?.apiKey ?? getEnv("OPENAI_API_KEY");
 
     if (init?.azure || shouldUseAzure()) {
       const azureConfig = {
@@ -225,25 +186,26 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
         ...init?.azure,
       };
 
-      this.apiKey = azureConfig.apiKey;
-      this.session =
+      this.lazySession = async () =>
         init?.session ??
-        getOpenAISession({
-          azure: true,
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          ...this.additionalSessionOptions,
-          ...azureConfig,
+        import("openai").then(({ AzureOpenAI }) => {
+          return new AzureOpenAI({
+            maxRetries: this.maxRetries,
+            timeout: this.timeout!,
+            ...this.additionalSessionOptions,
+            ...azureConfig,
+          });
         });
     } else {
-      this.apiKey = init?.apiKey ?? undefined;
-      this.session =
+      this.lazySession = async () =>
         init?.session ??
-        getOpenAISession({
-          apiKey: this.apiKey,
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          ...this.additionalSessionOptions,
+        import("openai").then(({ OpenAI }) => {
+          return new OpenAI({
+            apiKey: this.apiKey,
+            maxRetries: this.maxRetries,
+            timeout: this.timeout!,
+            ...this.additionalSessionOptions,
+          });
         });
     }
   }
@@ -382,7 +344,9 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
     }
 
     // Non-streaming
-    const response = await this.session.openai.chat.completions.create({
+    const response = await (
+      await this.session
+    ).chat.completions.create({
       ...baseRequestParams,
       stream: false,
     });
@@ -414,11 +378,12 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
   protected async *streamChat(
     baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams,
   ): AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>> {
-    const stream: AsyncIterable<OpenAILLM.Chat.ChatCompletionChunk> =
-      await this.session.openai.chat.completions.create({
-        ...baseRequestParams,
-        stream: true,
-      });
+    const stream: AsyncIterable<OpenAILLM.Chat.ChatCompletionChunk> = await (
+      await this.session
+    ).chat.completions.create({
+      ...baseRequestParams,
+      stream: true,
+    });
 
     // TODO: add callback to streamConverter and use streamConverter here
     // this will be used to keep track of the current tool call, make sure input are valid json object.

pnpm-lock.yaml

@@ -360,10 +360,10 @@ importers:
         specifier: ^0.53.0
         version: 0.53.0(typescript@5.6.2)
       '@llamaindex/core':
-        specifier: workspace:^0.2.8
+        specifier: workspace:*
         version: link:../core
       '@llamaindex/env':
-        specifier: workspace:^0.1.13
+        specifier: workspace:*
         version: link:../env
       bunchee:
         specifier: 5.3.2
@@ -19518,8 +19518,8 @@ snapshots:
       '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.2)
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
+      eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.0)
       eslint-plugin-jsx-a11y: 6.9.0(eslint@8.57.0)
       eslint-plugin-react: 7.35.0(eslint@8.57.0)
       eslint-plugin-react-hooks: 4.6.2(eslint@8.57.0)
@@ -19566,25 +19566,6 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0):
-    dependencies:
-      '@nolyfill/is-core-module': 1.0.39
-      debug: 4.3.7
-      enhanced-resolve: 5.17.1
-      eslint: 8.57.0
-      eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
-      fast-glob: 3.3.2
-      get-tsconfig: 4.8.0
-      is-bun-module: 1.1.0
-      is-glob: 4.0.3
-    optionalDependencies:
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
-    transitivePeerDependencies:
-      - '@typescript-eslint/parser'
-      - eslint-import-resolver-node
-      - eslint-import-resolver-webpack
-      - supports-color
-
   eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
     dependencies:
       '@nolyfill/is-core-module': 1.0.39
@@ -19604,17 +19585,6 @@ snapshots:
       - eslint-import-resolver-webpack
       - supports-color
 
-  eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
-    dependencies:
-      debug: 3.2.7
-    optionalDependencies:
-      '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.2)
-      eslint: 8.57.0
-      eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0)
-    transitivePeerDependencies:
-      - supports-color
-
   eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
       debug: 3.2.7
@@ -19626,33 +19596,6 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
-    dependencies:
-      array-includes: 3.1.8
-      array.prototype.findlastindex: 1.2.5
-      array.prototype.flat: 1.3.2
-      array.prototype.flatmap: 1.3.2
-      debug: 3.2.7
-      doctrine: 2.1.0
-      eslint: 8.57.0
-      eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
-      hasown: 2.0.2
-      is-core-module: 2.15.1
-      is-glob: 4.0.3
-      minimatch: 3.1.2
-      object.fromentries: 2.0.8
-      object.groupby: 1.0.3
-      object.values: 1.2.0
-      semver: 6.3.1
-      tsconfig-paths: 3.15.0
-    optionalDependencies:
-      '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.2)
-    transitivePeerDependencies:
-      - eslint-import-resolver-typescript
-      - eslint-import-resolver-webpack
-      - supports-color
-
   eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.0):
     dependencies:
       array-includes: 3.1.8
@@ -19679,7 +19622,6 @@ snapshots:
       - eslint-import-resolver-typescript
       - eslint-import-resolver-webpack
       - supports-color
-    optional: true
 
   eslint-plugin-import@2.29.1(@typescript-eslint/parser@8.5.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0):
     dependencies: