Release 0.4.10 (#1003)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

README.md

@@ -76,7 +76,7 @@ main();
 node --import tsx ./main.ts
 ```
 
-### Next.js
+### React Server Component (Next.js, Waku, Redwood.JS...)
 
 First, you will need to add a llamaindex plugin to your Next.js project.
 
@@ -154,40 +154,6 @@ export async function chatWithAgent(
 }
 ```
 
-### Cloudflare Workers
-
-```ts
-// src/index.ts
-export default {
-  async fetch(
-    request: Request,
-    env: Env,
-    ctx: ExecutionContext,
-  ): Promise<Response> {
-    const { setEnvs } = await import("@llamaindex/env");
-    // set environment variables so that the OpenAIAgent can use them
-    setEnvs(env);
-    const { OpenAIAgent } = await import("llamaindex");
-    const agent = new OpenAIAgent({
-      tools: [],
-    });
-    const responseStream = await agent.chat({
-      stream: true,
-      message: "Hello? What is the weather today?",
-    });
-    const textEncoder = new TextEncoder();
-    const response = responseStream.pipeThrough(
-      new TransformStream({
-        transform: (chunk, controller) => {
-          controller.enqueue(textEncoder.encode(chunk.response.delta));
-        },
-      }),
-    );
-    return new Response(response);
-  },
-};
-```
-
 ## Playground
 
 Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground

apps/docs/CHANGELOG.md

@@ -1,5 +1,19 @@
 # docs
 
+## 0.0.36
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.34
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.34",
+  "version": "0.0.36",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.20
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.19
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+  - @llamaindex/autotool@1.0.0
+
 ## 0.1.18
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.18",
+  "version": "0.1.20",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.4.8",
+    "llamaindex": "^0.4.10",
     "openai": "^4",
     "typescript": "^4"
   },

packages/community/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @llamaindex/community
 
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.12
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.12",
+  "version": "0.0.14",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/package.json

@@ -4,6 +4,20 @@
   "version": "0.0.2",
   "description": "LlamaIndex Core Module",
   "exports": {
+    "./llms": {
+      "require": {
+        "types": "./dist/llms/index.d.cts",
+        "default": "./dist/llms/index.cjs"
+      },
+      "import": {
+        "types": "./dist/llms/index.d.ts",
+        "default": "./dist/llms/index.js"
+      },
+      "default": {
+        "types": "./dist/llms/index.d.ts",
+        "default": "./dist/llms/index.js"
+      }
+    },
     "./decorator": {
       "require": {
         "types": "./dist/decorator/index.d.cts",
@@ -60,6 +74,7 @@
     "url": "https://github.com/himself65/LlamaIndexTS.git"
   },
   "devDependencies": {
+    "ajv": "^8.16.0",
     "bunchee": "^5.2.1"
   },
   "dependencies": {

packages/core/src/global/type.ts

@@ -0,0 +1,9 @@
+export type UUID = `${string}-${string}-${string}-${string}-${string}`;
+
+export type JSONValue = string | number | boolean | JSONObject | JSONArray;
+
+export type JSONObject = {
+  [key: string]: JSONValue;
+};
+
+export type JSONArray = Array<JSONValue>;

packages/core/src/index.ts

@@ -1 +0,0 @@
-export * from "./schema";

packages/core/src/llms/index.ts

@@ -0,0 +1,31 @@
+export type {
+  BaseTool,
+  BaseToolWithCall,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChat,
+  LLMChatParamsBase,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsBase,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+  MessageContent,
+  MessageContentDetail,
+  MessageContentImageDetail,
+  MessageContentTextDetail,
+  MessageType,
+  PartialToolCall,
+  TextChatMessage,
+  ToolCall,
+  ToolCallLLMMessageOptions,
+  ToolCallOptions,
+  ToolMetadata,
+  ToolOutput,
+  ToolResult,
+  ToolResultOptions,
+} from "./type";

packages/core/src/llms/type.ts

@@ -0,0 +1,245 @@
+import type { Tokenizers } from "@llamaindex/env";
+import type { JSONSchemaType } from "ajv";
+import type { JSONObject, JSONValue } from "../global/type";
+
+/**
+ * @internal
+ */
+export interface LLMChat<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> {
+  chat(
+    params:
+      | LLMChatParamsStreaming<AdditionalChatOptions>
+      | LLMChatParamsNonStreaming<AdditionalChatOptions>,
+  ): Promise<
+    | ChatResponse<AdditionalMessageOptions>
+    | AsyncIterable<ChatResponseChunk<AdditionalMessageOptions>>
+  >;
+}
+
+/**
+ * Unified language model interface
+ */
+export interface LLM<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChat<AdditionalChatOptions> {
+  metadata: LLMMetadata;
+  /**
+   * Get a chat response from the LLM
+   */
+  chat(
+    params: LLMChatParamsStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(
+    params: LLMChatParamsNonStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
+  ): Promise<ChatResponse<AdditionalMessageOptions>>;
+
+  /**
+   * Get a prompt completion from the LLM
+   */
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+}
+
+export type MessageType = "user" | "assistant" | "system" | "memory";
+
+export type TextChatMessage<AdditionalMessageOptions extends object = object> =
+  {
+    content: string;
+    role: MessageType;
+    options?: undefined | AdditionalMessageOptions;
+  };
+
+export type ChatMessage<AdditionalMessageOptions extends object = object> = {
+  content: MessageContent;
+  role: MessageType;
+  options?: undefined | AdditionalMessageOptions;
+};
+
+export interface ChatResponse<
+  AdditionalMessageOptions extends object = object,
+> {
+  message: ChatMessage<AdditionalMessageOptions>;
+  /**
+   * Raw response from the LLM
+   *
+   * If LLM response an iterable of chunks, this will be an array of those chunks
+   */
+  raw: object | null;
+}
+
+export type ChatResponseChunk<
+  AdditionalMessageOptions extends object = object,
+> = {
+  raw: object | null;
+  delta: string;
+  options?: undefined | AdditionalMessageOptions;
+};
+
+export interface CompletionResponse {
+  text: string;
+  /**
+   * Raw response from the LLM
+   *
+   * It's possible that this is `null` if the LLM response an iterable of chunks
+   */
+  raw: object | null;
+}
+
+export type LLMMetadata = {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  contextWindow: number;
+  tokenizer: Tokenizers | undefined;
+};
+
+export interface LLMChatParamsBase<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> {
+  messages: ChatMessage<AdditionalMessageOptions>[];
+  additionalChatOptions?: AdditionalChatOptions;
+  tools?: BaseTool[];
+}
+
+export interface LLMChatParamsStreaming<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
+  stream: true;
+}
+
+export interface LLMChatParamsNonStreaming<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
+  stream?: false;
+}
+
+export interface LLMCompletionParamsBase {
+  prompt: MessageContent;
+}
+
+export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
+  stream: true;
+}
+
+export interface LLMCompletionParamsNonStreaming
+  extends LLMCompletionParamsBase {
+  stream?: false | null;
+}
+
+export type MessageContentTextDetail = {
+  type: "text";
+  text: string;
+};
+
+export type MessageContentImageDetail = {
+  type: "image_url";
+  image_url: { url: string };
+};
+
+export type MessageContentDetail =
+  | MessageContentTextDetail
+  | MessageContentImageDetail;
+
+/**
+ * Extended type for the content of a message that allows for multi-modal messages.
+ */
+export type MessageContent = string | MessageContentDetail[];
+
+export type ToolCall = {
+  name: string;
+  input: JSONObject;
+  id: string;
+};
+
+// happened in streaming response, the tool call is not ready yet
+export type PartialToolCall = {
+  name: string;
+  id: string;
+  // input is not ready yet, JSON.parse(input) will throw an error
+  input: string;
+};
+
+export type ToolResult = {
+  id: string;
+  result: string;
+  isError: boolean;
+};
+
+export type ToolCallOptions = {
+  toolCall: (ToolCall | PartialToolCall)[];
+};
+
+export type ToolResultOptions = {
+  toolResult: ToolResult;
+};
+
+export type ToolCallLLMMessageOptions =
+  | ToolResultOptions
+  | ToolCallOptions
+  | {};
+
+type Known =
+  | { [key: string]: Known }
+  | [Known, ...Known[]]
+  | Known[]
+  | number
+  | string
+  | boolean
+  | null;
+
+export type ToolMetadata<
+  Parameters extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  description: string;
+  name: string;
+  /**
+   * OpenAI uses JSON Schema to describe the parameters that a tool can take.
+   * @link https://json-schema.org/understanding-json-schema
+   */
+  parameters?: Parameters;
+};
+
+/**
+ * Simple Tool interface. Likely to change.
+ */
+export interface BaseTool<Input = any> {
+  /**
+   * This could be undefined if the implementation is not provided,
+   *  which might be the case when communicating with a llm.
+   *
+   * @return {JSONValue | Promise<JSONValue>} The output of the tool.
+   */
+  call?: (input: Input) => JSONValue | Promise<JSONValue>;
+  metadata: // if user input any, we cannot check the schema
+  Input extends Known ? ToolMetadata<JSONSchemaType<Input>> : ToolMetadata;
+}
+
+export type BaseToolWithCall<Input = any> = Omit<BaseTool<Input>, "call"> & {
+  call: NonNullable<Pick<BaseTool<Input>, "call">["call"]>;
+};
+
+export type ToolOutput = {
+  tool: BaseTool | undefined;
+  // all of existing function calling LLMs only support object input
+  input: JSONObject;
+  output: JSONValue;
+  isError: boolean;
+};

packages/core/src/schema/zod.ts

@@ -1,5 +1,7 @@
 import { z } from "zod";
 
+export const anyFunctionSchema = z.function(z.tuple([]).rest(z.any()), z.any());
+
 export const toolMetadataSchema = z.object({
   description: z.string(),
   name: z.string(),
@@ -7,7 +9,7 @@ export const toolMetadataSchema = z.object({
 });
 
 export const baseToolSchema = z.object({
-  call: z.optional(z.function()),
+  call: anyFunctionSchema.optional(),
   metadata: toolMetadataSchema,
 });
 

packages/experimental/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @llamaindex/experimental
 
+## 0.0.45
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.43
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.43",
+  "version": "0.0.45",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,17 @@
 # llamaindex
 
+## 0.4.10
+
+### Patch Changes
+
+- 7dce3d2: fix: disable External Filters for Gemini
+
+## 0.4.9
+
+### Patch Changes
+
+- 3a96a48: fix: anthroipic image input
+
 ## 0.4.8
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.27
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.27",
+  "version": "0.0.29",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @llamaindex/next-agent-test
 
+## 0.1.29
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.1.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.1.27
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.27",
+  "version": "0.1.29",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-agent/src/actions/index.tsx

@@ -1,7 +1,7 @@
 "use server";
 import { createStreamableUI } from "ai/rsc";
+import type { ChatMessage } from "llamaindex";
 import { OpenAIAgent } from "llamaindex";
-import type { ChatMessage } from "llamaindex/llm/types";
 
 export async function chatWithAgent(
   question: string,

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,19 @@
 # test-edge-runtime
 
+## 0.1.28
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.1.27
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.1.26
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.26",
+  "version": "0.1.28",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.10
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.9
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.8
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.27
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.27",
+  "version": "0.0.29",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/fixtures/llm/openai.ts

@@ -7,7 +7,7 @@ import type {
   LLMChatParamsStreaming,
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
-} from "llamaindex/llm/types";
+} from "llamaindex";
 import { extractText } from "llamaindex/llm/utils";
 import { deepStrictEqual, strictEqual } from "node:assert";
 import { llmCompleteMockStorage } from "../../node/utils.js";

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.4.8",
+  "version": "0.4.10",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/ChatHistory.ts

@@ -1,8 +1,8 @@
+import type { ChatMessage, LLM, MessageType } from "@llamaindex/core/llms";
 import { tokenizers, type Tokenizer } from "@llamaindex/env";
 import type { SummaryPrompt } from "./Prompt.js";
 import { defaultSummaryPrompt, messagesToHistoryStr } from "./Prompt.js";
 import { OpenAI } from "./llm/openai.js";
-import type { ChatMessage, LLM, MessageType } from "./llm/types.js";
 import { extractText } from "./llm/utils.js";
 
 /**

packages/llamaindex/src/EngineResponse.ts

@@ -1,9 +1,9 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
 import type {
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
-} from "./llm/types.js";
+} from "@llamaindex/core/llms";
+import type { NodeWithScore } from "@llamaindex/core/schema";
 import { extractText } from "./llm/utils.js";
 
 export class EngineResponse implements ChatResponse, ChatResponseChunk {

packages/llamaindex/src/Prompt.ts

@@ -1,6 +1,5 @@
+import type { ChatMessage, ToolMetadata } from "@llamaindex/core/llms";
 import type { SubQuestion } from "./engines/query/types.js";
-import type { ChatMessage } from "./llm/types.js";
-import type { ToolMetadata } from "./types.js";
 
 /**
  * A SimplePrompt is a function that takes a dictionary of inputs and returns a string.

packages/llamaindex/src/QuestionGenerator.ts

@@ -1,3 +1,4 @@
+import type { LLM, ToolMetadata } from "@llamaindex/core/llms";
 import { SubQuestionOutputParser } from "./OutputParser.js";
 import type { SubQuestionPrompt } from "./Prompt.js";
 import { buildToolsText, defaultSubQuestionPrompt } from "./Prompt.js";
@@ -6,13 +7,8 @@ import type {
   SubQuestion,
 } from "./engines/query/types.js";
 import { OpenAI } from "./llm/openai.js";
-import type { LLM } from "./llm/types.js";
 import { PromptMixin } from "./prompts/index.js";
-import type {
-  BaseOutputParser,
-  StructuredOutput,
-  ToolMetadata,
-} from "./types.js";
+import type { BaseOutputParser, StructuredOutput } from "./types.js";
 
 /**
  * LLMQuestionGenerator uses the LLM to generate new questions for the LLM using tools and a user query.

packages/llamaindex/src/ServiceContext.ts

@@ -1,8 +1,8 @@
+import type { LLM } from "@llamaindex/core/llms";
 import { PromptHelper } from "./PromptHelper.js";
 import { OpenAIEmbedding } from "./embeddings/OpenAIEmbedding.js";
 import type { BaseEmbedding } from "./embeddings/types.js";
 import { OpenAI } from "./llm/openai.js";
-import type { LLM } from "./llm/types.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 import type { NodeParser } from "./nodeParsers/types.js";
 

packages/llamaindex/src/Settings.ts

@@ -5,6 +5,7 @@ import { OpenAI } from "./llm/openai.js";
 import { PromptHelper } from "./PromptHelper.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 
+import type { LLM } from "@llamaindex/core/llms";
 import { AsyncLocalStorage, getEnv } from "@llamaindex/env";
 import type { ServiceContext } from "./ServiceContext.js";
 import type { BaseEmbedding } from "./embeddings/types.js";
@@ -18,7 +19,6 @@ import {
   setEmbeddedModel,
   withEmbeddedModel,
 } from "./internal/settings/EmbedModel.js";
-import type { LLM } from "./llm/types.js";
 import type { NodeParser } from "./nodeParsers/types.js";
 
 export type PromptConfig = {

packages/llamaindex/src/agent/base.ts

@@ -1,3 +1,10 @@
+import type {
+  BaseToolWithCall,
+  ChatMessage,
+  LLM,
+  MessageContent,
+  ToolOutput,
+} from "@llamaindex/core/llms";
 import { ReadableStream, TransformStream, randomUUID } from "@llamaindex/env";
 import { ChatHistory } from "../ChatHistory.js";
 import { EngineResponse } from "../EngineResponse.js";
@@ -11,9 +18,7 @@ import { wrapEventCaller } from "../internal/context/EventCaller.js";
 import { consoleLogger, emptyLogger } from "../internal/logger.js";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import { isAsyncIterable } from "../internal/utils.js";
-import type { ChatMessage, LLM, MessageContent } from "../llm/index.js";
 import { ObjectRetriever } from "../objects/index.js";
-import type { BaseToolWithCall, ToolOutput } from "../types.js";
 import type {
   AgentTaskContext,
   TaskHandler,
@@ -229,13 +234,12 @@ export abstract class AgentRunner<
     const { llm, getTools, stream } = step.context;
     const lastMessage = step.context.store.messages.at(-1)!.content;
     const tools = await getTools(lastMessage);
-    const response = await llm.chat({
-      // @ts-expect-error
-      stream,
-      tools,
-      messages: [...step.context.store.messages],
-    });
     if (!stream) {
+      const response = await llm.chat({
+        stream,
+        tools,
+        messages: [...step.context.store.messages],
+      });
       await stepTools<LLM>({
         response,
         tools,
@@ -243,6 +247,11 @@ export abstract class AgentRunner<
         enqueueOutput,
       });
     } else {
+      const response = await llm.chat({
+        stream,
+        tools,
+        messages: [...step.context.store.messages],
+      });
       await stepToolsStreaming<LLM>({
         response,
         tools,

packages/llamaindex/src/agent/llm.ts

@@ -1,7 +1,6 @@
-import type { LLM } from "../llm/index.js";
+import type { BaseToolWithCall, LLM } from "@llamaindex/core/llms";
 import { ObjectRetriever } from "../objects/index.js";
 import { Settings } from "../Settings.js";
-import type { BaseToolWithCall } from "../types.js";
 import { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
 import { validateAgentParams } from "./utils.js";
 

packages/llamaindex/src/agent/react.ts

@@ -1,18 +1,19 @@
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLM,
+} from "@llamaindex/core/llms";
 import { randomUUID, ReadableStream } from "@llamaindex/env";
 import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
 import {
   isAsyncIterable,
   stringifyJSONToMessageContent,
 } from "../internal/utils.js";
-import {
-  type ChatMessage,
-  type ChatResponse,
-  type ChatResponseChunk,
-  type LLM,
-} from "../llm/index.js";
 import { extractText } from "../llm/utils.js";
 import { Settings } from "../Settings.js";
-import type { BaseTool, JSONObject, JSONValue } from "../types.js";
+import type { JSONObject, JSONValue } from "../types.js";
 import { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
 import type { TaskHandler } from "./types.js";
 import {

packages/llamaindex/src/agent/types.ts

@@ -1,14 +1,16 @@
-import { ReadableStream } from "@llamaindex/env";
-import type { Logger } from "../internal/logger.js";
-import type { BaseEvent } from "../internal/type.js";
 import type {
+  BaseToolWithCall,
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLM,
   MessageContent,
-} from "../llm/types.js";
-import type { BaseToolWithCall, ToolOutput, UUID } from "../types.js";
+  ToolOutput,
+} from "@llamaindex/core/llms";
+import { ReadableStream } from "@llamaindex/env";
+import type { Logger } from "../internal/logger.js";
+import type { BaseEvent } from "../internal/type.js";
+import type { UUID } from "../types.js";
 
 export type AgentTaskContext<
   Model extends LLM,

packages/llamaindex/src/agent/utils.ts

@@ -1,3 +1,15 @@
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLM,
+  PartialToolCall,
+  TextChatMessage,
+  ToolCall,
+  ToolCallLLMMessageOptions,
+  ToolOutput,
+} from "@llamaindex/core/llms";
 import { baseToolWithCallSchema } from "@llamaindex/core/schema";
 import { ReadableStream } from "@llamaindex/env";
 import { z } from "zod";
@@ -8,17 +20,7 @@ import {
   prettifyError,
   stringifyJSONToMessageContent,
 } from "../internal/utils.js";
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-  LLM,
-  PartialToolCall,
-  TextChatMessage,
-  ToolCall,
-  ToolCallLLMMessageOptions,
-} from "../llm/index.js";
-import type { BaseTool, JSONObject, JSONValue, ToolOutput } from "../types.js";
+import type { JSONObject, JSONValue } from "../types.js";
 import type { AgentParamsBase } from "./base.js";
 import type { TaskHandler } from "./types.js";
 
@@ -31,10 +33,12 @@ type StepToolsResponseParams<Model extends LLM> = {
   >[1];
 };
 
-type StepToolsStreamingResponseParams<Model extends LLM> =
-  StepToolsResponseParams<Model> & {
-    response: AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>;
-  };
+type StepToolsStreamingResponseParams<Model extends LLM> = Omit<
+  StepToolsResponseParams<Model>,
+  "response"
+> & {
+  response: AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>;
+};
 
 // #TODO stepTools and stepToolsStreaming should be moved to a better abstraction
 
@@ -83,7 +87,7 @@ export async function stepToolsStreaming<Model extends LLM>({
       }
     }
 
-    // If there are toolCalls but they didn't get read into the stream, used for Gemini
+    // If there are toolCalls, but they didn't get read into the stream, used for Gemini
     if (!toolCalls.size && value.options && "toolCall" in value.options) {
       value.options.toolCall.forEach((toolCall) => {
         toolCalls.set(toolCall.id, toolCall);

packages/llamaindex/src/callbacks/CallbackManager.ts

@@ -1,4 +1,5 @@
 import type { Anthropic } from "@anthropic-ai/sdk";
+import type { MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import { CustomEvent } from "@llamaindex/env";
 import type { AgentEndEvent, AgentStartEvent } from "../agent/types.js";
@@ -12,7 +13,6 @@ import type {
   LLMStreamEvent,
   LLMToolCallEvent,
   LLMToolResultEvent,
-  MessageContent,
   RetrievalEndEvent,
   RetrievalStartEvent,
 } from "../llm/types.js";

packages/llamaindex/src/embeddings/DeepInfraEmbedding.ts

@@ -1,5 +1,5 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import { getEnv } from "@llamaindex/env";
-import type { MessageContentDetail } from "../llm/index.js";
 import { extractSingleText } from "../llm/utils.js";
 import { BaseEmbedding } from "./types.js";
 

packages/llamaindex/src/embeddings/MultiModalEmbedding.ts

@@ -1,3 +1,4 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import {
   ImageNode,
   MetadataMode,
@@ -6,7 +7,6 @@ import {
   type BaseNode,
   type ImageType,
 } from "@llamaindex/core/schema";
-import type { MessageContentDetail } from "../llm/types.js";
 import { extractImage, extractSingleText } from "../llm/utils.js";
 import { BaseEmbedding, batchEmbeddings } from "./types.js";
 

packages/llamaindex/src/embeddings/types.ts

@@ -1,8 +1,8 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import type { BaseNode } from "@llamaindex/core/schema";
 import { MetadataMode } from "@llamaindex/core/schema";
 import { type Tokenizers } from "@llamaindex/env";
 import type { TransformComponent } from "../ingestion/types.js";
-import type { MessageContentDetail } from "../llm/types.js";
 import { extractSingleText } from "../llm/utils.js";
 import { truncateMaxTokens } from "./tokenizer.js";
 import { SimilarityType, similarity } from "./utils.js";

packages/llamaindex/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -1,3 +1,4 @@
+import type { ChatMessage, LLM } from "@llamaindex/core/llms";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import type { EngineResponse } from "../../EngineResponse.js";
@@ -9,7 +10,6 @@ import {
 import type { ServiceContext } from "../../ServiceContext.js";
 import { llmFromSettingsOrContext } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import type { ChatMessage, LLM } from "../../llm/index.js";
 import { extractText, streamReducer } from "../../llm/utils.js";
 import { PromptMixin } from "../../prompts/index.js";
 import type { QueryEngine } from "../../types.js";

packages/llamaindex/src/engines/chat/ContextChatEngine.ts

@@ -1,3 +1,9 @@
+import type {
+  ChatMessage,
+  LLM,
+  MessageContent,
+  MessageType,
+} from "@llamaindex/core/llms";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import { EngineResponse } from "../../EngineResponse.js";
@@ -5,8 +11,6 @@ import type { ContextSystemPrompt } from "../../Prompt.js";
 import type { BaseRetriever } from "../../Retriever.js";
 import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import type { ChatMessage, LLM } from "../../llm/index.js";
-import type { MessageContent, MessageType } from "../../llm/types.js";
 import {
   extractText,
   streamConverter,

packages/llamaindex/src/engines/chat/DefaultContextGenerator.ts

@@ -1,10 +1,10 @@
+import type { MessageContent, MessageType } from "@llamaindex/core/llms";
 import { type NodeWithScore } from "@llamaindex/core/schema";
+import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import { defaultContextSystemPrompt } from "../../Prompt.js";
-import type { BaseRetriever } from "../../Retriever.js";
-import type { MessageContent, MessageType } from "../../llm/types.js";
-import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import { PromptMixin } from "../../prompts/index.js";
+import type { BaseRetriever } from "../../Retriever.js";
 import { createMessageContent } from "../../synthesizers/utils.js";
 import type { Context, ContextGenerator } from "./types.js";
 

packages/llamaindex/src/engines/chat/SimpleChatEngine.ts

@@ -1,9 +1,9 @@
+import type { LLM } from "@llamaindex/core/llms";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import { EngineResponse } from "../../EngineResponse.js";
 import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import type { LLM } from "../../llm/index.js";
 import { streamConverter, streamReducer } from "../../llm/utils.js";
 import type {
   ChatEngine,

packages/llamaindex/src/engines/chat/types.ts

@@ -1,8 +1,7 @@
+import type { ChatMessage, MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import type { ChatHistory } from "../../ChatHistory.js";
 import type { EngineResponse } from "../../EngineResponse.js";
-import type { ChatMessage } from "../../llm/index.js";
-import type { MessageContent } from "../../llm/types.js";
 
 /**
  * Represents the base parameters for ChatEngine.

packages/llamaindex/src/engines/query/SubQuestionQueryEngine.ts

@@ -11,13 +11,12 @@ import {
 } from "../../synthesizers/index.js";
 
 import type {
-  BaseTool,
   QueryEngine,
   QueryEngineParamsNonStreaming,
   QueryEngineParamsStreaming,
-  ToolMetadata,
 } from "../../types.js";
 
+import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import type { BaseQuestionGenerator, SubQuestion } from "./types.js";
 

packages/llamaindex/src/engines/query/types.ts

@@ -1,4 +1,4 @@
-import type { ToolMetadata } from "../../types.js";
+import type { ToolMetadata } from "@llamaindex/core/llms";
 
 /**
  * QuestionGenerators generate new questions for the LLM using tools and a user query.

packages/llamaindex/src/evaluation/Correctness.ts

@@ -1,9 +1,9 @@
+import type { ChatMessage, LLM } from "@llamaindex/core/llms";
 import { MetadataMode } from "@llamaindex/core/schema";
-import type { ServiceContext } from "../ServiceContext.js";
-import { llmFromSettingsOrContext } from "../Settings.js";
-import type { ChatMessage, LLM } from "../llm/types.js";
 import { extractText } from "../llm/utils.js";
 import { PromptMixin } from "../prompts/Mixin.js";
+import type { ServiceContext } from "../ServiceContext.js";
+import { llmFromSettingsOrContext } from "../Settings.js";
 import type { CorrectnessSystemPrompt } from "./prompts.js";
 import {
   defaultCorrectnessSystemPrompt,

packages/llamaindex/src/extractors/MetadataExtractors.ts

@@ -1,6 +1,6 @@
+import type { LLM } from "@llamaindex/core/llms";
 import type { BaseNode } from "@llamaindex/core/schema";
 import { MetadataMode, TextNode } from "@llamaindex/core/schema";
-import type { LLM } from "../llm/index.js";
 import { OpenAI } from "../llm/index.js";
 import {
   defaultKeywordExtractorPromptTemplate,

packages/llamaindex/src/index.edge.ts

@@ -1,3 +1,4 @@
+export * from "@llamaindex/core/llms";
 export * from "@llamaindex/core/schema";
 export * from "./agent/index.js";
 export * from "./callbacks/CallbackManager.js";

packages/llamaindex/src/indices/keyword/index.ts

@@ -31,8 +31,8 @@ import {
   simpleExtractKeywords,
 } from "./utils.js";
 
+import type { LLM } from "@llamaindex/core/llms";
 import { llmFromSettingsOrContext } from "../../Settings.js";
-import type { LLM } from "../../llm/types.js";
 import { extractText } from "../../llm/utils.js";
 
 export interface KeywordIndexOptions {

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -1,3 +1,4 @@
+import type { MessageContent } from "@llamaindex/core/llms";
 import {
   ImageNode,
   ModalityType,
@@ -23,7 +24,6 @@ import {
 } from "../../ingestion/strategies/index.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
-import type { MessageContent } from "../../llm/types.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/types.js";
 import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";

packages/llamaindex/src/internal/prompt/react.ts

@@ -1,4 +1,4 @@
-import type { BaseTool } from "../../types.js";
+import type { BaseTool } from "@llamaindex/core/llms";
 
 export const getReACTAgentSystemHeader = (tools: BaseTool[]) => {
   const description = tools

packages/llamaindex/src/llm/anthropic.ts

@@ -13,19 +13,22 @@ import type {
   TextBlock,
   TextBlockParam,
 } from "@anthropic-ai/sdk/resources/index";
-import type { MessageParam } from "@anthropic-ai/sdk/resources/messages";
-import { getEnv } from "@llamaindex/env";
-import _ from "lodash";
-import type { BaseTool } from "../types.js";
-import { ToolCallLLM } from "./base.js";
 import type {
+  ImageBlockParam,
+  MessageParam,
+} from "@anthropic-ai/sdk/resources/messages";
+import type {
+  BaseTool,
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
   ToolCallLLMMessageOptions,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import _ from "lodash";
+import { ToolCallLLM } from "./base.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 export class AnthropicSession {
@@ -214,7 +217,32 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
         }
 
         return {
-          content: extractText(message.content),
+          content:
+            typeof message.content === "string"
+              ? message.content
+              : message.content.map(
+                  (content): TextBlockParam | ImageBlockParam =>
+                    content.type === "text"
+                      ? {
+                          type: "text",
+                          text: content.text,
+                        }
+                      : {
+                          type: "image",
+                          source: {
+                            data: content.image_url.url.substring(
+                              content.image_url.url.indexOf(",") + 1,
+                            ),
+                            media_type:
+                              `image/${content.image_url.url.substring("data:image/".length, content.image_url.url.indexOf(";base64"))}` as
+                                | "image/jpeg"
+                                | "image/png"
+                                | "image/gif"
+                                | "image/webp",
+                            type: "base64",
+                          },
+                        },
+                ),
           role: message.role as "user" | "assistant",
         } satisfies MessageParam;
       });

packages/llamaindex/src/llm/base.ts

@@ -9,7 +9,7 @@ import type {
   LLMCompletionParamsStreaming,
   LLMMetadata,
   ToolCallLLMMessageOptions,
-} from "./types.js";
+} from "@llamaindex/core/llms";
 import { extractText, streamConverter } from "./utils.js";
 
 export abstract class BaseLLM<

packages/llamaindex/src/llm/gemini/base.ts

@@ -7,8 +7,6 @@ import {
   type GenerateContentStreamResult as GoogleStreamGenerateContentResult,
 } from "@google/generative-ai";
 
-import { getEnv, randomUUID } from "@llamaindex/env";
-import { ToolCallLLM } from "../base.js";
 import type {
   CompletionResponse,
   LLMCompletionParamsNonStreaming,
@@ -16,7 +14,9 @@ import type {
   LLMMetadata,
   ToolCall,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
+import { getEnv, randomUUID } from "@llamaindex/env";
+import { ToolCallLLM } from "../base.js";
 import { streamConverter, wrapLLMEvent } from "../utils.js";
 import {
   GEMINI_BACKENDS,
@@ -33,6 +33,7 @@ import {
   type IGeminiSession,
 } from "./types.js";
 import {
+  DEFAULT_SAFETY_SETTINGS,
   GeminiHelper,
   getChatContext,
   getPartsText,
@@ -87,7 +88,10 @@ export class GeminiSession implements IGeminiSession {
   }
 
   getGenerativeModel(metadata: GoogleModelParams): GoogleGenerativeModel {
-    return this.gemini.getGenerativeModel(metadata);
+    return this.gemini.getGenerativeModel({
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
+      ...metadata,
+    });
   }
 
   getResponseText(response: EnhancedGenerateContentResponse): string {
@@ -143,8 +147,9 @@ export class GeminiSessionStore {
   }> = [];
 
   private static getSessionId(options: GeminiSessionOptions): string {
-    if (options.backend === GEMINI_BACKENDS.GOOGLE)
+    if (options.backend === GEMINI_BACKENDS.GOOGLE) {
       return options?.apiKey ?? "";
+    }
     return "";
   }
   private static sessionMatched(
@@ -223,6 +228,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
           ),
         },
       ],
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
     });
     const { response } = await chat.sendMessage(context.message);
     const topCandidate = response.candidates![0];
@@ -258,6 +264,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
           ),
         },
       ],
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
     });
     const result = await chat.sendMessageStream(context.message);
     yield* this.session.getChatStream(result);

packages/llamaindex/src/llm/gemini/types.ts

@@ -33,7 +33,7 @@ import type {
   LLMChatParamsStreaming,
   ToolCall,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
 
 export enum GEMINI_BACKENDS {
   GOOGLE = "google",

packages/llamaindex/src/llm/gemini/utils.ts

@@ -1,17 +1,20 @@
 import {
   type FunctionCall,
   type Content as GeminiMessageContent,
+  HarmBlockThreshold,
+  HarmCategory,
+  type SafetySetting,
 } from "@google/generative-ai";
 
 import { type GenerateContentResponse } from "@google-cloud/vertexai";
-import type { BaseTool } from "../../types.js";
 import type {
+  BaseTool,
   ChatMessage,
   MessageContentImageDetail,
   MessageContentTextDetail,
   MessageType,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
 import { extractDataUrlComponents } from "../utils.js";
 import type {
   ChatContext,
@@ -53,10 +56,13 @@ const getImageParts = (
     const { mimeType, base64: data } = extractDataUrlComponents(
       message.image_url.url,
     );
-    if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+    if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType)) {
       throw new Error(
-        `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+        `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join(
+          "\n",
+        )}`,
       );
+    }
     return {
       inlineData: {
         mimeType,
@@ -65,10 +71,13 @@ const getImageParts = (
     };
   }
   const mimeType = getFileURLMimeType(message.image_url.url);
-  if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+  if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType)) {
     throw new Error(
-      `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+      `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join(
+        "\n",
+      )}`,
     );
+  }
   return {
     fileData: { mimeType, fileUri: message.image_url.url },
   };
@@ -124,10 +133,11 @@ export const getChatContext = (
   // 2. Parts that have empty text
   const fnMap = params.messages.reduce(
     (result, message) => {
-      if (message.options && "toolCall" in message.options)
+      if (message.options && "toolCall" in message.options) {
         message.options.toolCall.forEach((call) => {
           result[call.id] = call.name;
         });
+      }
 
       return result;
     },
@@ -224,10 +234,11 @@ export class GeminiHelper {
     if (options && "toolResult" in options) {
       if (!fnMap) throw Error("fnMap must be set");
       const name = fnMap[options.toolResult.id];
-      if (!name)
+      if (!name) {
         throw Error(
           `Could not find the name for fn call with id ${options.toolResult.id}`,
         );
+      }
 
       return [
         {
@@ -299,3 +310,26 @@ export function getFunctionCalls(
     return undefined;
   }
 }
+
+/**
+ * Safety settings to disable external filters
+ * Documentation: https://ai.google.dev/gemini-api/docs/safety-settings
+ */
+export const DEFAULT_SAFETY_SETTINGS: SafetySetting[] = [
+  {
+    category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_HARASSMENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+];

packages/llamaindex/src/llm/gemini/vertex.ts

@@ -1,8 +1,8 @@
 import {
+  type GenerateContentResponse,
   VertexAI,
   GenerativeModel as VertexGenerativeModel,
   GenerativeModelPreview as VertexGenerativeModelPreview,
-  type GenerateContentResponse,
   type ModelParams as VertexModelParams,
   type StreamGenerateContentResult as VertexStreamGenerateContentResult,
 } from "@google-cloud/vertexai";
@@ -14,14 +14,14 @@ import type {
 } from "./types.js";
 
 import type { FunctionCall } from "@google/generative-ai";
-import { getEnv, randomUUID } from "@llamaindex/env";
 import type {
   CompletionResponse,
   ToolCall,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
+import { getEnv, randomUUID } from "@llamaindex/env";
 import { streamConverter } from "../utils.js";
-import { getFunctionCalls, getText } from "./utils.js";
+import { DEFAULT_SAFETY_SETTINGS, getFunctionCalls, getText } from "./utils.js";
 
 /* To use Google's Vertex AI backend, it doesn't use api key authentication.
  *
@@ -59,8 +59,16 @@ export class GeminiVertexSession implements IGeminiSession {
   getGenerativeModel(
     metadata: VertexModelParams,
   ): VertexGenerativeModelPreview | VertexGenerativeModel {
-    if (this.preview) return this.vertex.preview.getGenerativeModel(metadata);
-    return this.vertex.getGenerativeModel(metadata);
+    if (this.preview) {
+      return this.vertex.preview.getGenerativeModel({
+        safetySettings: DEFAULT_SAFETY_SETTINGS,
+        ...metadata,
+      });
+    }
+    return this.vertex.getGenerativeModel({
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
+      ...metadata,
+    });
   }
 
   getResponseText(response: GenerateContentResponse): string {

packages/llamaindex/src/llm/huggingface.ts

@@ -1,11 +1,4 @@
 import { HfInference } from "@huggingface/inference";
-import type {
-  PreTrainedModel,
-  PreTrainedTokenizer,
-  Tensor,
-} from "@xenova/transformers";
-import { lazyLoadTransformers } from "../internal/deps/transformers.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -14,7 +7,14 @@ import type {
   LLMChatParamsStreaming,
   LLMMetadata,
   ToolCallLLMMessageOptions,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import type {
+  PreTrainedModel,
+  PreTrainedTokenizer,
+  Tensor,
+} from "@xenova/transformers";
+import { lazyLoadTransformers } from "../internal/deps/transformers.js";
+import { BaseLLM } from "./base.js";
 import { streamConverter, wrapLLMEvent } from "./utils.js";
 
 // TODO workaround issue with @huggingface/inference@2.7.0

packages/llamaindex/src/llm/mistral.ts

@@ -1,14 +1,14 @@
-import { getEnv } from "@llamaindex/env";
-import { Settings } from "../Settings.js";
-import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
+import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
+import { BaseLLM } from "./base.js";
 
 export const ALL_AVAILABLE_MISTRAL_MODELS = {
   "mistral-tiny": { contextWindow: 32000 },

packages/llamaindex/src/llm/ollama.ts

@@ -1,3 +1,14 @@
+import type {
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+} from "@llamaindex/core/llms";
 import { BaseEmbedding } from "../embeddings/types.js";
 import {
   Ollama as OllamaBase,
@@ -19,17 +30,6 @@ import {
   type ShowResponse,
   type StatusResponse,
 } from "../internal/deps/ollama.js";
-import type {
-  ChatResponse,
-  ChatResponseChunk,
-  CompletionResponse,
-  LLM,
-  LLMChatParamsNonStreaming,
-  LLMChatParamsStreaming,
-  LLMCompletionParamsNonStreaming,
-  LLMCompletionParamsStreaming,
-  LLMMetadata,
-} from "./types.js";
 import { extractText, streamConverter } from "./utils.js";
 
 const messageAccessor = (part: OllamaChatResponse): ChatResponseChunk => {

packages/llamaindex/src/llm/openai.ts

@@ -7,6 +7,19 @@ import type {
 } from "openai";
 import { AzureOpenAI, OpenAI as OrigOpenAI } from "openai";
 
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMMetadata,
+  MessageType,
+  PartialToolCall,
+  ToolCallLLMMessageOptions,
+} from "@llamaindex/core/llms";
 import { Tokenizers } from "@llamaindex/env";
 import type {
   ChatCompletionAssistantMessageParam,
@@ -20,7 +33,6 @@ import type {
 import type { ChatCompletionMessageParam } from "openai/resources/index.js";
 import { wrapEventCaller } from "../internal/context/EventCaller.js";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import type { BaseTool } from "../types.js";
 import type { AzureOpenAIConfig } from "./azure.js";
 import {
   getAzureConfigFromEnv,
@@ -28,18 +40,6 @@ import {
   shouldUseAzure,
 } from "./azure.js";
 import { ToolCallLLM } from "./base.js";
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-  LLM,
-  LLMChatParamsNonStreaming,
-  LLMChatParamsStreaming,
-  LLMMetadata,
-  MessageType,
-  PartialToolCall,
-  ToolCallLLMMessageOptions,
-} from "./types.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 export class OpenAISession {

packages/llamaindex/src/llm/portkey.ts

@@ -1,10 +1,3 @@
-import { getEnv } from "@llamaindex/env";
-import _ from "lodash";
-import type { LLMOptions } from "portkey-ai";
-import { Portkey as OrigPortKey } from "portkey-ai";
-import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -13,7 +6,14 @@ import type {
   LLMChatParamsStreaming,
   LLMMetadata,
   MessageType,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import _ from "lodash";
+import type { LLMOptions } from "portkey-ai";
+import { Portkey as OrigPortKey } from "portkey-ai";
+import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { BaseLLM } from "./base.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 interface PortkeyOptions {

packages/llamaindex/src/llm/replicate_ai.ts

@@ -1,6 +1,3 @@
-import { getEnv } from "@llamaindex/env";
-import Replicate from "../internal/deps/replicate.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -8,7 +5,10 @@ import type {
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
   MessageType,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import Replicate from "../internal/deps/replicate.js";
+import { BaseLLM } from "./base.js";
 import {
   extractText,
   streamCallbacks,

packages/llamaindex/src/llm/types.ts

@@ -1,7 +1,14 @@
+import type {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  MessageContent,
+  ToolCall,
+  ToolOutput,
+} from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
-import type { Tokenizers } from "@llamaindex/env";
 import type { BaseEvent } from "../internal/type.js";
-import type { BaseTool, JSONObject, ToolOutput, UUID } from "../types.js";
+import type { UUID } from "../types.js";
 
 export type RetrievalStartEvent = BaseEvent<{
   query: MessageContent;
@@ -29,197 +36,3 @@ export type LLMStreamEvent = BaseEvent<{
   id: UUID;
   chunk: ChatResponseChunk;
 }>;
-
-/**
- * @internal
- */
-export interface LLMChat<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> {
-  chat(
-    params:
-      | LLMChatParamsStreaming<AdditionalChatOptions>
-      | LLMChatParamsNonStreaming<AdditionalChatOptions>,
-  ): Promise<
-    | ChatResponse<AdditionalMessageOptions>
-    | AsyncIterable<ChatResponseChunk<AdditionalMessageOptions>>
-  >;
-}
-
-/**
- * Unified language model interface
- */
-export interface LLM<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> extends LLMChat<AdditionalChatOptions> {
-  metadata: LLMMetadata;
-  /**
-   * Get a chat response from the LLM
-   */
-  chat(
-    params: LLMChatParamsStreaming<
-      AdditionalChatOptions,
-      AdditionalMessageOptions
-    >,
-  ): Promise<AsyncIterable<ChatResponseChunk>>;
-  chat(
-    params: LLMChatParamsNonStreaming<
-      AdditionalChatOptions,
-      AdditionalMessageOptions
-    >,
-  ): Promise<ChatResponse<AdditionalMessageOptions>>;
-
-  /**
-   * Get a prompt completion from the LLM
-   */
-  complete(
-    params: LLMCompletionParamsStreaming,
-  ): Promise<AsyncIterable<CompletionResponse>>;
-  complete(
-    params: LLMCompletionParamsNonStreaming,
-  ): Promise<CompletionResponse>;
-}
-
-export type MessageType = "user" | "assistant" | "system" | "memory";
-
-export type TextChatMessage<AdditionalMessageOptions extends object = object> =
-  {
-    content: string;
-    role: MessageType;
-    options?: undefined | AdditionalMessageOptions;
-  };
-
-export type ChatMessage<AdditionalMessageOptions extends object = object> = {
-  content: MessageContent;
-  role: MessageType;
-  options?: undefined | AdditionalMessageOptions;
-};
-
-export interface ChatResponse<
-  AdditionalMessageOptions extends object = object,
-> {
-  message: ChatMessage<AdditionalMessageOptions>;
-  /**
-   * Raw response from the LLM
-   *
-   * If LLM response an iterable of chunks, this will be an array of those chunks
-   */
-  raw: object | null;
-}
-
-export type ChatResponseChunk<
-  AdditionalMessageOptions extends object = object,
-> = {
-  raw: object | null;
-  delta: string;
-  options?: undefined | AdditionalMessageOptions;
-};
-
-export interface CompletionResponse {
-  text: string;
-  /**
-   * Raw response from the LLM
-   *
-   * It's possible that this is `null` if the LLM response an iterable of chunks
-   */
-  raw: object | null;
-}
-
-export type LLMMetadata = {
-  model: string;
-  temperature: number;
-  topP: number;
-  maxTokens?: number;
-  contextWindow: number;
-  tokenizer: Tokenizers | undefined;
-};
-
-export interface LLMChatParamsBase<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> {
-  messages: ChatMessage<AdditionalMessageOptions>[];
-  additionalChatOptions?: AdditionalChatOptions;
-  tools?: BaseTool[];
-}
-
-export interface LLMChatParamsStreaming<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
-  stream: true;
-}
-
-export interface LLMChatParamsNonStreaming<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
-  stream?: false;
-}
-
-export interface LLMCompletionParamsBase {
-  prompt: MessageContent;
-}
-
-export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
-  stream: true;
-}
-
-export interface LLMCompletionParamsNonStreaming
-  extends LLMCompletionParamsBase {
-  stream?: false | null;
-}
-
-export type MessageContentTextDetail = {
-  type: "text";
-  text: string;
-};
-
-export type MessageContentImageDetail = {
-  type: "image_url";
-  image_url: { url: string };
-};
-
-export type MessageContentDetail =
-  | MessageContentTextDetail
-  | MessageContentImageDetail;
-
-/**
- * Extended type for the content of a message that allows for multi-modal messages.
- */
-export type MessageContent = string | MessageContentDetail[];
-
-export type ToolCall = {
-  name: string;
-  input: JSONObject;
-  id: string;
-};
-
-// happened in streaming response, the tool call is not ready yet
-export type PartialToolCall = {
-  name: string;
-  id: string;
-  // input is not ready yet, JSON.parse(input) will throw an error
-  input: string;
-};
-
-export type ToolResult = {
-  id: string;
-  result: string;
-  isError: boolean;
-};
-
-export type ToolCallOptions = {
-  toolCall: (ToolCall | PartialToolCall)[];
-};
-
-export type ToolResultOptions = {
-  toolResult: ToolResult;
-};
-
-export type ToolCallLLMMessageOptions =
-  | ToolResultOptions
-  | ToolCallOptions
-  | {};

packages/llamaindex/src/llm/utils.ts

@@ -1,6 +1,3 @@
-import type { ImageType } from "@llamaindex/core/schema";
-import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import type {
   ChatResponse,
   ChatResponseChunk,
@@ -9,7 +6,10 @@ import type {
   MessageContent,
   MessageContentDetail,
   MessageContentTextDetail,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import type { ImageType } from "@llamaindex/core/schema";
+import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 
 export async function* streamConverter<S, D>(
   stream: AsyncIterable<S>,

packages/llamaindex/src/memory/ChatMemoryBuffer.ts

@@ -1,5 +1,5 @@
+import type { ChatMessage, LLM } from "@llamaindex/core/llms";
 import type { ChatHistory } from "../ChatHistory.js";
-import type { ChatMessage, LLM } from "../llm/index.js";
 import { SimpleChatStore } from "../storage/chatStore/SimpleChatStore.js";
 import type { BaseChatStore } from "../storage/chatStore/types.js";
 import type { BaseMemory } from "./types.js";

packages/llamaindex/src/memory/types.ts

@@ -1,4 +1,4 @@
-import type { ChatMessage } from "../llm/index.js";
+import type { ChatMessage } from "@llamaindex/core/llms";
 
 export interface BaseMemory<AdditionalMessageOptions extends object = object> {
   tokenLimit: number;

packages/llamaindex/src/objects/base.ts

@@ -1,10 +1,9 @@
+import type { BaseTool, MessageContent } from "@llamaindex/core/llms";
 import type { BaseNode, Metadata } from "@llamaindex/core/schema";
 import { TextNode } from "@llamaindex/core/schema";
 import type { BaseRetriever } from "../Retriever.js";
 import type { VectorStoreIndex } from "../indices/vectorStore/index.js";
-import type { MessageContent } from "../llm/index.js";
 import { extractText } from "../llm/utils.js";
-import type { BaseTool } from "../types.js";
 
 // Assuming that necessary interfaces and classes (like OT, TextNode, BaseNode, etc.) are defined elsewhere
 // Import statements (e.g., for TextNode, BaseNode) should be added based on your project's structure

packages/llamaindex/src/postprocessors/rerankers/CohereRerank.ts

@@ -1,8 +1,8 @@
 import { CohereClient } from "cohere-ai";
 
+import type { MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import { MetadataMode } from "@llamaindex/core/schema";
-import type { MessageContent } from "../../llm/types.js";
 import { extractText } from "../../llm/utils.js";
 import type { BaseNodePostprocessor } from "../types.js";
 

packages/llamaindex/src/postprocessors/rerankers/JinaAIReranker.ts

@@ -1,7 +1,7 @@
+import type { MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import { MetadataMode } from "@llamaindex/core/schema";
 import { getEnv } from "@llamaindex/env";
-import type { MessageContent } from "../../llm/types.js";
 import { extractText } from "../../llm/utils.js";
 import type { BaseNodePostprocessor } from "../types.js";
 

packages/llamaindex/src/postprocessors/types.ts

@@ -1,5 +1,5 @@
+import type { MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
-import type { MessageContent } from "../llm/types.js";
 
 export interface BaseNodePostprocessor {
   /**

packages/llamaindex/src/selectors/llmSelectors.ts

@@ -1,4 +1,4 @@
-import type { LLM } from "../llm/index.js";
+import type { LLM } from "@llamaindex/core/llms";
 import type { Answer } from "../outputParsers/selectors.js";
 import { SelectionOutputParser } from "../outputParsers/selectors.js";
 import type {

packages/llamaindex/src/storage/chatStore/SimpleChatStore.ts

@@ -1,4 +1,4 @@
-import type { ChatMessage } from "../../llm/index.js";
+import type { ChatMessage } from "@llamaindex/core/llms";
 import type { BaseChatStore } from "./types.js";
 
 /**

packages/llamaindex/src/storage/chatStore/types.ts

@@ -1,4 +1,4 @@
-import type { ChatMessage } from "../../llm/index.js";
+import type { ChatMessage } from "@llamaindex/core/llms";
 
 export interface BaseChatStore<
   AdditionalMessageOptions extends object = object,

packages/llamaindex/src/synthesizers/builders.ts

@@ -1,4 +1,4 @@
-import type { LLM } from "../llm/index.js";
+import type { LLM } from "@llamaindex/core/llms";
 import { streamConverter } from "../llm/utils.js";
 import type {
   RefinePrompt,

packages/llamaindex/src/synthesizers/utils.ts

@@ -1,3 +1,4 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import {
   ImageNode,
   MetadataMode,
@@ -7,7 +8,6 @@ import {
 } from "@llamaindex/core/schema";
 import type { SimplePrompt } from "../Prompt.js";
 import { imageToDataUrl } from "../embeddings/utils.js";
-import type { MessageContentDetail } from "../llm/types.js";
 
 export async function createMessageContent(
   prompt: SimplePrompt,

packages/llamaindex/src/tools/AzureDynamicSessionTool.node.ts

@@ -2,6 +2,7 @@ import {
   DefaultAzureCredential,
   getBearerTokenProvider,
 } from "@azure/identity";
+import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
 import {
   Readable,
   createWriteStream,
@@ -11,7 +12,6 @@ import {
   path,
   randomUUID,
 } from "@llamaindex/env";
-import type { BaseTool, ToolMetadata } from "../types.js";
 export type InterpreterParameter = {
   code: string;
 };

packages/llamaindex/src/tools/QueryEngineTool.ts

@@ -1,5 +1,6 @@
+import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
 import type { JSONSchemaType } from "ajv";
-import type { BaseTool, QueryEngine, ToolMetadata } from "../types.js";
+import type { QueryEngine } from "../types.js";
 
 const DEFAULT_NAME = "query_engine_tool";
 const DEFAULT_DESCRIPTION =

packages/llamaindex/src/tools/WikipediaTool.ts

@@ -1,6 +1,6 @@
+import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
 import type { JSONSchemaType } from "ajv";
 import { default as wiki } from "wikipedia";
-import type { BaseTool, ToolMetadata } from "../types.js";
 
 type WikipediaParameter = {
   query: string;

packages/llamaindex/src/tools/functionTool.ts

@@ -1,5 +1,6 @@
+import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
 import type { JSONSchemaType } from "ajv";
-import type { BaseTool, JSONValue, ToolMetadata } from "../types.js";
+import type { JSONValue } from "../types.js";
 
 export class FunctionTool<T, R extends JSONValue | Promise<JSONValue>>
   implements BaseTool<T>

packages/llamaindex/src/types.ts

@@ -1,7 +1,7 @@
 /**
  * Top level types to avoid circular dependencies
  */
-import { type JSONSchemaType } from "ajv";
+import type { ToolMetadata } from "@llamaindex/core/llms";
 import type { EngineResponse } from "./EngineResponse.js";
 
 /**
@@ -33,46 +33,6 @@ export interface QueryEngine {
   query(params: QueryEngineParamsNonStreaming): Promise<EngineResponse>;
 }
 
-type Known =
-  | { [key: string]: Known }
-  | [Known, ...Known[]]
-  | Known[]
-  | number
-  | string
-  | boolean
-  | null;
-
-export type ToolMetadata<
-  Parameters extends Record<string, unknown> = Record<string, unknown>,
-> = {
-  description: string;
-  name: string;
-  /**
-   * OpenAI uses JSON Schema to describe the parameters that a tool can take.
-   * @link https://json-schema.org/understanding-json-schema
-   */
-  parameters?: Parameters;
-};
-
-/**
- * Simple Tool interface. Likely to change.
- */
-export interface BaseTool<Input = any> {
-  /**
-   * This could be undefined if the implementation is not provided,
-   *  which might be the case when communicating with a llm.
-   *
-   * @return {JSONValue | Promise<JSONValue>} The output of the tool.
-   */
-  call?: (input: Input) => JSONValue | Promise<JSONValue>;
-  metadata: // if user input any, we cannot check the schema
-  Input extends Known ? ToolMetadata<JSONSchemaType<Input>> : ToolMetadata;
-}
-
-export type BaseToolWithCall<Input = any> = Omit<BaseTool<Input>, "call"> & {
-  call: NonNullable<Pick<BaseTool<Input>, "call">["call"]>;
-};
-
 /**
  * An OutputParser is used to extract structured data from the raw output of the LLM.
  */
@@ -113,11 +73,3 @@ export type JSONObject = {
 };
 
 type JSONArray = Array<JSONValue>;
-
-export type ToolOutput = {
-  tool: BaseTool | undefined;
-  // all of existing function calling LLMs only support object input
-  input: JSONObject;
-  output: JSONValue;
-  isError: boolean;
-};

packages/llamaindex/tests/llm/index.test.ts

@@ -62,5 +62,51 @@ describe("Anthropic llm", () => {
         role: "user",
       },
     ]);
+
+    expect(
+      anthropic.formatMessages([
+        {
+          content: "You are a helpful assistant.",
+          role: "assistant",
+        },
+        {
+          content: [
+            {
+              text: "What do you see in the image?",
+              type: "text",
+            },
+            {
+              type: "image_url",
+              image_url: {
+                url: `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAQDAwQDAwQEAwQFBAQFBgoHBgYGBg0JCggKDw0QEA8NDw4RExgUERIXEg4PFRwVFxkZGxsbEBQdHx0aHxgaGxr/2wBDAQQFBQYFBgwHBwwaEQ8RGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhr/wAARCAAgACADASIAAhEBAxEB/8QAGAABAAMBAAAAAAAAAAAAAAAACAQHCQb/xAAvEAABAgUCBAUDBQEAAAAAAAACAQMEBQYHERIhAAgTYSIxMkJxI2KCFBVBUVKh/8QAGAEAAwEBAAAAAAAAAAAAAAAAAwQFAQL/xAAnEQABBAECAwkAAAAAAAAAAAACAQMEEQAFMiExYRITFCJBcXKBof/aAAwDAQACEQMRAD8Aufmb5mnbWREFRdvIMZ3cWcaBh2NHUGEFwtIKQp63CX0h+S7YQgRzGSq6kgqGAS8NQRc6fmkIMWwSxJEyP+m0bwggQr5iIom6KnnxXty61jK+uJUVUxzxm/M5g5EASr6G9WGwTsIIIp2FOHJfi0kyvzS9Cv0zGwEF+2whOAUY4a6mnm2lREURLPoTggNG5tS6xpmOT4GQptwNUZc6sbexzcZRVSTKTOgudMPEL0j7E2uQNOxIqcaYcqXNaxe2HKnauBiAraDZ6n0k0tTBpPNwE9pptqDP3DtlBC1Q8qNw5K4AwLEunYkWMwcYg6fnqoH/ADPHA2/qeZWquhJJ3pODmEhmg/qGl2XAloebL5HWK/K8dOMOM7xVPfJrMhmQiq0SFXOlyPc+jIq3lwakpeYNq27K491kfvbzls07ECiSdlThhWKvj1LLx0VVLWGqSBuFJ1jc3WBEUb8K4TUieHz3xni7ea3lSZvZDhUVImxAVtBso39VdLUe0nk2a+0030n+K7YUc95/J66tRIp3SVXUpGyUI7wvPxDBoJ/UaLIuIqtuInRwiiqp4z3XbBYr3cGp9P30zJXiSjk1HLsqdIvxvzV1q8ZtB3ppa5bkwZkDz7LsF09Qxgi0Roa6UUU1LnxYH5JP74D1LUjNrkXigabc6kZM5vPFZi3NPi3dVXnFT+EQUM17IvEi1tL1xUkcEHb+lo6duvRUO644wwSDpaPWgG7sAApIKqqqm4jvxo1yvcrjdoTiqtrQ2I+u5nr19ItbUA2a5IAX3GvuP8U2ypMS5pSwFC5peTtM0lnSkMWVVUJb48a+8//Z`,
+              },
+            },
+          ],
+          role: "user",
+        },
+      ]),
+    ).toEqual([
+      {
+        content: "You are a helpful assistant.",
+        role: "assistant",
+      },
+      {
+        content: [
+          {
+            text: "What do you see in the image?",
+            type: "text",
+          },
+          {
+            source: {
+              data: "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAQDAwQDAwQEAwQFBAQFBgoHBgYGBg0JCggKDw0QEA8NDw4RExgUERIXEg4PFRwVFxkZGxsbEBQdHx0aHxgaGxr/2wBDAQQFBQYFBgwHBwwaEQ8RGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhr/wAARCAAgACADASIAAhEBAxEB/8QAGAABAAMBAAAAAAAAAAAAAAAACAQHCQb/xAAvEAABAgUCBAUDBQEAAAAAAAACAQMEBQYHERIhAAgTYSIxMkJxI2KCFBVBUVKh/8QAGAEAAwEBAAAAAAAAAAAAAAAAAwQFAQL/xAAnEQABBAECAwkAAAAAAAAAAAACAQMEEQAFMiExYRITFCJBcXKBof/aAAwDAQACEQMRAD8Aufmb5mnbWREFRdvIMZ3cWcaBh2NHUGEFwtIKQp63CX0h+S7YQgRzGSq6kgqGAS8NQRc6fmkIMWwSxJEyP+m0bwggQr5iIom6KnnxXty61jK+uJUVUxzxm/M5g5EASr6G9WGwTsIIIp2FOHJfi0kyvzS9Cv0zGwEF+2whOAUY4a6mnm2lREURLPoTggNG5tS6xpmOT4GQptwNUZc6sbexzcZRVSTKTOgudMPEL0j7E2uQNOxIqcaYcqXNaxe2HKnauBiAraDZ6n0k0tTBpPNwE9pptqDP3DtlBC1Q8qNw5K4AwLEunYkWMwcYg6fnqoH/ADPHA2/qeZWquhJJ3pODmEhmg/qGl2XAloebL5HWK/K8dOMOM7xVPfJrMhmQiq0SFXOlyPc+jIq3lwakpeYNq27K491kfvbzls07ECiSdlThhWKvj1LLx0VVLWGqSBuFJ1jc3WBEUb8K4TUieHz3xni7ea3lSZvZDhUVImxAVtBso39VdLUe0nk2a+0030n+K7YUc95/J66tRIp3SVXUpGyUI7wvPxDBoJ/UaLIuIqtuInRwiiqp4z3XbBYr3cGp9P30zJXiSjk1HLsqdIvxvzV1q8ZtB3ppa5bkwZkDz7LsF09Qxgi0Roa6UUU1LnxYH5JP74D1LUjNrkXigabc6kZM5vPFZi3NPi3dVXnFT+EQUM17IvEi1tL1xUkcEHb+lo6duvRUO644wwSDpaPWgG7sAApIKqqqm4jvxo1yvcrjdoTiqtrQ2I+u5nr19ItbUA2a5IAX3GvuP8U2ypMS5pSwFC5peTtM0lnSkMWVVUJb48a+8//Z",
+              media_type: "image/jpeg",
+              type: "base64",
+            },
+            type: "image",
+          },
+        ],
+        role: "user",
+      },
+    ]);
   });
 });

packages/llamaindex/tests/utility/mockOpenAI.ts

@@ -1,8 +1,8 @@
+import type { LLMChatParamsBase } from "llamaindex";
 import { Settings } from "llamaindex";
 import type { CallbackManager } from "llamaindex/callbacks/CallbackManager";
 import type { OpenAIEmbedding } from "llamaindex/embeddings/index";
 import { OpenAI } from "llamaindex/llm/openai";
-import type { LLMChatParamsBase } from "llamaindex/llm/types";
 import { vi } from "vitest";
 
 export const DEFAULT_LLM_TEXT_OUTPUT = "MOCK_TOKEN_1-MOCK_TOKEN_2";

pnpm-lock.yaml

@@ -383,6 +383,9 @@ importers:
         specifier: ^3.23.8
         version: 3.23.8
     devDependencies:
+      ajv:
+        specifier: ^8.16.0
+        version: 8.16.0
       bunchee:
         specifier: ^5.2.1
         version: 5.2.1(patch_hash=or7rmtlcau3uwknbkedxicrvyi)(typescript@5.5.2)
@@ -17160,8 +17163,8 @@ snapshots:
       '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2)
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0)
+      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0))(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
       eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0)
       eslint-plugin-react: 7.34.1(eslint@8.57.0)
       eslint-plugin-react-hooks: 4.6.2(eslint@8.57.0)
@@ -17206,6 +17209,23 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0))(eslint@8.57.0):
+    dependencies:
+      debug: 4.3.4
+      enhanced-resolve: 5.17.0
+      eslint: 8.57.0
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
+      fast-glob: 3.3.2
+      get-tsconfig: 4.7.5
+      is-core-module: 2.13.1
+      is-glob: 4.0.3
+    transitivePeerDependencies:
+      - '@typescript-eslint/parser'
+      - eslint-import-resolver-node
+      - eslint-import-resolver-webpack
+      - supports-color
+
   eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
     dependencies:
       debug: 4.3.4
@@ -17223,6 +17243,17 @@ snapshots:
       - eslint-import-resolver-webpack
       - supports-color
 
+  eslint-module-utils@2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
+    dependencies:
+      debug: 3.2.7
+    optionalDependencies:
+      '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2)
+      eslint: 8.57.0
+      eslint-import-resolver-node: 0.3.9
+      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0))(eslint@8.57.0)
+    transitivePeerDependencies:
+      - supports-color
+
   eslint-module-utils@2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
       debug: 3.2.7
@@ -17234,14 +17265,31 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-module-utils@2.8.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0):
+  eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
+      array-includes: 3.1.8
+      array.prototype.findlastindex: 1.2.5
+      array.prototype.flat: 1.3.2
+      array.prototype.flatmap: 1.3.2
       debug: 3.2.7
-    optionalDependencies:
-      '@typescript-eslint/parser': 7.8.0(eslint@8.57.0)(typescript@5.5.2)
+      doctrine: 2.1.0
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
+      hasown: 2.0.2
+      is-core-module: 2.13.1
+      is-glob: 4.0.3
+      minimatch: 3.1.2
+      object.fromentries: 2.0.8
+      object.groupby: 1.0.3
+      object.values: 1.2.0
+      semver: 6.3.1
+      tsconfig-paths: 3.15.0
+    optionalDependencies:
+      '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2)
     transitivePeerDependencies:
+      - eslint-import-resolver-typescript
+      - eslint-import-resolver-webpack
       - supports-color
 
   eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint@8.57.0):
@@ -17254,7 +17302,7 @@ snapshots:
       doctrine: 2.1.0
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.8.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0)
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
       hasown: 2.0.2
       is-core-module: 2.13.1
       is-glob: 4.0.3