Release 0.4.10 (#1003)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

README.md

@@ -76,7 +76,7 @@ main();
 node --import tsx ./main.ts
 ```
 
-### Next.js
+### React Server Component (Next.js, Waku, Redwood.JS...)
 
 First, you will need to add a llamaindex plugin to your Next.js project.
 
@@ -154,40 +154,6 @@ export async function chatWithAgent(
 }
 ```
 
-### Cloudflare Workers
-
-```ts
-// src/index.ts
-export default {
-  async fetch(
-    request: Request,
-    env: Env,
-    ctx: ExecutionContext,
-  ): Promise<Response> {
-    const { setEnvs } = await import("@llamaindex/env");
-    // set environment variables so that the OpenAIAgent can use them
-    setEnvs(env);
-    const { OpenAIAgent } = await import("llamaindex");
-    const agent = new OpenAIAgent({
-      tools: [],
-    });
-    const responseStream = await agent.chat({
-      stream: true,
-      message: "Hello? What is the weather today?",
-    });
-    const textEncoder = new TextEncoder();
-    const response = responseStream.pipeThrough(
-      new TransformStream({
-        transform: (chunk, controller) => {
-          controller.enqueue(textEncoder.encode(chunk.response.delta));
-        },
-      }),
-    );
-    return new Response(response);
-  },
-};
-```
-
 ## Playground
 
 Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground

apps/docs/CHANGELOG.md

@@ -1,5 +1,35 @@
 # docs
 
+## 0.0.36
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.34
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.33
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.32
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.32",
+  "version": "0.0.36",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/README.md

@@ -9,7 +9,7 @@ make sure you have basic knowledge of the [LlamaIndexTS](https://ts.llamaindex.a
 # export your API key
 export OPENAI_API_KEY="sk-..."
 
-npx ts-node ./chatEngine.ts
+npx tsx ./chatEngine.ts
 ```
 
 ## Build your own RAG app

examples/agent/azure_dynamic_session.ts

@@ -0,0 +1,54 @@
+import "dotenv/config";
+
+import {
+  DefaultAzureCredential,
+  getBearerTokenProvider,
+} from "@azure/identity";
+import { AzureDynamicSessionTool, OpenAI, ReActAgent } from "llamaindex";
+
+async function main() {
+  const credential = new DefaultAzureCredential();
+  const azureADTokenProvider = getBearerTokenProvider(
+    credential,
+    "https://cognitiveservices.azure.com/.default",
+  );
+
+  const azure = {
+    azureADTokenProvider,
+    deployment: process.env.AZURE_OPENAI_DEPLOYMENT ?? "gpt-35-turbo",
+  };
+
+  // configure LLM model
+  const llm = new OpenAI({
+    azure,
+  });
+
+  const azureDynamicSession = new AzureDynamicSessionTool();
+
+  // Create an ReActAgent with the azure dynamic session tool
+  const agent = new ReActAgent({
+    llm,
+    tools: [azureDynamicSession],
+    // verbose: true,
+    systemPrompt: `You are a Python interpreter.
+      - You are given tasks to complete and you run python code to solve them.
+      - The python code runs by the python runtime. Every time you call $(interpreter) tool, the python code is executed in a separate cell. It's okay to make multiple calls to $(interpreter).
+      - You can run any python code you want in a secure environment.
+      - For images, return the full URL, not the base64 data.
+      - Return any image content as an HTML tag with the src attribute set to the URL of the image.`,
+  });
+
+  // Chat with the agent
+  const response = await agent.chat({
+    message:
+      "plot a chart of 5 random numbers and save it to /mnt/data/chart.png",
+    stream: false,
+  });
+
+  // Print the response
+  console.log({ response });
+}
+
+void main().then(() => {
+  console.log("Done");
+});

examples/astradb/README.md

@@ -24,7 +24,7 @@ Here are two sample scripts which work well with the sample data in the Astra Po
 
 Loads and queries a simple vectorstore with some documents about Astra DB
 
-run `ts-node astradb/example`
+run `tsx astradb/example`
 
 ## Movie Reviews Example
 
@@ -32,10 +32,10 @@ run `ts-node astradb/example`
 
 This sample loads the same dataset of movie reviews as the Astra Portal sample dataset. (Feel free to load the data in your the Astra Data Explorer to compare)
 
-run `npx ts-node astradb/load`
+run `npx tsx astradb/load`
 
 ### Use RAG to Query the data
 
 Check out your data in the Astra Data Explorer and change the sample query as you see fit.
 
-run `npx ts-node astradb/query`
+run `npx tsx astradb/query`

examples/chromadb/README.md

@@ -6,7 +6,7 @@ Export your OpenAI API Key using `export OPEN_API_KEY=insert your api key here`
 
 If you haven't installed chromadb, run `pip install chromadb`. Start the server using `chroma run`.
 
-Now, open a new terminal window and inside `examples`, run `pnpm dlx ts-node chromadb/test.ts`.
+Now, open a new terminal window and inside `examples`, run `pnpm dlx tsx chromadb/test.ts`.
 
 Here's the output for the input query `Tell me about Godfrey Cheshire's rating of La Sapienza.`:
 

examples/cloud/README.md

@@ -21,7 +21,7 @@ export LLAMA_CLOUD_BASE_URL="https://api.staging.llamaindex.ai"
 This example is using the managed index named `test` from the project `default` to create a chat engine.
 
 ```shell
-pnpx ts-node cloud/chat.ts
+pnpx tsx cloud/chat.ts
 ```
 
 ## Query Engine
@@ -29,7 +29,7 @@ pnpx ts-node cloud/chat.ts
 This example shows how to use the managed index with a query engine.
 
 ```shell
-pnpx ts-node cloud/query.ts
+pnpx tsx cloud/query.ts
 ```
 
 ## Pipeline
@@ -37,5 +37,5 @@ pnpx ts-node cloud/query.ts
 This example shows how to create a managed index with a pipeline.
 
 ```shell
-pnpx ts-node cloud/pipeline.ts
+pnpx tsx cloud/pipeline.ts
 ```

examples/cloud/chat.ts

@@ -6,7 +6,7 @@ import { ContextChatEngine, LlamaCloudIndex } from "llamaindex";
 async function main() {
   const index = new LlamaCloudIndex({
     name: "test",
-    projectName: "default",
+    projectName: "Default",
     baseUrl: process.env.LLAMA_CLOUD_BASE_URL,
     apiKey: process.env.LLAMA_CLOUD_API_KEY,
   });
@@ -19,10 +19,10 @@ async function main() {
   while (true) {
     const query = await rl.question("User: ");
     const stream = await chatEngine.chat({ message: query, stream: true });
-    console.log();
     for await (const chunk of stream) {
       process.stdout.write(chunk.response);
     }
+    process.stdout.write("\n");
   }
 }
 

examples/milvus/README.md

@@ -25,10 +25,10 @@ Here are two sample scripts which work with loading and querying data from a Mil
 
 This sample loads the same dataset of movie reviews as sample dataset. You can install https://github.com/zilliztech/attu to inspect the loaded data.
 
-run `npx ts-node milvus/load`
+run `npx tsx milvus/load`
 
 ## Use RAG to Query the data
 
 Check out your data in Attu and change the sample query as you see fit.
 
-run `npx ts-node milvus/query`
+run `npx tsx milvus/query`

examples/mongodb/README.md

@@ -34,7 +34,7 @@ MONGODB_COLLECTION=tiny_tweets_collection
 You are now ready to import our ready-made data set into Mongo. This is the file `tinytweets.json`, a selection of approximately 1000 tweets from @seldo on Twitter in mid-2019. With your environment set up you can do this by running
 
 ```
-npx ts-node mongodb/1_import.ts
+npx tsx mongodb/1_import.ts
 ```
 
 If you don't want to use tweets, you can replace `json_file` with any other array of JSON objects, but you will need to modify some code later to make sure the correct field gets indexed. There is no LlamaIndex-specific code here; you can load your data into Mongo any way you want to.
@@ -59,7 +59,7 @@ MONGODB_VECTOR_INDEX=tiny_tweets_vector_index
 If the data you're indexing is the tweets we gave you, you're ready to go:
 
 ```bash
-npx ts-node mongodb/2_load_and_index.ts
+npx tsx mongodb/2_load_and_index.ts
 ```
 
 > Note: this script is running a couple of minutes and currently doesn't show any progress.
@@ -112,7 +112,7 @@ Now you're ready to query your data!
 You can do this by running
 
 ```bash
-npx ts-node mongodb/3_query.ts
+npx tsx mongodb/3_query.ts
 ```
 
 This sets up a connection to Atlas just like `2_load_and_index.ts` did, then it creates a [query engine](https://docs.llamaindex.ai/en/stable/understanding/querying/querying.html#getting-started) and runs a query against it.

examples/package.json

@@ -4,6 +4,7 @@
   "version": "0.0.6",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
+    "@azure/identity": "^4.2.1",
     "@datastax/astra-db-ts": "^1.2.1",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^2.2.2",
@@ -18,7 +19,6 @@
   },
   "devDependencies": {
     "@types/node": "^20.14.1",
-    "ts-node": "^10.9.2",
     "tsx": "^4.15.6",
     "typescript": "^5.5.2"
   },

examples/pg-vector-store/README.md

@@ -37,7 +37,7 @@ Read and follow the instructions in the README.md file located one directory up
 
 To import documents and save the embedding vectors to your database:
 
-> `npx ts-node pg-vector-store/load-docs.ts data`
+> `npx tsx pg-vector-store/load-docs.ts data`
 
 where data is the directory containing your input files. Using the `data` directory in the example above will read all of the files in that directory using the LlamaIndexTS default readers for each file type.
 
@@ -45,6 +45,6 @@ where data is the directory containing your input files. Using the `data` direct
 
 To query using the resulting vector store:
 
-> `npx ts-node pg-vector-store/query.ts`
+> `npx tsx pg-vector-store/query.ts`
 
 The script will prompt for a question, then process and present the answer using the PGVectorStore data and your OpenAI API key. It will continue to prompt until you enter `q`, `quit` or `exit` as the next query.

examples/pinecone-vector-store/README.md

@@ -19,7 +19,7 @@ Read and follow the instructions in the README.md file located one directory up
 
 To import documents and save the embedding vectors to your database:
 
-> `npx ts-node pinecone-vector-store/load-docs.ts data`
+> `npx tsx pinecone-vector-store/load-docs.ts data`
 
 where data is the directory containing your input files. Using the _data_ directory in the example above will read all of the files in that directory using the llamaindexTS default readers for each file type.
 
@@ -29,6 +29,6 @@ where data is the directory containing your input files. Using the _data_ direct
 
 To query using the resulting vector store:
 
-> `npx ts-node pinecone-vector-store/query.ts`
+> `npx tsx pinecone-vector-store/query.ts`
 
 The script will prompt for a question, then process and present the answer using the PineconeVectorStore data and your OpenAI API key. It will continue to prompt until you enter `q`, `quit` or `exit` as the next query.

examples/qdrantdb/README.md

@@ -8,4 +8,4 @@ Add your OpenAI API Key into a file called `.env` in the parent folder of this d
 OPEN_API_KEY=sk-you-key
 ```
 
-Now, open a new terminal window and inside `examples`, run `npx ts-node qdrantdb/preFilters.ts`.
+Now, open a new terminal window and inside `examples`, run `npx tsx qdrantdb/preFilters.ts`.

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,39 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.20
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.19
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.18
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.17
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+  - @llamaindex/autotool@1.0.0
+
 ## 0.1.16
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.16",
+  "version": "0.1.20",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.4.6",
+    "llamaindex": "^0.4.10",
     "openai": "^4",
     "typescript": "^4"
   },

packages/cloud/CHANGELOG.md

@@ -0,0 +1,7 @@
+# @llamaindex/cloud
+
+## 0.1.1
+
+### Patch Changes
+
+- 321c39d: fix: generate api as class

packages/cloud/README.md

@@ -5,7 +5,7 @@
 ## Usage
 
 ```ts
-import { OpenAPI, Service } from "@llamaindex/cloud/api";
+import { OpenAPI } from "@llamaindex/cloud/api";
 OpenAPI.TOKEN = "YOUR_API_KEY";
 OpenAPI.BASE = "https://api.cloud.llamaindex.ai/";
 // ...

packages/cloud/openapi-ts.config.ts

@@ -9,6 +9,9 @@ export default defineConfig({
     format: "prettier",
     lint: "eslint",
   },
+  services: {
+    asClass: true,
+  },
   types: {
     enums: "javascript",
   },

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/cloud/src/api.ts

@@ -1,4 +1 @@
-import * as Service from "./client/services.gen";
-
 export * from "./client";
-export { Service };

packages/community/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/community
 
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.10
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.10",
+  "version": "0.0.14",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -0,0 +1,9 @@
+# @llamaindex/core
+
+## 0.0.2
+
+### Patch Changes
+
+- f10b41d: fix: release files
+- Updated dependencies [41fe871]
+  - @llamaindex/env@0.1.7

packages/core/package.json

@@ -1,9 +1,23 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "LlamaIndex Core Module",
   "exports": {
+    "./llms": {
+      "require": {
+        "types": "./dist/llms/index.d.cts",
+        "default": "./dist/llms/index.cjs"
+      },
+      "import": {
+        "types": "./dist/llms/index.d.ts",
+        "default": "./dist/llms/index.js"
+      },
+      "default": {
+        "types": "./dist/llms/index.d.ts",
+        "default": "./dist/llms/index.js"
+      }
+    },
     "./decorator": {
       "require": {
         "types": "./dist/decorator/index.d.cts",
@@ -47,6 +61,9 @@
       }
     }
   },
+  "files": [
+    "dist"
+  ],
   "scripts": {
     "dev": "bunchee --watch",
     "build": "bunchee"
@@ -57,6 +74,7 @@
     "url": "https://github.com/himself65/LlamaIndexTS.git"
   },
   "devDependencies": {
+    "ajv": "^8.16.0",
     "bunchee": "^5.2.1"
   },
   "dependencies": {

packages/core/src/global/type.ts

@@ -0,0 +1,9 @@
+export type UUID = `${string}-${string}-${string}-${string}-${string}`;
+
+export type JSONValue = string | number | boolean | JSONObject | JSONArray;
+
+export type JSONObject = {
+  [key: string]: JSONValue;
+};
+
+export type JSONArray = Array<JSONValue>;

packages/core/src/index.ts

@@ -1 +0,0 @@
-export * from "./schema";

packages/core/src/llms/index.ts

@@ -0,0 +1,31 @@
+export type {
+  BaseTool,
+  BaseToolWithCall,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChat,
+  LLMChatParamsBase,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsBase,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+  MessageContent,
+  MessageContentDetail,
+  MessageContentImageDetail,
+  MessageContentTextDetail,
+  MessageType,
+  PartialToolCall,
+  TextChatMessage,
+  ToolCall,
+  ToolCallLLMMessageOptions,
+  ToolCallOptions,
+  ToolMetadata,
+  ToolOutput,
+  ToolResult,
+  ToolResultOptions,
+} from "./type";

packages/core/src/llms/type.ts

@@ -0,0 +1,245 @@
+import type { Tokenizers } from "@llamaindex/env";
+import type { JSONSchemaType } from "ajv";
+import type { JSONObject, JSONValue } from "../global/type";
+
+/**
+ * @internal
+ */
+export interface LLMChat<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> {
+  chat(
+    params:
+      | LLMChatParamsStreaming<AdditionalChatOptions>
+      | LLMChatParamsNonStreaming<AdditionalChatOptions>,
+  ): Promise<
+    | ChatResponse<AdditionalMessageOptions>
+    | AsyncIterable<ChatResponseChunk<AdditionalMessageOptions>>
+  >;
+}
+
+/**
+ * Unified language model interface
+ */
+export interface LLM<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChat<AdditionalChatOptions> {
+  metadata: LLMMetadata;
+  /**
+   * Get a chat response from the LLM
+   */
+  chat(
+    params: LLMChatParamsStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(
+    params: LLMChatParamsNonStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
+  ): Promise<ChatResponse<AdditionalMessageOptions>>;
+
+  /**
+   * Get a prompt completion from the LLM
+   */
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+}
+
+export type MessageType = "user" | "assistant" | "system" | "memory";
+
+export type TextChatMessage<AdditionalMessageOptions extends object = object> =
+  {
+    content: string;
+    role: MessageType;
+    options?: undefined | AdditionalMessageOptions;
+  };
+
+export type ChatMessage<AdditionalMessageOptions extends object = object> = {
+  content: MessageContent;
+  role: MessageType;
+  options?: undefined | AdditionalMessageOptions;
+};
+
+export interface ChatResponse<
+  AdditionalMessageOptions extends object = object,
+> {
+  message: ChatMessage<AdditionalMessageOptions>;
+  /**
+   * Raw response from the LLM
+   *
+   * If LLM response an iterable of chunks, this will be an array of those chunks
+   */
+  raw: object | null;
+}
+
+export type ChatResponseChunk<
+  AdditionalMessageOptions extends object = object,
+> = {
+  raw: object | null;
+  delta: string;
+  options?: undefined | AdditionalMessageOptions;
+};
+
+export interface CompletionResponse {
+  text: string;
+  /**
+   * Raw response from the LLM
+   *
+   * It's possible that this is `null` if the LLM response an iterable of chunks
+   */
+  raw: object | null;
+}
+
+export type LLMMetadata = {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  contextWindow: number;
+  tokenizer: Tokenizers | undefined;
+};
+
+export interface LLMChatParamsBase<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> {
+  messages: ChatMessage<AdditionalMessageOptions>[];
+  additionalChatOptions?: AdditionalChatOptions;
+  tools?: BaseTool[];
+}
+
+export interface LLMChatParamsStreaming<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
+  stream: true;
+}
+
+export interface LLMChatParamsNonStreaming<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
+  stream?: false;
+}
+
+export interface LLMCompletionParamsBase {
+  prompt: MessageContent;
+}
+
+export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
+  stream: true;
+}
+
+export interface LLMCompletionParamsNonStreaming
+  extends LLMCompletionParamsBase {
+  stream?: false | null;
+}
+
+export type MessageContentTextDetail = {
+  type: "text";
+  text: string;
+};
+
+export type MessageContentImageDetail = {
+  type: "image_url";
+  image_url: { url: string };
+};
+
+export type MessageContentDetail =
+  | MessageContentTextDetail
+  | MessageContentImageDetail;
+
+/**
+ * Extended type for the content of a message that allows for multi-modal messages.
+ */
+export type MessageContent = string | MessageContentDetail[];
+
+export type ToolCall = {
+  name: string;
+  input: JSONObject;
+  id: string;
+};
+
+// happened in streaming response, the tool call is not ready yet
+export type PartialToolCall = {
+  name: string;
+  id: string;
+  // input is not ready yet, JSON.parse(input) will throw an error
+  input: string;
+};
+
+export type ToolResult = {
+  id: string;
+  result: string;
+  isError: boolean;
+};
+
+export type ToolCallOptions = {
+  toolCall: (ToolCall | PartialToolCall)[];
+};
+
+export type ToolResultOptions = {
+  toolResult: ToolResult;
+};
+
+export type ToolCallLLMMessageOptions =
+  | ToolResultOptions
+  | ToolCallOptions
+  | {};
+
+type Known =
+  | { [key: string]: Known }
+  | [Known, ...Known[]]
+  | Known[]
+  | number
+  | string
+  | boolean
+  | null;
+
+export type ToolMetadata<
+  Parameters extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  description: string;
+  name: string;
+  /**
+   * OpenAI uses JSON Schema to describe the parameters that a tool can take.
+   * @link https://json-schema.org/understanding-json-schema
+   */
+  parameters?: Parameters;
+};
+
+/**
+ * Simple Tool interface. Likely to change.
+ */
+export interface BaseTool<Input = any> {
+  /**
+   * This could be undefined if the implementation is not provided,
+   *  which might be the case when communicating with a llm.
+   *
+   * @return {JSONValue | Promise<JSONValue>} The output of the tool.
+   */
+  call?: (input: Input) => JSONValue | Promise<JSONValue>;
+  metadata: // if user input any, we cannot check the schema
+  Input extends Known ? ToolMetadata<JSONSchemaType<Input>> : ToolMetadata;
+}
+
+export type BaseToolWithCall<Input = any> = Omit<BaseTool<Input>, "call"> & {
+  call: NonNullable<Pick<BaseTool<Input>, "call">["call"]>;
+};
+
+export type ToolOutput = {
+  tool: BaseTool | undefined;
+  // all of existing function calling LLMs only support object input
+  input: JSONObject;
+  output: JSONValue;
+  isError: boolean;
+};

packages/core/src/schema/zod.ts

@@ -1,5 +1,7 @@
 import { z } from "zod";
 
+export const anyFunctionSchema = z.function(z.tuple([]).rest(z.any()), z.any());
+
 export const toolMetadataSchema = z.object({
   description: z.string(),
   name: z.string(),
@@ -7,7 +9,7 @@ export const toolMetadataSchema = z.object({
 });
 
 export const baseToolSchema = z.object({
-  call: z.optional(z.function()),
+  call: anyFunctionSchema.optional(),
   metadata: toolMetadataSchema,
 });
 

packages/env/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/env
 
+## 0.1.7
+
+### Patch Changes
+
+- 41fe871: Add support for azure dynamic session tool
+
 ## 0.1.6
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/env/src/fs/node.ts

@@ -5,6 +5,7 @@
  *
  * @module
  */
+import { createWriteStream } from "node:fs";
 import fs from "node:fs/promises";
 
-export { fs };
+export { createWriteStream, fs };

packages/env/src/index.ts

@@ -15,12 +15,14 @@ import { ok } from "node:assert";
 import { createHash, randomUUID } from "node:crypto";
 import { EOL } from "node:os";
 import path from "node:path";
+import { Readable } from "node:stream";
 import {
   ReadableStream,
   TransformStream,
   WritableStream,
 } from "node:stream/web";
-import { fs } from "./fs/node.js";
+import { fileURLToPath } from "node:url";
+import { createWriteStream, fs } from "./fs/node.js";
 import type { SHA256 } from "./polyfill.js";
 
 export function createSHA256(): SHA256 {
@@ -38,11 +40,14 @@ export function createSHA256(): SHA256 {
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js";
 export { AsyncLocalStorage, CustomEvent, getEnv, setEnvs } from "./utils.js";
 export {
+  createWriteStream,
   EOL,
+  fileURLToPath,
   fs,
   ok,
   path,
   randomUUID,
+  Readable,
   ReadableStream,
   TransformStream,
   WritableStream,

packages/experimental/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/experimental
 
+## 0.0.45
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.43
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.42
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.41
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.41",
+  "version": "0.0.45",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,37 @@
 # llamaindex
 
+## 0.4.10
+
+### Patch Changes
+
+- 7dce3d2: fix: disable External Filters for Gemini
+
+## 0.4.9
+
+### Patch Changes
+
+- 3a96a48: fix: anthroipic image input
+
+## 0.4.8
+
+### Patch Changes
+
+- 83ebdfb: fix: next.js build error
+
+## 0.4.7
+
+### Patch Changes
+
+- 41fe871: Add support for azure dynamic session tool
+- 321c39d: fix: generate api as class
+- f7f1af0: fix: throw error when no pipeline found
+- Updated dependencies [41fe871]
+- Updated dependencies [f10b41d]
+- Updated dependencies [321c39d]
+  - @llamaindex/env@0.1.7
+  - @llamaindex/core@0.0.2
+  - @llamaindex/cloud@0.1.1
+
 ## 0.4.6
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.25",
+  "version": "0.0.29",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/next-agent-test
 
+## 0.1.29
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.1.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.1.27
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.1.26
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.1.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.25",
+  "version": "0.1.29",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-agent/src/actions/index.tsx

@@ -1,7 +1,7 @@
 "use server";
 import { createStreamableUI } from "ai/rsc";
+import type { ChatMessage } from "llamaindex";
 import { OpenAIAgent } from "llamaindex";
-import type { ChatMessage } from "llamaindex/llm/types";
 
 export async function chatWithAgent(
   question: string,

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,35 @@
 # test-edge-runtime
 
+## 0.1.28
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.1.27
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.1.26
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.1.25
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.1.24
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.24",
+  "version": "0.1.28",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,36 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.10
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.9
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.8
+
+### Patch Changes
+
+- 83ebdfb: fix: next.js build error
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.7
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.6
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.6",
+  "version": "0.0.10",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/src/actions/openai.ts

@@ -0,0 +1,68 @@
+"use server";
+import {
+  OpenAI,
+  OpenAIAgent,
+  QueryEngineTool,
+  Settings,
+  VectorStoreIndex,
+} from "llamaindex";
+import { HuggingFaceEmbedding } from "llamaindex/embeddings/HuggingFaceEmbedding";
+import { SimpleDirectoryReader } from "llamaindex/readers/SimpleDirectoryReader";
+
+Settings.llm = new OpenAI({
+  // eslint-disable-next-line turbo/no-undeclared-env-vars
+  apiKey: process.env.NEXT_PUBLIC_OPENAI_KEY ?? "FAKE_KEY_TO_PASS_TESTS",
+  model: "gpt-4o",
+});
+Settings.embedModel = new HuggingFaceEmbedding({
+  modelType: "BAAI/bge-small-en-v1.5",
+  quantized: false,
+});
+Settings.callbackManager.on("llm-tool-call", (event) => {
+  console.log(event.detail.payload);
+});
+Settings.callbackManager.on("llm-tool-result", (event) => {
+  console.log(event.detail.payload);
+});
+
+export async function getOpenAIModelRequest(query: string) {
+  try {
+    const currentDir = __dirname;
+
+    // load our data and create a query engine
+    const reader = new SimpleDirectoryReader();
+    const documents = await reader.loadData(currentDir);
+    const index = await VectorStoreIndex.fromDocuments(documents);
+    const retriever = index.asRetriever({
+      similarityTopK: 10,
+    });
+    const queryEngine = index.asQueryEngine({
+      retriever,
+    });
+
+    // define the query engine as a tool
+    const tools = [
+      new QueryEngineTool({
+        queryEngine: queryEngine,
+        metadata: {
+          name: "deployment_details_per_env",
+          description: `This tool can answer detailed questions about deployments happened in various environments.`,
+        },
+      }),
+    ];
+    // create the agent
+    const agent = new OpenAIAgent({ tools });
+
+    const { response } = await agent.chat({
+      message: query,
+    });
+    return {
+      message: response,
+    };
+  } catch (err) {
+    console.error(err);
+    return {
+      errors: "Error Calling OpenAI Model",
+    };
+  }
+}

packages/llamaindex/e2e/examples/nextjs-node-runtime/src/app/api/openai/route.ts

@@ -0,0 +1,10 @@
+import { getOpenAIModelRequest } from "@/actions/openai";
+import { NextRequest, NextResponse } from "next/server";
+
+// POST /api/openai
+export async function POST(request: NextRequest) {
+  const body = await request.json();
+  const content = await getOpenAIModelRequest(body.query);
+
+  return NextResponse.json(content, { status: 200 });
+}

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.25",
+  "version": "0.0.29",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/fixtures/llm/openai.ts

@@ -7,7 +7,7 @@ import type {
   LLMChatParamsStreaming,
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
-} from "llamaindex/llm/types";
+} from "llamaindex";
 import { extractText } from "llamaindex/llm/utils";
 import { deepStrictEqual, strictEqual } from "node:assert";
 import { llmCompleteMockStorage } from "../../node/utils.js";

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.4.6",
+  "version": "0.4.10",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -22,6 +22,7 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.21.1",
     "@aws-crypto/sha256-js": "^5.2.0",
+    "@azure/identity": "^4.2.1",
     "@datastax/astra-db-ts": "^1.2.1",
     "@google-cloud/vertexai": "^1.2.0",
     "@google/generative-ai": "^0.12.0",
@@ -33,7 +34,8 @@
     "@mistralai/mistralai": "^0.4.0",
     "@pinecone-database/pinecone": "^2.2.2",
     "@qdrant/js-client-rest": "^1.9.0",
-    "@types/lodash": "^4.17.5",
+    "@types/lodash": "^4.17.4",
+    "@types/node": "^20.14.5",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.6",
     "@xenova/transformers": "^2.17.2",

packages/llamaindex/src/ChatHistory.ts

@@ -1,8 +1,8 @@
+import type { ChatMessage, LLM, MessageType } from "@llamaindex/core/llms";
 import { tokenizers, type Tokenizer } from "@llamaindex/env";
 import type { SummaryPrompt } from "./Prompt.js";
 import { defaultSummaryPrompt, messagesToHistoryStr } from "./Prompt.js";
 import { OpenAI } from "./llm/openai.js";
-import type { ChatMessage, LLM, MessageType } from "./llm/types.js";
 import { extractText } from "./llm/utils.js";
 
 /**

packages/llamaindex/src/EngineResponse.ts

@@ -1,9 +1,9 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
 import type {
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
-} from "./llm/types.js";
+} from "@llamaindex/core/llms";
+import type { NodeWithScore } from "@llamaindex/core/schema";
 import { extractText } from "./llm/utils.js";
 
 export class EngineResponse implements ChatResponse, ChatResponseChunk {

packages/llamaindex/src/Prompt.ts

@@ -1,6 +1,5 @@
+import type { ChatMessage, ToolMetadata } from "@llamaindex/core/llms";
 import type { SubQuestion } from "./engines/query/types.js";
-import type { ChatMessage } from "./llm/types.js";
-import type { ToolMetadata } from "./types.js";
 
 /**
  * A SimplePrompt is a function that takes a dictionary of inputs and returns a string.

packages/llamaindex/src/QuestionGenerator.ts

@@ -1,3 +1,4 @@
+import type { LLM, ToolMetadata } from "@llamaindex/core/llms";
 import { SubQuestionOutputParser } from "./OutputParser.js";
 import type { SubQuestionPrompt } from "./Prompt.js";
 import { buildToolsText, defaultSubQuestionPrompt } from "./Prompt.js";
@@ -6,13 +7,8 @@ import type {
   SubQuestion,
 } from "./engines/query/types.js";
 import { OpenAI } from "./llm/openai.js";
-import type { LLM } from "./llm/types.js";
 import { PromptMixin } from "./prompts/index.js";
-import type {
-  BaseOutputParser,
-  StructuredOutput,
-  ToolMetadata,
-} from "./types.js";
+import type { BaseOutputParser, StructuredOutput } from "./types.js";
 
 /**
  * LLMQuestionGenerator uses the LLM to generate new questions for the LLM using tools and a user query.

packages/llamaindex/src/ServiceContext.ts

@@ -1,8 +1,8 @@
+import type { LLM } from "@llamaindex/core/llms";
 import { PromptHelper } from "./PromptHelper.js";
 import { OpenAIEmbedding } from "./embeddings/OpenAIEmbedding.js";
 import type { BaseEmbedding } from "./embeddings/types.js";
 import { OpenAI } from "./llm/openai.js";
-import type { LLM } from "./llm/types.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 import type { NodeParser } from "./nodeParsers/types.js";
 

packages/llamaindex/src/Settings.ts

@@ -5,6 +5,7 @@ import { OpenAI } from "./llm/openai.js";
 import { PromptHelper } from "./PromptHelper.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 
+import type { LLM } from "@llamaindex/core/llms";
 import { AsyncLocalStorage, getEnv } from "@llamaindex/env";
 import type { ServiceContext } from "./ServiceContext.js";
 import type { BaseEmbedding } from "./embeddings/types.js";
@@ -18,7 +19,6 @@ import {
   setEmbeddedModel,
   withEmbeddedModel,
 } from "./internal/settings/EmbedModel.js";
-import type { LLM } from "./llm/types.js";
 import type { NodeParser } from "./nodeParsers/types.js";
 
 export type PromptConfig = {

packages/llamaindex/src/agent/base.ts

@@ -1,3 +1,10 @@
+import type {
+  BaseToolWithCall,
+  ChatMessage,
+  LLM,
+  MessageContent,
+  ToolOutput,
+} from "@llamaindex/core/llms";
 import { ReadableStream, TransformStream, randomUUID } from "@llamaindex/env";
 import { ChatHistory } from "../ChatHistory.js";
 import { EngineResponse } from "../EngineResponse.js";
@@ -11,9 +18,7 @@ import { wrapEventCaller } from "../internal/context/EventCaller.js";
 import { consoleLogger, emptyLogger } from "../internal/logger.js";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import { isAsyncIterable } from "../internal/utils.js";
-import type { ChatMessage, LLM, MessageContent } from "../llm/index.js";
 import { ObjectRetriever } from "../objects/index.js";
-import type { BaseToolWithCall, ToolOutput } from "../types.js";
 import type {
   AgentTaskContext,
   TaskHandler,
@@ -229,13 +234,12 @@ export abstract class AgentRunner<
     const { llm, getTools, stream } = step.context;
     const lastMessage = step.context.store.messages.at(-1)!.content;
     const tools = await getTools(lastMessage);
-    const response = await llm.chat({
-      // @ts-expect-error
-      stream,
-      tools,
-      messages: [...step.context.store.messages],
-    });
     if (!stream) {
+      const response = await llm.chat({
+        stream,
+        tools,
+        messages: [...step.context.store.messages],
+      });
       await stepTools<LLM>({
         response,
         tools,
@@ -243,6 +247,11 @@ export abstract class AgentRunner<
         enqueueOutput,
       });
     } else {
+      const response = await llm.chat({
+        stream,
+        tools,
+        messages: [...step.context.store.messages],
+      });
       await stepToolsStreaming<LLM>({
         response,
         tools,

packages/llamaindex/src/agent/llm.ts

@@ -1,7 +1,6 @@
-import type { LLM } from "../llm/index.js";
+import type { BaseToolWithCall, LLM } from "@llamaindex/core/llms";
 import { ObjectRetriever } from "../objects/index.js";
 import { Settings } from "../Settings.js";
-import type { BaseToolWithCall } from "../types.js";
 import { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
 import { validateAgentParams } from "./utils.js";
 

packages/llamaindex/src/agent/react.ts

@@ -1,18 +1,19 @@
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLM,
+} from "@llamaindex/core/llms";
 import { randomUUID, ReadableStream } from "@llamaindex/env";
 import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
 import {
   isAsyncIterable,
   stringifyJSONToMessageContent,
 } from "../internal/utils.js";
-import {
-  type ChatMessage,
-  type ChatResponse,
-  type ChatResponseChunk,
-  type LLM,
-} from "../llm/index.js";
 import { extractText } from "../llm/utils.js";
 import { Settings } from "../Settings.js";
-import type { BaseTool, JSONObject, JSONValue } from "../types.js";
+import type { JSONObject, JSONValue } from "../types.js";
 import { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
 import type { TaskHandler } from "./types.js";
 import {

packages/llamaindex/src/agent/types.ts

@@ -1,14 +1,16 @@
-import { ReadableStream } from "@llamaindex/env";
-import type { Logger } from "../internal/logger.js";
-import type { BaseEvent } from "../internal/type.js";
 import type {
+  BaseToolWithCall,
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLM,
   MessageContent,
-} from "../llm/types.js";
-import type { BaseToolWithCall, ToolOutput, UUID } from "../types.js";
+  ToolOutput,
+} from "@llamaindex/core/llms";
+import { ReadableStream } from "@llamaindex/env";
+import type { Logger } from "../internal/logger.js";
+import type { BaseEvent } from "../internal/type.js";
+import type { UUID } from "../types.js";
 
 export type AgentTaskContext<
   Model extends LLM,

packages/llamaindex/src/agent/utils.ts

@@ -1,3 +1,15 @@
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLM,
+  PartialToolCall,
+  TextChatMessage,
+  ToolCall,
+  ToolCallLLMMessageOptions,
+  ToolOutput,
+} from "@llamaindex/core/llms";
 import { baseToolWithCallSchema } from "@llamaindex/core/schema";
 import { ReadableStream } from "@llamaindex/env";
 import { z } from "zod";
@@ -8,17 +20,7 @@ import {
   prettifyError,
   stringifyJSONToMessageContent,
 } from "../internal/utils.js";
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-  LLM,
-  PartialToolCall,
-  TextChatMessage,
-  ToolCall,
-  ToolCallLLMMessageOptions,
-} from "../llm/index.js";
-import type { BaseTool, JSONObject, JSONValue, ToolOutput } from "../types.js";
+import type { JSONObject, JSONValue } from "../types.js";
 import type { AgentParamsBase } from "./base.js";
 import type { TaskHandler } from "./types.js";
 
@@ -31,10 +33,12 @@ type StepToolsResponseParams<Model extends LLM> = {
   >[1];
 };
 
-type StepToolsStreamingResponseParams<Model extends LLM> =
-  StepToolsResponseParams<Model> & {
-    response: AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>;
-  };
+type StepToolsStreamingResponseParams<Model extends LLM> = Omit<
+  StepToolsResponseParams<Model>,
+  "response"
+> & {
+  response: AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>;
+};
 
 // #TODO stepTools and stepToolsStreaming should be moved to a better abstraction
 
@@ -83,7 +87,7 @@ export async function stepToolsStreaming<Model extends LLM>({
       }
     }
 
-    // If there are toolCalls but they didn't get read into the stream, used for Gemini
+    // If there are toolCalls, but they didn't get read into the stream, used for Gemini
     if (!toolCalls.size && value.options && "toolCall" in value.options) {
       value.options.toolCall.forEach((toolCall) => {
         toolCalls.set(toolCall.id, toolCall);

packages/llamaindex/src/callbacks/CallbackManager.ts

@@ -1,4 +1,5 @@
 import type { Anthropic } from "@anthropic-ai/sdk";
+import type { MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import { CustomEvent } from "@llamaindex/env";
 import type { AgentEndEvent, AgentStartEvent } from "../agent/types.js";
@@ -12,7 +13,6 @@ import type {
   LLMStreamEvent,
   LLMToolCallEvent,
   LLMToolResultEvent,
-  MessageContent,
   RetrievalEndEvent,
   RetrievalStartEvent,
 } from "../llm/types.js";

packages/llamaindex/src/cloud/LlamaCloudIndex.ts

@@ -11,8 +11,9 @@ import { getPipelineCreate } from "./config.js";
 import type { CloudConstructorParams } from "./constants.js";
 import { getAppBaseUrl, initService } from "./utils.js";
 
-import { Service } from "@llamaindex/cloud/api";
+import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api";
 import { getEnv } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
 import { OpenAIEmbedding } from "../embeddings/OpenAIEmbedding.js";
 import { SimpleNodeParser } from "../nodeParsers/SimpleNodeParser.js";
 
@@ -25,7 +26,7 @@ export class LlamaCloudIndex {
   }
 
   private async waitForPipelineIngestion(
-    verbose = false,
+    verbose = Settings.debug,
     raiseOnError = false,
   ): Promise<void> {
     const pipelineId = await this.getPipelineId(
@@ -39,9 +40,11 @@ export class LlamaCloudIndex {
 
     while (true) {
       const pipelineStatus =
-        await Service.getPipelineStatusApiV1PipelinesPipelineIdStatusGet({
-          pipelineId,
-        });
+        await PipelinesService.getPipelineStatusApiV1PipelinesPipelineIdStatusGet(
+          {
+            pipelineId,
+          },
+        );
 
       if (pipelineStatus.status === "SUCCESS") {
         if (verbose) {
@@ -70,7 +73,7 @@ export class LlamaCloudIndex {
 
   private async waitForDocumentIngestion(
     docIds: string[],
-    verbose = false,
+    verbose = Settings.debug,
     raiseOnError = false,
   ): Promise<void> {
     const pipelineId = await this.getPipelineId(
@@ -78,11 +81,6 @@ export class LlamaCloudIndex {
       this.params.projectName,
     );
 
-    const client = await initService({
-      ...this.params,
-      baseUrl: this.params.baseUrl,
-    });
-
     if (verbose) {
       console.log("Loading data: ");
     }
@@ -94,7 +92,7 @@ export class LlamaCloudIndex {
 
       for (const doc of pendingDocs) {
         const { status } =
-          await Service.getPipelineDocumentStatusApiV1PipelinesPipelineIdDocumentsDocumentIdStatusGet(
+          await PipelinesService.getPipelineDocumentStatusApiV1PipelinesPipelineIdDocumentsDocumentIdStatusGet(
             { pipelineId, documentId: doc },
           );
 
@@ -139,7 +137,7 @@ export class LlamaCloudIndex {
     name: string,
     projectName: string,
   ): Promise<string> {
-    const pipelines = await Service.searchPipelinesApiV1PipelinesGet({
+    const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
       projectName,
       pipelineName: name,
     });
@@ -160,10 +158,7 @@ export class LlamaCloudIndex {
         apiKey: getEnv("OPENAI_API_KEY"),
       }),
     ];
-
-    const appUrl = getAppBaseUrl(params.baseUrl);
-
-    const client = await initService({ ...params, baseUrl: appUrl });
+    const apiUrl = getAppBaseUrl();
 
     const pipelineCreateParams = await getPipelineCreate({
       pipelineName: params.name,
@@ -172,7 +167,7 @@ export class LlamaCloudIndex {
       transformations: params.transformations ?? defaultTransformations,
     });
 
-    const project = await Service.upsertProjectApiV1ProjectsPut({
+    const project = await ProjectsService.upsertProjectApiV1ProjectsPut({
       requestBody: {
         name: params.projectName ?? "default",
       },
@@ -182,7 +177,7 @@ export class LlamaCloudIndex {
       throw new Error("Project ID should be defined");
     }
 
-    const pipeline = await Service.upsertPipelineApiV1PipelinesPut({
+    const pipeline = await PipelinesService.upsertPipelineApiV1PipelinesPut({
       projectId: project.id,
       requestBody: {
         name: params.name,
@@ -200,7 +195,7 @@ export class LlamaCloudIndex {
       console.log(`Created pipeline ${pipeline.id} with name ${params.name}`);
     }
 
-    await Service.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
+    await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
       {
         pipelineId: pipeline.id,
         requestBody: params.documents.map((doc) => ({
@@ -215,9 +210,11 @@ export class LlamaCloudIndex {
 
     while (true) {
       const pipelineStatus =
-        await Service.getPipelineStatusApiV1PipelinesPipelineIdStatusGet({
-          pipelineId: pipeline.id,
-        });
+        await PipelinesService.getPipelineStatusApiV1PipelinesPipelineIdStatusGet(
+          {
+            pipelineId: pipeline.id,
+          },
+        );
 
       if (pipelineStatus.status === "SUCCESS") {
         console.info(
@@ -228,14 +225,14 @@ export class LlamaCloudIndex {
 
       if (pipelineStatus.status === "ERROR") {
         console.error(
-          `Some documents failed to ingest, check your pipeline logs at ${appUrl}/project/${project.id}/deploy/${pipeline.id}`,
+          `Some documents failed to ingest, check your pipeline logs at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
         );
         throw new Error("Some documents failed to ingest");
       }
 
       if (pipelineStatus.status === "PARTIAL_SUCCESS") {
         console.info(
-          `Documents ingestion partially succeeded, to check a more complete status check your pipeline at ${appUrl}/project/${project.id}/deploy/${pipeline.id}`,
+          `Documents ingestion partially succeeded, to check a more complete status check your pipeline at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
         );
         break;
       }
@@ -249,7 +246,7 @@ export class LlamaCloudIndex {
 
     if (params.verbose) {
       console.info(
-        `Ingestion completed, find your index at ${appUrl}/project/${project.id}/deploy/${pipeline.id}`,
+        `Ingestion completed, find your index at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
       );
     }
 
@@ -280,10 +277,6 @@ export class LlamaCloudIndex {
   }
 
   async insert(document: Document) {
-    const appUrl = getAppBaseUrl(this.params.baseUrl);
-
-    const client = await initService({ ...this.params, baseUrl: appUrl });
-
     const pipelineId = await this.getPipelineId(
       this.params.name,
       this.params.projectName,
@@ -293,7 +286,7 @@ export class LlamaCloudIndex {
       throw new Error("We couldn't find the pipeline ID for the given name");
     }
 
-    await Service.createBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPost(
+    await PipelinesService.createBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPost(
       {
         pipelineId: pipelineId,
         requestBody: [
@@ -312,10 +305,6 @@ export class LlamaCloudIndex {
   }
 
   async delete(document: Document) {
-    const appUrl = getAppBaseUrl(this.params.baseUrl);
-
-    const client = await initService({ ...this.params, baseUrl: appUrl });
-
     const pipelineId = await this.getPipelineId(
       this.params.name,
       this.params.projectName,
@@ -325,7 +314,7 @@ export class LlamaCloudIndex {
       throw new Error("We couldn't find the pipeline ID for the given name");
     }
 
-    await Service.deletePipelineDocumentApiV1PipelinesPipelineIdDocumentsDocumentIdDelete(
+    await PipelinesService.deletePipelineDocumentApiV1PipelinesPipelineIdDocumentsDocumentIdDelete(
       {
         pipelineId,
         documentId: document.id_,
@@ -336,10 +325,6 @@ export class LlamaCloudIndex {
   }
 
   async refreshDoc(document: Document) {
-    const appUrl = getAppBaseUrl(this.params.baseUrl);
-
-    const client = await initService({ ...this.params, baseUrl: appUrl });
-
     const pipelineId = await this.getPipelineId(
       this.params.name,
       this.params.projectName,
@@ -349,7 +334,7 @@ export class LlamaCloudIndex {
       throw new Error("We couldn't find the pipeline ID for the given name");
     }
 
-    await Service.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
+    await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
       {
         pipelineId,
         requestBody: [

packages/llamaindex/src/cloud/LlamaCloudRetriever.ts

@@ -1,7 +1,7 @@
 import {
   type MetadataFilters,
+  PipelinesService,
   type RetrievalParams,
-  Service,
   type TextNodeWithScore,
 } from "@llamaindex/cloud/api";
 import type { NodeWithScore } from "@llamaindex/core/schema";
@@ -51,20 +51,21 @@ export class LlamaCloudRetriever implements BaseRetriever {
     query,
     preFilters,
   }: RetrieveParams): Promise<NodeWithScore[]> {
-    const pipelines = await Service.searchPipelinesApiV1PipelinesGet({
+    const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
       projectName: this.projectName,
       pipelineName: this.pipelineName,
     });
 
-    if (!pipelines) {
+    if (pipelines.length === 0 || !pipelines[0].id) {
       throw new Error(
         `No pipeline found with name ${this.pipelineName} in project ${this.projectName}`,
       );
     }
 
-    const pipeline = await Service.getPipelineApiV1PipelinesPipelineIdGet({
-      pipelineId: pipelines[0].id,
-    });
+    const pipeline =
+      await PipelinesService.getPipelineApiV1PipelinesPipelineIdGet({
+        pipelineId: pipelines[0].id,
+      });
 
     if (!pipeline) {
       throw new Error(
@@ -72,16 +73,15 @@ export class LlamaCloudRetriever implements BaseRetriever {
       );
     }
 
-    const results = await Service.runSearchApiV1PipelinesPipelineIdRetrievePost(
-      {
+    const results =
+      await PipelinesService.runSearchApiV1PipelinesPipelineIdRetrievePost({
         pipelineId: pipeline.id,
         requestBody: {
           ...this.retrieveParams,
           query: extractText(query),
           search_filters: preFilters as MetadataFilters,
         },
-      },
-    );
+      });
 
     return this.resultNodesToNodeWithScore(results.retrieval_nodes);
   }

packages/llamaindex/src/cloud/constants.ts

@@ -1,6 +1,5 @@
 import type { ServiceContext } from "../ServiceContext.js";
 
-export const DEFAULT_PIPELINE_NAME = "default";
 export const DEFAULT_PROJECT_NAME = "Default";
 export const DEFAULT_BASE_URL = "https://api.cloud.llamaindex.ai";
 

packages/llamaindex/src/cloud/utils.ts

@@ -1,4 +1,4 @@
-import { OpenAPI, Service } from "@llamaindex/cloud/api";
+import { OpenAPI } from "@llamaindex/cloud/api";
 import { getEnv } from "@llamaindex/env";
 import type { ClientParams } from "./constants.js";
 import { DEFAULT_BASE_URL } from "./constants.js";
@@ -7,14 +7,11 @@ function getBaseUrl(baseUrl?: string): string {
   return baseUrl ?? getEnv("LLAMA_CLOUD_BASE_URL") ?? DEFAULT_BASE_URL;
 }
 
-export function getAppBaseUrl(baseUrl?: string): string {
-  return getBaseUrl(baseUrl).replace(/api\./, "");
+export function getAppBaseUrl(): string {
+  return OpenAPI.BASE.replace(/api\./, "");
 }
 
-export function initService({
-  apiKey,
-  baseUrl,
-}: ClientParams = {}): typeof Service {
+export function initService({ apiKey, baseUrl }: ClientParams = {}) {
   OpenAPI.TOKEN = apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
   OpenAPI.BASE = getBaseUrl(baseUrl);
   if (!OpenAPI.TOKEN) {
@@ -22,6 +19,4 @@ export function initService({
       "API Key is required for LlamaCloudIndex. Please pass the apiKey parameter",
     );
   }
-
-  return Service;
 }

packages/llamaindex/src/embeddings/DeepInfraEmbedding.ts

@@ -1,5 +1,5 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import { getEnv } from "@llamaindex/env";
-import type { MessageContentDetail } from "../llm/index.js";
 import { extractSingleText } from "../llm/utils.js";
 import { BaseEmbedding } from "./types.js";
 

packages/llamaindex/src/embeddings/MultiModalEmbedding.ts

@@ -1,3 +1,4 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import {
   ImageNode,
   MetadataMode,
@@ -6,7 +7,6 @@ import {
   type BaseNode,
   type ImageType,
 } from "@llamaindex/core/schema";
-import type { MessageContentDetail } from "../llm/types.js";
 import { extractImage, extractSingleText } from "../llm/utils.js";
 import { BaseEmbedding, batchEmbeddings } from "./types.js";
 

packages/llamaindex/src/embeddings/types.ts

@@ -1,8 +1,8 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import type { BaseNode } from "@llamaindex/core/schema";
 import { MetadataMode } from "@llamaindex/core/schema";
 import { type Tokenizers } from "@llamaindex/env";
 import type { TransformComponent } from "../ingestion/types.js";
-import type { MessageContentDetail } from "../llm/types.js";
 import { extractSingleText } from "../llm/utils.js";
 import { truncateMaxTokens } from "./tokenizer.js";
 import { SimilarityType, similarity } from "./utils.js";

packages/llamaindex/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -1,3 +1,4 @@
+import type { ChatMessage, LLM } from "@llamaindex/core/llms";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import type { EngineResponse } from "../../EngineResponse.js";
@@ -9,7 +10,6 @@ import {
 import type { ServiceContext } from "../../ServiceContext.js";
 import { llmFromSettingsOrContext } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import type { ChatMessage, LLM } from "../../llm/index.js";
 import { extractText, streamReducer } from "../../llm/utils.js";
 import { PromptMixin } from "../../prompts/index.js";
 import type { QueryEngine } from "../../types.js";

packages/llamaindex/src/engines/chat/ContextChatEngine.ts

@@ -1,3 +1,9 @@
+import type {
+  ChatMessage,
+  LLM,
+  MessageContent,
+  MessageType,
+} from "@llamaindex/core/llms";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import { EngineResponse } from "../../EngineResponse.js";
@@ -5,8 +11,6 @@ import type { ContextSystemPrompt } from "../../Prompt.js";
 import type { BaseRetriever } from "../../Retriever.js";
 import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import type { ChatMessage, LLM } from "../../llm/index.js";
-import type { MessageContent, MessageType } from "../../llm/types.js";
 import {
   extractText,
   streamConverter,

packages/llamaindex/src/engines/chat/DefaultContextGenerator.ts

@@ -1,10 +1,10 @@
+import type { MessageContent, MessageType } from "@llamaindex/core/llms";
 import { type NodeWithScore } from "@llamaindex/core/schema";
+import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import { defaultContextSystemPrompt } from "../../Prompt.js";
-import type { BaseRetriever } from "../../Retriever.js";
-import type { MessageContent, MessageType } from "../../llm/types.js";
-import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import { PromptMixin } from "../../prompts/index.js";
+import type { BaseRetriever } from "../../Retriever.js";
 import { createMessageContent } from "../../synthesizers/utils.js";
 import type { Context, ContextGenerator } from "./types.js";
 

packages/llamaindex/src/engines/chat/SimpleChatEngine.ts

@@ -1,9 +1,9 @@
+import type { LLM } from "@llamaindex/core/llms";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import { EngineResponse } from "../../EngineResponse.js";
 import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import type { LLM } from "../../llm/index.js";
 import { streamConverter, streamReducer } from "../../llm/utils.js";
 import type {
   ChatEngine,

packages/llamaindex/src/engines/chat/types.ts

@@ -1,8 +1,7 @@
+import type { ChatMessage, MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import type { ChatHistory } from "../../ChatHistory.js";
 import type { EngineResponse } from "../../EngineResponse.js";
-import type { ChatMessage } from "../../llm/index.js";
-import type { MessageContent } from "../../llm/types.js";
 
 /**
  * Represents the base parameters for ChatEngine.

packages/llamaindex/src/engines/query/SubQuestionQueryEngine.ts

@@ -11,13 +11,12 @@ import {
 } from "../../synthesizers/index.js";
 
 import type {
-  BaseTool,
   QueryEngine,
   QueryEngineParamsNonStreaming,
   QueryEngineParamsStreaming,
-  ToolMetadata,
 } from "../../types.js";
 
+import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import type { BaseQuestionGenerator, SubQuestion } from "./types.js";
 

packages/llamaindex/src/engines/query/types.ts

@@ -1,4 +1,4 @@
-import type { ToolMetadata } from "../../types.js";
+import type { ToolMetadata } from "@llamaindex/core/llms";
 
 /**
  * QuestionGenerators generate new questions for the LLM using tools and a user query.

packages/llamaindex/src/evaluation/Correctness.ts

@@ -1,9 +1,9 @@
+import type { ChatMessage, LLM } from "@llamaindex/core/llms";
 import { MetadataMode } from "@llamaindex/core/schema";
-import type { ServiceContext } from "../ServiceContext.js";
-import { llmFromSettingsOrContext } from "../Settings.js";
-import type { ChatMessage, LLM } from "../llm/types.js";
 import { extractText } from "../llm/utils.js";
 import { PromptMixin } from "../prompts/Mixin.js";
+import type { ServiceContext } from "../ServiceContext.js";
+import { llmFromSettingsOrContext } from "../Settings.js";
 import type { CorrectnessSystemPrompt } from "./prompts.js";
 import {
   defaultCorrectnessSystemPrompt,

packages/llamaindex/src/extractors/MetadataExtractors.ts

@@ -1,6 +1,6 @@
+import type { LLM } from "@llamaindex/core/llms";
 import type { BaseNode } from "@llamaindex/core/schema";
 import { MetadataMode, TextNode } from "@llamaindex/core/schema";
-import type { LLM } from "../llm/index.js";
 import { OpenAI } from "../llm/index.js";
 import {
   defaultKeywordExtractorPromptTemplate,

packages/llamaindex/src/index.edge.ts

@@ -1,3 +1,4 @@
+export * from "@llamaindex/core/llms";
 export * from "@llamaindex/core/schema";
 export * from "./agent/index.js";
 export * from "./callbacks/CallbackManager.js";

packages/llamaindex/src/index.ts

@@ -13,3 +13,6 @@ export {
 
 export { type VertexGeminiSessionOptions } from "./llm/gemini/types.js";
 export { GeminiVertexSession } from "./llm/gemini/vertex.js";
+
+// Expose AzureDynamicSessionTool for node.js runtime only
+export { AzureDynamicSessionTool } from "./tools/AzureDynamicSessionTool.node.js";

packages/llamaindex/src/indices/keyword/index.ts

@@ -31,8 +31,8 @@ import {
   simpleExtractKeywords,
 } from "./utils.js";
 
+import type { LLM } from "@llamaindex/core/llms";
 import { llmFromSettingsOrContext } from "../../Settings.js";
-import type { LLM } from "../../llm/types.js";
 import { extractText } from "../../llm/utils.js";
 
 export interface KeywordIndexOptions {

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -1,3 +1,4 @@
+import type { MessageContent } from "@llamaindex/core/llms";
 import {
   ImageNode,
   ModalityType,
@@ -23,7 +24,6 @@ import {
 } from "../../ingestion/strategies/index.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
-import type { MessageContent } from "../../llm/types.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/types.js";
 import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";

packages/llamaindex/src/internal/prompt/react.ts

@@ -1,4 +1,4 @@
-import type { BaseTool } from "../../types.js";
+import type { BaseTool } from "@llamaindex/core/llms";
 
 export const getReACTAgentSystemHeader = (tools: BaseTool[]) => {
   const description = tools

packages/llamaindex/src/llm/anthropic.ts

@@ -13,19 +13,22 @@ import type {
   TextBlock,
   TextBlockParam,
 } from "@anthropic-ai/sdk/resources/index";
-import type { MessageParam } from "@anthropic-ai/sdk/resources/messages";
-import { getEnv } from "@llamaindex/env";
-import _ from "lodash";
-import type { BaseTool } from "../types.js";
-import { ToolCallLLM } from "./base.js";
 import type {
+  ImageBlockParam,
+  MessageParam,
+} from "@anthropic-ai/sdk/resources/messages";
+import type {
+  BaseTool,
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
   ToolCallLLMMessageOptions,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import _ from "lodash";
+import { ToolCallLLM } from "./base.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 export class AnthropicSession {
@@ -214,7 +217,32 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
         }
 
         return {
-          content: extractText(message.content),
+          content:
+            typeof message.content === "string"
+              ? message.content
+              : message.content.map(
+                  (content): TextBlockParam | ImageBlockParam =>
+                    content.type === "text"
+                      ? {
+                          type: "text",
+                          text: content.text,
+                        }
+                      : {
+                          type: "image",
+                          source: {
+                            data: content.image_url.url.substring(
+                              content.image_url.url.indexOf(",") + 1,
+                            ),
+                            media_type:
+                              `image/${content.image_url.url.substring("data:image/".length, content.image_url.url.indexOf(";base64"))}` as
+                                | "image/jpeg"
+                                | "image/png"
+                                | "image/gif"
+                                | "image/webp",
+                            type: "base64",
+                          },
+                        },
+                ),
           role: message.role as "user" | "assistant",
         } satisfies MessageParam;
       });

packages/llamaindex/src/llm/base.ts

@@ -9,7 +9,7 @@ import type {
   LLMCompletionParamsStreaming,
   LLMMetadata,
   ToolCallLLMMessageOptions,
-} from "./types.js";
+} from "@llamaindex/core/llms";
 import { extractText, streamConverter } from "./utils.js";
 
 export abstract class BaseLLM<

packages/llamaindex/src/llm/gemini/base.ts

@@ -7,8 +7,6 @@ import {
   type GenerateContentStreamResult as GoogleStreamGenerateContentResult,
 } from "@google/generative-ai";
 
-import { getEnv, randomUUID } from "@llamaindex/env";
-import { ToolCallLLM } from "../base.js";
 import type {
   CompletionResponse,
   LLMCompletionParamsNonStreaming,
@@ -16,7 +14,9 @@ import type {
   LLMMetadata,
   ToolCall,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
+import { getEnv, randomUUID } from "@llamaindex/env";
+import { ToolCallLLM } from "../base.js";
 import { streamConverter, wrapLLMEvent } from "../utils.js";
 import {
   GEMINI_BACKENDS,
@@ -33,6 +33,7 @@ import {
   type IGeminiSession,
 } from "./types.js";
 import {
+  DEFAULT_SAFETY_SETTINGS,
   GeminiHelper,
   getChatContext,
   getPartsText,
@@ -87,7 +88,10 @@ export class GeminiSession implements IGeminiSession {
   }
 
   getGenerativeModel(metadata: GoogleModelParams): GoogleGenerativeModel {
-    return this.gemini.getGenerativeModel(metadata);
+    return this.gemini.getGenerativeModel({
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
+      ...metadata,
+    });
   }
 
   getResponseText(response: EnhancedGenerateContentResponse): string {
@@ -143,8 +147,9 @@ export class GeminiSessionStore {
   }> = [];
 
   private static getSessionId(options: GeminiSessionOptions): string {
-    if (options.backend === GEMINI_BACKENDS.GOOGLE)
+    if (options.backend === GEMINI_BACKENDS.GOOGLE) {
       return options?.apiKey ?? "";
+    }
     return "";
   }
   private static sessionMatched(
@@ -223,6 +228,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
           ),
         },
       ],
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
     });
     const { response } = await chat.sendMessage(context.message);
     const topCandidate = response.candidates![0];
@@ -258,6 +264,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
           ),
         },
       ],
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
     });
     const result = await chat.sendMessageStream(context.message);
     yield* this.session.getChatStream(result);

packages/llamaindex/src/llm/gemini/types.ts

@@ -33,7 +33,7 @@ import type {
   LLMChatParamsStreaming,
   ToolCall,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
 
 export enum GEMINI_BACKENDS {
   GOOGLE = "google",

packages/llamaindex/src/llm/gemini/utils.ts

@@ -1,17 +1,20 @@
 import {
   type FunctionCall,
   type Content as GeminiMessageContent,
+  HarmBlockThreshold,
+  HarmCategory,
+  type SafetySetting,
 } from "@google/generative-ai";
 
 import { type GenerateContentResponse } from "@google-cloud/vertexai";
-import type { BaseTool } from "../../types.js";
 import type {
+  BaseTool,
   ChatMessage,
   MessageContentImageDetail,
   MessageContentTextDetail,
   MessageType,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
 import { extractDataUrlComponents } from "../utils.js";
 import type {
   ChatContext,
@@ -53,10 +56,13 @@ const getImageParts = (
     const { mimeType, base64: data } = extractDataUrlComponents(
       message.image_url.url,
     );
-    if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+    if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType)) {
       throw new Error(
-        `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+        `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join(
+          "\n",
+        )}`,
       );
+    }
     return {
       inlineData: {
         mimeType,
@@ -65,10 +71,13 @@ const getImageParts = (
     };
   }
   const mimeType = getFileURLMimeType(message.image_url.url);
-  if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+  if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType)) {
     throw new Error(
-      `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+      `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join(
+        "\n",
+      )}`,
     );
+  }
   return {
     fileData: { mimeType, fileUri: message.image_url.url },
   };
@@ -124,10 +133,11 @@ export const getChatContext = (
   // 2. Parts that have empty text
   const fnMap = params.messages.reduce(
     (result, message) => {
-      if (message.options && "toolCall" in message.options)
+      if (message.options && "toolCall" in message.options) {
         message.options.toolCall.forEach((call) => {
           result[call.id] = call.name;
         });
+      }
 
       return result;
     },
@@ -224,10 +234,11 @@ export class GeminiHelper {
     if (options && "toolResult" in options) {
       if (!fnMap) throw Error("fnMap must be set");
       const name = fnMap[options.toolResult.id];
-      if (!name)
+      if (!name) {
         throw Error(
           `Could not find the name for fn call with id ${options.toolResult.id}`,
         );
+      }
 
       return [
         {
@@ -299,3 +310,26 @@ export function getFunctionCalls(
     return undefined;
   }
 }
+
+/**
+ * Safety settings to disable external filters
+ * Documentation: https://ai.google.dev/gemini-api/docs/safety-settings
+ */
+export const DEFAULT_SAFETY_SETTINGS: SafetySetting[] = [
+  {
+    category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_HARASSMENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+];

packages/llamaindex/src/llm/gemini/vertex.ts

@@ -1,8 +1,8 @@
 import {
+  type GenerateContentResponse,
   VertexAI,
   GenerativeModel as VertexGenerativeModel,
   GenerativeModelPreview as VertexGenerativeModelPreview,
-  type GenerateContentResponse,
   type ModelParams as VertexModelParams,
   type StreamGenerateContentResult as VertexStreamGenerateContentResult,
 } from "@google-cloud/vertexai";
@@ -14,14 +14,14 @@ import type {
 } from "./types.js";
 
 import type { FunctionCall } from "@google/generative-ai";
-import { getEnv, randomUUID } from "@llamaindex/env";
 import type {
   CompletionResponse,
   ToolCall,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
+import { getEnv, randomUUID } from "@llamaindex/env";
 import { streamConverter } from "../utils.js";
-import { getFunctionCalls, getText } from "./utils.js";
+import { DEFAULT_SAFETY_SETTINGS, getFunctionCalls, getText } from "./utils.js";
 
 /* To use Google's Vertex AI backend, it doesn't use api key authentication.
  *
@@ -59,8 +59,16 @@ export class GeminiVertexSession implements IGeminiSession {
   getGenerativeModel(
     metadata: VertexModelParams,
   ): VertexGenerativeModelPreview | VertexGenerativeModel {
-    if (this.preview) return this.vertex.preview.getGenerativeModel(metadata);
-    return this.vertex.getGenerativeModel(metadata);
+    if (this.preview) {
+      return this.vertex.preview.getGenerativeModel({
+        safetySettings: DEFAULT_SAFETY_SETTINGS,
+        ...metadata,
+      });
+    }
+    return this.vertex.getGenerativeModel({
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
+      ...metadata,
+    });
   }
 
   getResponseText(response: GenerateContentResponse): string {

packages/llamaindex/src/llm/huggingface.ts

@@ -1,11 +1,4 @@
 import { HfInference } from "@huggingface/inference";
-import type {
-  PreTrainedModel,
-  PreTrainedTokenizer,
-  Tensor,
-} from "@xenova/transformers";
-import { lazyLoadTransformers } from "../internal/deps/transformers.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -14,7 +7,14 @@ import type {
   LLMChatParamsStreaming,
   LLMMetadata,
   ToolCallLLMMessageOptions,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import type {
+  PreTrainedModel,
+  PreTrainedTokenizer,
+  Tensor,
+} from "@xenova/transformers";
+import { lazyLoadTransformers } from "../internal/deps/transformers.js";
+import { BaseLLM } from "./base.js";
 import { streamConverter, wrapLLMEvent } from "./utils.js";
 
 // TODO workaround issue with @huggingface/inference@2.7.0

packages/llamaindex/src/llm/mistral.ts

@@ -1,14 +1,14 @@
-import { getEnv } from "@llamaindex/env";
-import { Settings } from "../Settings.js";
-import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
+import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
+import { BaseLLM } from "./base.js";
 
 export const ALL_AVAILABLE_MISTRAL_MODELS = {
   "mistral-tiny": { contextWindow: 32000 },

packages/llamaindex/src/llm/ollama.ts

@@ -1,3 +1,14 @@
+import type {
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+} from "@llamaindex/core/llms";
 import { BaseEmbedding } from "../embeddings/types.js";
 import {
   Ollama as OllamaBase,
@@ -19,17 +30,6 @@ import {
   type ShowResponse,
   type StatusResponse,
 } from "../internal/deps/ollama.js";
-import type {
-  ChatResponse,
-  ChatResponseChunk,
-  CompletionResponse,
-  LLM,
-  LLMChatParamsNonStreaming,
-  LLMChatParamsStreaming,
-  LLMCompletionParamsNonStreaming,
-  LLMCompletionParamsStreaming,
-  LLMMetadata,
-} from "./types.js";
 import { extractText, streamConverter } from "./utils.js";
 
 const messageAccessor = (part: OllamaChatResponse): ChatResponseChunk => {

packages/llamaindex/src/llm/openai.ts

@@ -7,6 +7,19 @@ import type {
 } from "openai";
 import { AzureOpenAI, OpenAI as OrigOpenAI } from "openai";
 
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMMetadata,
+  MessageType,
+  PartialToolCall,
+  ToolCallLLMMessageOptions,
+} from "@llamaindex/core/llms";
 import { Tokenizers } from "@llamaindex/env";
 import type {
   ChatCompletionAssistantMessageParam,
@@ -20,7 +33,6 @@ import type {
 import type { ChatCompletionMessageParam } from "openai/resources/index.js";
 import { wrapEventCaller } from "../internal/context/EventCaller.js";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import type { BaseTool } from "../types.js";
 import type { AzureOpenAIConfig } from "./azure.js";
 import {
   getAzureConfigFromEnv,
@@ -28,18 +40,6 @@ import {
   shouldUseAzure,
 } from "./azure.js";
 import { ToolCallLLM } from "./base.js";
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-  LLM,
-  LLMChatParamsNonStreaming,
-  LLMChatParamsStreaming,
-  LLMMetadata,
-  MessageType,
-  PartialToolCall,
-  ToolCallLLMMessageOptions,
-} from "./types.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 export class OpenAISession {

packages/llamaindex/src/llm/portkey.ts

@@ -1,10 +1,3 @@
-import { getEnv } from "@llamaindex/env";
-import _ from "lodash";
-import type { LLMOptions } from "portkey-ai";
-import { Portkey as OrigPortKey } from "portkey-ai";
-import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -13,7 +6,14 @@ import type {
   LLMChatParamsStreaming,
   LLMMetadata,
   MessageType,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import _ from "lodash";
+import type { LLMOptions } from "portkey-ai";
+import { Portkey as OrigPortKey } from "portkey-ai";
+import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { BaseLLM } from "./base.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 interface PortkeyOptions {

packages/llamaindex/src/llm/replicate_ai.ts

@@ -1,6 +1,3 @@
-import { getEnv } from "@llamaindex/env";
-import Replicate from "../internal/deps/replicate.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -8,7 +5,10 @@ import type {
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
   MessageType,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import Replicate from "../internal/deps/replicate.js";
+import { BaseLLM } from "./base.js";
 import {
   extractText,
   streamCallbacks,