Release 0.4.12 (#1013)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/preview.yml

@@ -0,0 +1,28 @@
+name: Publish Preview
+on: [pull_request]
+
+jobs:
+  pre_release:
+    name: Pre Release
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v3
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build
+        run: pnpm run build
+
+      - name: Pre Release
+        run: pnpx pkg-pr-new publish ./packages/*

README.md

@@ -76,7 +76,7 @@ main();
 node --import tsx ./main.ts
 ```
 
-### Next.js
+### React Server Component (Next.js, Waku, Redwood.JS...)
 
 First, you will need to add a llamaindex plugin to your Next.js project.
 
@@ -154,40 +154,6 @@ export async function chatWithAgent(
 }
 ```
 
-### Cloudflare Workers
-
-```ts
-// src/index.ts
-export default {
-  async fetch(
-    request: Request,
-    env: Env,
-    ctx: ExecutionContext,
-  ): Promise<Response> {
-    const { setEnvs } = await import("@llamaindex/env");
-    // set environment variables so that the OpenAIAgent can use them
-    setEnvs(env);
-    const { OpenAIAgent } = await import("llamaindex");
-    const agent = new OpenAIAgent({
-      tools: [],
-    });
-    const responseStream = await agent.chat({
-      stream: true,
-      message: "Hello? What is the weather today?",
-    });
-    const textEncoder = new TextEncoder();
-    const response = responseStream.pipeThrough(
-      new TransformStream({
-        transform: (chunk, controller) => {
-          controller.enqueue(textEncoder.encode(chunk.response.delta));
-        },
-      }),
-    );
-    return new Response(response);
-  },
-};
-```
-
 ## Playground
 
 Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground

apps/docs/CHANGELOG.md

@@ -1,5 +1,33 @@
 # docs
 
+## 0.0.38
+
+### Patch Changes
+
+- Updated dependencies [f326ab8]
+  - llamaindex@0.4.12
+
+## 0.0.37
+
+### Patch Changes
+
+- Updated dependencies [8bf5b4a]
+  - llamaindex@0.4.11
+
+## 0.0.36
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.34
 
 ### Patch Changes

apps/docs/docs/modules/llms/available_llms/bedrock.md

@@ -15,7 +15,7 @@ Settings.llm = new Bedrock({
 });
 ```
 
-Currently only supports Anthropic models:
+Currently only supports Anthropic and Meta models:
 
 ```ts
 ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1";
@@ -25,6 +25,10 @@ ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0";
 ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0";
 ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0"; // available on us-west-2
 ANTHROPIC_CLAUDE_3_5_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0";
+META_LLAMA2_13B_CHAT = "meta.llama2-13b-chat-v1";
+META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1";
+META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0";
+META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0";
 ```
 
 Sonnet, Haiku and Opus are multimodal, image_url only supports base64 data url format, e.g. `data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==`

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.34",
+  "version": "0.0.38",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.22
+
+### Patch Changes
+
+- Updated dependencies [f326ab8]
+  - llamaindex@0.4.12
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.21
+
+### Patch Changes
+
+- Updated dependencies [8bf5b4a]
+  - llamaindex@0.4.11
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.20
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.19
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+  - @llamaindex/autotool@1.0.0
+
 ## 0.1.18
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.18",
+  "version": "0.1.22",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.4.8",
+    "llamaindex": "^0.4.12",
     "openai": "^4",
     "typescript": "^4"
   },

packages/cloud/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/cloud
 
+## 0.1.2
+
+### Patch Changes
+
+- f326ab8: chore: bump version
+
 ## 0.1.1
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,5 +1,34 @@
 # @llamaindex/community
 
+## 0.0.16
+
+### Patch Changes
+
+- f326ab8: chore: bump version
+- Updated dependencies [f326ab8]
+  - llamaindex@0.4.12
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [8bf5b4a]
+  - llamaindex@0.4.11
+
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.12
 
 ### Patch Changes

packages/community/README.md

@@ -5,6 +5,7 @@
 ## Current Features:
 
 - Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
+- Bedrock support for the Meta LLama 2 and 3 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
 
 ## LICENSE
 

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.12",
+  "version": "0.0.16",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/community/src/llm/bedrock/base.ts

@@ -1,53 +1,34 @@
 import {
   BedrockRuntimeClient,
+  type BedrockRuntimeClientConfig,
   InvokeModelCommand,
   InvokeModelWithResponseStreamCommand,
-  ResponseStream,
-  type BedrockRuntimeClientConfig,
-  type InvokeModelCommandInput,
-  type InvokeModelWithResponseStreamCommandInput,
 } from "@aws-sdk/client-bedrock-runtime";
 import type {
-  BaseTool,
   ChatMessage,
   ChatResponse,
-  ChatResponseChunk,
   CompletionResponse,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
   LLMMetadata,
-  PartialToolCall,
-  ToolCall,
   ToolCallLLMMessageOptions,
 } from "llamaindex";
-import { ToolCallLLM, streamConverter, wrapLLMEvent } from "llamaindex";
-import type {
-  AnthropicNoneStreamingResponse,
-  AnthropicTextContent,
-  StreamEvent,
-  ToolBlock,
-  ToolChoice,
-} from "./types.js";
+import { streamConverter, ToolCallLLM, wrapLLMEvent } from "llamaindex";
 import {
-  mapBaseToolsToAnthropicTools,
-  mapChatMessagesToAnthropicMessages,
-  mapMessageContentToMessageContentDetails,
-  toUtf8,
-} from "./utils.js";
-
-export type BedrockAdditionalChatOptions = { toolChoice: ToolChoice };
+  type BedrockAdditionalChatOptions,
+  type BedrockChatStreamResponse,
+  Provider,
+} from "./provider";
+import { PROVIDERS } from "./providers";
+import { mapMessageContentToMessageContentDetails } from "./utils.js";
 
 export type BedrockChatParamsStreaming = LLMChatParamsStreaming<
   BedrockAdditionalChatOptions,
   ToolCallLLMMessageOptions
 >;
 
-export type BedrockChatStreamResponse = AsyncIterable<
-  ChatResponseChunk<ToolCallLLMMessageOptions>
->;
-
 export type BedrockChatParamsNonStreaming = LLMChatParamsNonStreaming<
   BedrockAdditionalChatOptions,
   ToolCallLLMMessageOptions
@@ -151,174 +132,6 @@ export const TOOL_CALL_MODELS = [
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
 ];
 
-abstract class Provider<ProviderStreamEvent extends {} = {}> {
-  abstract getTextFromResponse(response: Record<string, any>): string;
-
-  abstract getToolsFromResponse<T extends {} = {}>(
-    response: Record<string, any>,
-  ): T[];
-
-  getStreamingEventResponse(
-    response: Record<string, any>,
-  ): ProviderStreamEvent | undefined {
-    return response.chunk?.bytes
-      ? (JSON.parse(toUtf8(response.chunk?.bytes)) as ProviderStreamEvent)
-      : undefined;
-  }
-
-  async *reduceStream(
-    stream: AsyncIterable<ResponseStream>,
-  ): BedrockChatStreamResponse {
-    yield* streamConverter(stream, (response) => {
-      return {
-        delta: this.getTextFromStreamResponse(response),
-        raw: response,
-      };
-    });
-  }
-
-  getTextFromStreamResponse(response: Record<string, any>): string {
-    return this.getTextFromResponse(response);
-  }
-
-  abstract getRequestBody<T extends ChatMessage>(
-    metadata: LLMMetadata,
-    messages: T[],
-    tools?: BaseTool[],
-    options?: BedrockAdditionalChatOptions,
-  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput;
-}
-
-class AnthropicProvider extends Provider<StreamEvent> {
-  getResultFromResponse(
-    response: Record<string, any>,
-  ): AnthropicNoneStreamingResponse {
-    return JSON.parse(toUtf8(response.body));
-  }
-
-  getToolsFromResponse<AnthropicToolContent>(
-    response: Record<string, any>,
-  ): AnthropicToolContent[] {
-    const result = this.getResultFromResponse(response);
-    return result.content
-      .filter((item) => item.type === "tool_use")
-      .map((item) => item as AnthropicToolContent);
-  }
-
-  getTextFromResponse(response: Record<string, any>): string {
-    const result = this.getResultFromResponse(response);
-    return result.content
-      .filter((item) => item.type === "text")
-      .map((item) => (item as AnthropicTextContent).text)
-      .join(" ");
-  }
-
-  getTextFromStreamResponse(response: Record<string, any>): string {
-    const event = this.getStreamingEventResponse(response);
-    if (event?.type === "content_block_delta") {
-      if (event.delta.type === "text_delta") return event.delta.text;
-      if (event.delta.type === "input_json_delta")
-        return event.delta.partial_json;
-    }
-    return "";
-  }
-
-  async *reduceStream(
-    stream: AsyncIterable<ResponseStream>,
-  ): BedrockChatStreamResponse {
-    let collecting = [];
-    let tool: ToolBlock | undefined = undefined;
-    // #TODO this should be broken down into a separate consumer
-    for await (const response of stream) {
-      const event = this.getStreamingEventResponse(response);
-      if (
-        event?.type === "content_block_start" &&
-        event.content_block.type === "tool_use"
-      ) {
-        tool = event.content_block;
-        continue;
-      }
-
-      if (
-        event?.type === "content_block_delta" &&
-        event.delta.type === "input_json_delta"
-      ) {
-        collecting.push(event.delta.partial_json);
-      }
-
-      let options: undefined | ToolCallLLMMessageOptions = undefined;
-      if (tool && collecting.length) {
-        const input = collecting.filter((item) => item).join("");
-        // We have all we need to parse the tool_use json
-        if (event?.type === "content_block_stop") {
-          options = {
-            toolCall: [
-              {
-                id: tool.id,
-                name: tool.name,
-                input: JSON.parse(input),
-              } as ToolCall,
-            ],
-          };
-          // reset the collection/tool
-          collecting = [];
-          tool = undefined;
-        } else {
-          options = {
-            toolCall: [
-              {
-                id: tool.id,
-                name: tool.name,
-                input,
-              } as PartialToolCall,
-            ],
-          };
-        }
-      }
-      const delta = this.getTextFromStreamResponse(response);
-      if (!delta && !options) continue;
-
-      yield {
-        delta,
-        options,
-        raw: response,
-      };
-    }
-  }
-
-  getRequestBody<T extends ChatMessage<ToolCallLLMMessageOptions>>(
-    metadata: LLMMetadata,
-    messages: T[],
-    tools?: BaseTool[],
-    options?: BedrockAdditionalChatOptions,
-  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
-    const extra: Record<string, unknown> = {};
-    if (options?.toolChoice) {
-      extra["tool_choice"] = options?.toolChoice;
-    }
-    const mapped = mapChatMessagesToAnthropicMessages(messages);
-    return {
-      modelId: metadata.model,
-      contentType: "application/json",
-      accept: "application/json",
-      body: JSON.stringify({
-        anthropic_version: "bedrock-2023-05-31",
-        messages: mapped,
-        tools: mapBaseToolsToAnthropicTools(tools),
-        max_tokens: metadata.maxTokens,
-        temperature: metadata.temperature,
-        top_p: metadata.topP,
-        ...extra,
-      }),
-    };
-  }
-}
-
-// Other providers could go here
-const PROVIDERS: { [key: string]: Provider } = {
-  anthropic: new AnthropicProvider(),
-};
-
 const getProvider = (model: string): Provider => {
   const providerName = model.split(".")[0];
   if (!(providerName in PROVIDERS)) {
@@ -373,6 +186,10 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
     this.temperature = temperature ?? DEFAULT_BEDROCK_PARAMS.temperature;
     this.topP = topP ?? DEFAULT_BEDROCK_PARAMS.topP;
     this.client = new BedrockRuntimeClient(params);
+
+    if (!this.supportToolCall) {
+      console.warn(`The model "${this.model}" doesn't support ToolCall`);
+    }
   }
 
   get supportToolCall(): boolean {
@@ -402,10 +219,13 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
     );
     const command = new InvokeModelCommand(input);
     const response = await this.client.send(command);
-    const tools = this.provider.getToolsFromResponse(response);
-    const options: ToolCallLLMMessageOptions = tools.length
-      ? { toolCall: tools }
-      : {};
+    let options: ToolCallLLMMessageOptions = {};
+    if (this.supportToolCall) {
+      const tools = this.provider.getToolsFromResponse(response);
+      if (tools.length) {
+        options = { toolCall: tools };
+      }
+    }
     return {
       raw: response,
       message: {

packages/community/src/llm/bedrock/provider.ts

@@ -0,0 +1,59 @@
+import {
+  type InvokeModelCommandInput,
+  type InvokeModelWithResponseStreamCommandInput,
+  ResponseStream,
+} from "@aws-sdk/client-bedrock-runtime";
+import {
+  type BaseTool,
+  type ChatMessage,
+  type ChatResponseChunk,
+  type LLMMetadata,
+  streamConverter,
+  type ToolCallLLMMessageOptions,
+} from "llamaindex";
+import type { ToolChoice } from "./types";
+import { toUtf8 } from "./utils";
+
+export type BedrockAdditionalChatOptions = { toolChoice: ToolChoice };
+
+export type BedrockChatStreamResponse = AsyncIterable<
+  ChatResponseChunk<ToolCallLLMMessageOptions>
+>;
+
+export abstract class Provider<ProviderStreamEvent extends {} = {}> {
+  abstract getTextFromResponse(response: Record<string, any>): string;
+
+  abstract getToolsFromResponse<T extends {} = {}>(
+    response: Record<string, any>,
+  ): T[];
+
+  getStreamingEventResponse(
+    response: Record<string, any>,
+  ): ProviderStreamEvent | undefined {
+    return response.chunk?.bytes
+      ? (JSON.parse(toUtf8(response.chunk?.bytes)) as ProviderStreamEvent)
+      : undefined;
+  }
+
+  async *reduceStream(
+    stream: AsyncIterable<ResponseStream>,
+  ): BedrockChatStreamResponse {
+    yield* streamConverter(stream, (response) => {
+      return {
+        delta: this.getTextFromStreamResponse(response),
+        raw: response,
+      };
+    });
+  }
+
+  getTextFromStreamResponse(response: Record<string, any>): string {
+    return this.getTextFromResponse(response);
+  }
+
+  abstract getRequestBody<T extends ChatMessage>(
+    metadata: LLMMetadata,
+    messages: T[],
+    tools?: BaseTool[],
+    options?: BedrockAdditionalChatOptions,
+  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput;
+}

packages/community/src/llm/bedrock/providers/anthropic.ts

@@ -0,0 +1,154 @@
+import {
+  type InvokeModelCommandInput,
+  type InvokeModelWithResponseStreamCommandInput,
+  ResponseStream,
+} from "@aws-sdk/client-bedrock-runtime";
+import type {
+  BaseTool,
+  ChatMessage,
+  LLMMetadata,
+  PartialToolCall,
+  ToolCall,
+  ToolCallLLMMessageOptions,
+} from "llamaindex";
+import {
+  type BedrockAdditionalChatOptions,
+  type BedrockChatStreamResponse,
+  Provider,
+} from "../provider";
+import type {
+  AnthropicNoneStreamingResponse,
+  AnthropicStreamEvent,
+  AnthropicTextContent,
+  ToolBlock,
+} from "../types";
+import {
+  mapBaseToolsToAnthropicTools,
+  mapChatMessagesToAnthropicMessages,
+  toUtf8,
+} from "../utils";
+
+export class AnthropicProvider extends Provider<AnthropicStreamEvent> {
+  getResultFromResponse(
+    response: Record<string, any>,
+  ): AnthropicNoneStreamingResponse {
+    return JSON.parse(toUtf8(response.body));
+  }
+
+  getToolsFromResponse<AnthropicToolContent>(
+    response: Record<string, any>,
+  ): AnthropicToolContent[] {
+    const result = this.getResultFromResponse(response);
+    return result.content
+      .filter((item) => item.type === "tool_use")
+      .map((item) => item as AnthropicToolContent);
+  }
+
+  getTextFromResponse(response: Record<string, any>): string {
+    const result = this.getResultFromResponse(response);
+    return result.content
+      .filter((item) => item.type === "text")
+      .map((item) => (item as AnthropicTextContent).text)
+      .join(" ");
+  }
+
+  getTextFromStreamResponse(response: Record<string, any>): string {
+    const event = this.getStreamingEventResponse(response);
+    if (event?.type === "content_block_delta") {
+      if (event.delta.type === "text_delta") return event.delta.text;
+      if (event.delta.type === "input_json_delta")
+        return event.delta.partial_json;
+    }
+    return "";
+  }
+
+  async *reduceStream(
+    stream: AsyncIterable<ResponseStream>,
+  ): BedrockChatStreamResponse {
+    let collecting = [];
+    let tool: ToolBlock | undefined = undefined;
+    // #TODO this should be broken down into a separate consumer
+    for await (const response of stream) {
+      const event = this.getStreamingEventResponse(response);
+      if (
+        event?.type === "content_block_start" &&
+        event.content_block.type === "tool_use"
+      ) {
+        tool = event.content_block;
+        continue;
+      }
+
+      if (
+        event?.type === "content_block_delta" &&
+        event.delta.type === "input_json_delta"
+      ) {
+        collecting.push(event.delta.partial_json);
+      }
+
+      let options: undefined | ToolCallLLMMessageOptions = undefined;
+      if (tool && collecting.length) {
+        const input = collecting.filter((item) => item).join("");
+        // We have all we need to parse the tool_use json
+        if (event?.type === "content_block_stop") {
+          options = {
+            toolCall: [
+              {
+                id: tool.id,
+                name: tool.name,
+                input: JSON.parse(input),
+              } as ToolCall,
+            ],
+          };
+          // reset the collection/tool
+          collecting = [];
+          tool = undefined;
+        } else {
+          options = {
+            toolCall: [
+              {
+                id: tool.id,
+                name: tool.name,
+                input,
+              } as PartialToolCall,
+            ],
+          };
+        }
+      }
+      const delta = this.getTextFromStreamResponse(response);
+      if (!delta && !options) continue;
+
+      yield {
+        delta,
+        options,
+        raw: response,
+      };
+    }
+  }
+
+  getRequestBody<T extends ChatMessage<ToolCallLLMMessageOptions>>(
+    metadata: LLMMetadata,
+    messages: T[],
+    tools?: BaseTool[],
+    options?: BedrockAdditionalChatOptions,
+  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
+    const extra: Record<string, unknown> = {};
+    if (options?.toolChoice) {
+      extra["tool_choice"] = options?.toolChoice;
+    }
+    const mapped = mapChatMessagesToAnthropicMessages(messages);
+    return {
+      modelId: metadata.model,
+      contentType: "application/json",
+      accept: "application/json",
+      body: JSON.stringify({
+        anthropic_version: "bedrock-2023-05-31",
+        messages: mapped,
+        tools: mapBaseToolsToAnthropicTools(tools),
+        max_tokens: metadata.maxTokens,
+        temperature: metadata.temperature,
+        top_p: metadata.topP,
+        ...extra,
+      }),
+    };
+  }
+}

packages/community/src/llm/bedrock/providers/index.ts

@@ -0,0 +1,9 @@
+import { Provider } from "../provider";
+import { AnthropicProvider } from "./anthropic";
+import { MetaProvider } from "./meta";
+
+// Other providers should go here
+export const PROVIDERS: { [key: string]: Provider } = {
+  anthropic: new AnthropicProvider(),
+  meta: new MetaProvider(),
+};

packages/community/src/llm/bedrock/providers/meta.ts

@@ -0,0 +1,69 @@
+import type {
+  InvokeModelCommandInput,
+  InvokeModelWithResponseStreamCommandInput,
+} from "@aws-sdk/client-bedrock-runtime";
+import type { ChatMessage, LLMMetadata } from "llamaindex";
+import type { MetaNoneStreamingResponse, MetaStreamEvent } from "../types";
+import {
+  mapChatMessagesToMetaLlama2Messages,
+  mapChatMessagesToMetaLlama3Messages,
+  toUtf8,
+} from "../utils";
+
+import { Provider } from "../provider";
+
+export class MetaProvider extends Provider<MetaStreamEvent> {
+  constructor() {
+    super();
+  }
+
+  getResultFromResponse(
+    response: Record<string, any>,
+  ): MetaNoneStreamingResponse {
+    return JSON.parse(toUtf8(response.body));
+  }
+
+  getToolsFromResponse(_response: Record<string, any>): never {
+    throw new Error("Not supported by this provider.");
+  }
+
+  getTextFromResponse(response: Record<string, any>): string {
+    const result = this.getResultFromResponse(response);
+    return result.generation;
+  }
+
+  getTextFromStreamResponse(response: Record<string, any>): string {
+    const event = this.getStreamingEventResponse(response);
+    if (event?.generation) {
+      return event.generation;
+    }
+    return "";
+  }
+
+  getRequestBody<T extends ChatMessage>(
+    metadata: LLMMetadata,
+    messages: T[],
+  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
+    let promptFunction: (messages: ChatMessage[]) => string;
+
+    if (metadata.model.startsWith("meta.llama3")) {
+      promptFunction = mapChatMessagesToMetaLlama3Messages;
+    } else if (metadata.model.startsWith("meta.llama2")) {
+      promptFunction = mapChatMessagesToMetaLlama2Messages;
+    } else {
+      throw new Error(`Meta model ${metadata.model} is not supported`);
+    }
+
+    return {
+      modelId: metadata.model,
+      contentType: "application/json",
+      accept: "application/json",
+      body: JSON.stringify({
+        prompt: promptFunction(messages),
+        max_gen_len: metadata.maxTokens,
+        temperature: metadata.temperature,
+        top_p: metadata.topP,
+      }),
+    };
+  }
+}

packages/community/src/llm/bedrock/types.ts

@@ -79,7 +79,7 @@ export type ToolChoice =
   | { type: "auto" }
   | { type: "tool"; name: string };
 
-export type StreamEvent =
+export type AnthropicStreamEvent =
   | { type: "message_start"; message: Message }
   | ContentBlockStart
   | ContentBlockDelta
@@ -93,6 +93,8 @@ export type AnthropicContent =
   | AnthropicToolContent
   | AnthropicToolResultContent;
 
+export type MetaTextContent = string;
+
 export type AnthropicTextContent = {
   type: "text";
   text: string;
@@ -133,6 +135,11 @@ export type AnthropicMessage = {
   content: AnthropicContent[];
 };
 
+export type MetaMessage = {
+  role: "user" | "assistant" | "system";
+  content: MetaTextContent;
+};
+
 export type AnthropicNoneStreamingResponse = {
   id: string;
   type: "message";
@@ -143,3 +150,16 @@ export type AnthropicNoneStreamingResponse = {
   stop_sequence?: string;
   usage: { input_tokens: number; output_tokens: number };
 };
+
+type MetaResponse = {
+  generation: string;
+  prompt_token_count: number;
+  generation_token_count: number;
+  stop_reason: "stop" | "length";
+};
+
+export type MetaStreamEvent = MetaResponse & {
+  "amazon-bedrock-invocationMetrics": InvocationMetrics;
+};
+
+export type MetaNoneStreamingResponse = MetaResponse;

packages/community/src/llm/bedrock/utils.ts

@@ -4,6 +4,7 @@ import type {
   JSONObject,
   MessageContent,
   MessageContentDetail,
+  MessageContentTextDetail,
   ToolCallLLMMessageOptions,
   ToolMetadata,
 } from "llamaindex";
@@ -13,6 +14,7 @@ import type {
   AnthropicMediaTypes,
   AnthropicMessage,
   AnthropicTextContent,
+  MetaMessage,
 } from "./types.js";
 
 const ACCEPTED_IMAGE_MIME_TYPES = [
@@ -148,6 +150,85 @@ export const mapChatMessagesToAnthropicMessages = <
   return mergeNeighboringSameRoleMessages(mapped);
 };
 
+export const mapChatMessagesToMetaMessages = <T extends ChatMessage>(
+  messages: T[],
+): MetaMessage[] => {
+  return messages.map((msg) => {
+    let content: string;
+    if (typeof msg.content === "string") {
+      content = msg.content;
+    } else {
+      content = (msg.content[0] as MessageContentTextDetail).text;
+    }
+    return {
+      role:
+        msg.role === "assistant"
+          ? "assistant"
+          : msg.role === "user"
+            ? "user"
+            : "system",
+      content,
+    };
+  });
+};
+
+/**
+ * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
+ */
+export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>(
+  messages: T[],
+): string => {
+  const mapped = mapChatMessagesToMetaMessages(messages).map((message) => {
+    const text = message.content;
+    return `<|start_header_id|>${message.role}<|end_header_id|>\n${text}\n<|eot_id|>\n`;
+  });
+  return (
+    "<|begin_of_text|>" +
+    mapped.join("\n") +
+    "\n<|start_header_id|>assistant<|end_header_id|>\n"
+  );
+};
+
+/**
+ * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-2
+ */
+export const mapChatMessagesToMetaLlama2Messages = <T extends ChatMessage>(
+  messages: T[],
+): string => {
+  const mapped = mapChatMessagesToMetaMessages(messages);
+  let output = "<s>";
+  let insideInst = false;
+  let needsStartAgain = false;
+  for (const message of mapped) {
+    if (needsStartAgain) {
+      output += "<s>";
+      needsStartAgain = false;
+    }
+    const text = message.content;
+    if (message.role === "system") {
+      if (!insideInst) {
+        output += "[INST] ";
+        insideInst = true;
+      }
+      output += `<<SYS>>\n${text}\n<</SYS>>\n`;
+    } else if (message.role === "user") {
+      output += text;
+      if (insideInst) {
+        output += " [/INST]";
+        insideInst = false;
+      }
+    } else if (message.role === "assistant") {
+      if (insideInst) {
+        output += " [/INST]";
+        insideInst = false;
+      }
+      output += ` ${text} </s>\n`;
+      needsStartAgain = true;
+    }
+  }
+  return output;
+};
+
 export const mapTextContent = (text: string): AnthropicTextContent => {
   return { type: "text", text };
 };

packages/core/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/core
 
+## 0.0.3
+
+### Patch Changes
+
+- f326ab8: chore: bump version
+- Updated dependencies [f326ab8]
+  - @llamaindex/env@0.1.8
+
 ## 0.0.2
 
 ### Patch Changes

packages/core/package.json

@@ -1,9 +1,23 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "LlamaIndex Core Module",
   "exports": {
+    "./llms": {
+      "require": {
+        "types": "./dist/llms/index.d.cts",
+        "default": "./dist/llms/index.cjs"
+      },
+      "import": {
+        "types": "./dist/llms/index.d.ts",
+        "default": "./dist/llms/index.js"
+      },
+      "default": {
+        "types": "./dist/llms/index.d.ts",
+        "default": "./dist/llms/index.js"
+      }
+    },
     "./decorator": {
       "require": {
         "types": "./dist/decorator/index.d.cts",
@@ -60,6 +74,7 @@
     "url": "https://github.com/himself65/LlamaIndexTS.git"
   },
   "devDependencies": {
+    "ajv": "^8.16.0",
     "bunchee": "^5.2.1"
   },
   "dependencies": {

packages/core/src/global/type.ts

@@ -0,0 +1,9 @@
+export type UUID = `${string}-${string}-${string}-${string}-${string}`;
+
+export type JSONValue = string | number | boolean | JSONObject | JSONArray;
+
+export type JSONObject = {
+  [key: string]: JSONValue;
+};
+
+export type JSONArray = Array<JSONValue>;

packages/core/src/index.ts

@@ -1 +0,0 @@
-export * from "./schema";

packages/core/src/llms/index.ts

@@ -0,0 +1,31 @@
+export type {
+  BaseTool,
+  BaseToolWithCall,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChat,
+  LLMChatParamsBase,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsBase,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+  MessageContent,
+  MessageContentDetail,
+  MessageContentImageDetail,
+  MessageContentTextDetail,
+  MessageType,
+  PartialToolCall,
+  TextChatMessage,
+  ToolCall,
+  ToolCallLLMMessageOptions,
+  ToolCallOptions,
+  ToolMetadata,
+  ToolOutput,
+  ToolResult,
+  ToolResultOptions,
+} from "./type";

packages/core/src/llms/type.ts

@@ -0,0 +1,245 @@
+import type { Tokenizers } from "@llamaindex/env";
+import type { JSONSchemaType } from "ajv";
+import type { JSONObject, JSONValue } from "../global/type";
+
+/**
+ * @internal
+ */
+export interface LLMChat<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> {
+  chat(
+    params:
+      | LLMChatParamsStreaming<AdditionalChatOptions>
+      | LLMChatParamsNonStreaming<AdditionalChatOptions>,
+  ): Promise<
+    | ChatResponse<AdditionalMessageOptions>
+    | AsyncIterable<ChatResponseChunk<AdditionalMessageOptions>>
+  >;
+}
+
+/**
+ * Unified language model interface
+ */
+export interface LLM<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChat<AdditionalChatOptions> {
+  metadata: LLMMetadata;
+  /**
+   * Get a chat response from the LLM
+   */
+  chat(
+    params: LLMChatParamsStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(
+    params: LLMChatParamsNonStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
+  ): Promise<ChatResponse<AdditionalMessageOptions>>;
+
+  /**
+   * Get a prompt completion from the LLM
+   */
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+}
+
+export type MessageType = "user" | "assistant" | "system" | "memory";
+
+export type TextChatMessage<AdditionalMessageOptions extends object = object> =
+  {
+    content: string;
+    role: MessageType;
+    options?: undefined | AdditionalMessageOptions;
+  };
+
+export type ChatMessage<AdditionalMessageOptions extends object = object> = {
+  content: MessageContent;
+  role: MessageType;
+  options?: undefined | AdditionalMessageOptions;
+};
+
+export interface ChatResponse<
+  AdditionalMessageOptions extends object = object,
+> {
+  message: ChatMessage<AdditionalMessageOptions>;
+  /**
+   * Raw response from the LLM
+   *
+   * If LLM response an iterable of chunks, this will be an array of those chunks
+   */
+  raw: object | null;
+}
+
+export type ChatResponseChunk<
+  AdditionalMessageOptions extends object = object,
+> = {
+  raw: object | null;
+  delta: string;
+  options?: undefined | AdditionalMessageOptions;
+};
+
+export interface CompletionResponse {
+  text: string;
+  /**
+   * Raw response from the LLM
+   *
+   * It's possible that this is `null` if the LLM response an iterable of chunks
+   */
+  raw: object | null;
+}
+
+export type LLMMetadata = {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  contextWindow: number;
+  tokenizer: Tokenizers | undefined;
+};
+
+export interface LLMChatParamsBase<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> {
+  messages: ChatMessage<AdditionalMessageOptions>[];
+  additionalChatOptions?: AdditionalChatOptions;
+  tools?: BaseTool[];
+}
+
+export interface LLMChatParamsStreaming<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
+  stream: true;
+}
+
+export interface LLMChatParamsNonStreaming<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
+  stream?: false;
+}
+
+export interface LLMCompletionParamsBase {
+  prompt: MessageContent;
+}
+
+export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
+  stream: true;
+}
+
+export interface LLMCompletionParamsNonStreaming
+  extends LLMCompletionParamsBase {
+  stream?: false | null;
+}
+
+export type MessageContentTextDetail = {
+  type: "text";
+  text: string;
+};
+
+export type MessageContentImageDetail = {
+  type: "image_url";
+  image_url: { url: string };
+};
+
+export type MessageContentDetail =
+  | MessageContentTextDetail
+  | MessageContentImageDetail;
+
+/**
+ * Extended type for the content of a message that allows for multi-modal messages.
+ */
+export type MessageContent = string | MessageContentDetail[];
+
+export type ToolCall = {
+  name: string;
+  input: JSONObject;
+  id: string;
+};
+
+// happened in streaming response, the tool call is not ready yet
+export type PartialToolCall = {
+  name: string;
+  id: string;
+  // input is not ready yet, JSON.parse(input) will throw an error
+  input: string;
+};
+
+export type ToolResult = {
+  id: string;
+  result: string;
+  isError: boolean;
+};
+
+export type ToolCallOptions = {
+  toolCall: (ToolCall | PartialToolCall)[];
+};
+
+export type ToolResultOptions = {
+  toolResult: ToolResult;
+};
+
+export type ToolCallLLMMessageOptions =
+  | ToolResultOptions
+  | ToolCallOptions
+  | {};
+
+type Known =
+  | { [key: string]: Known }
+  | [Known, ...Known[]]
+  | Known[]
+  | number
+  | string
+  | boolean
+  | null;
+
+export type ToolMetadata<
+  Parameters extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  description: string;
+  name: string;
+  /**
+   * OpenAI uses JSON Schema to describe the parameters that a tool can take.
+   * @link https://json-schema.org/understanding-json-schema
+   */
+  parameters?: Parameters;
+};
+
+/**
+ * Simple Tool interface. Likely to change.
+ */
+export interface BaseTool<Input = any> {
+  /**
+   * This could be undefined if the implementation is not provided,
+   *  which might be the case when communicating with a llm.
+   *
+   * @return {JSONValue | Promise<JSONValue>} The output of the tool.
+   */
+  call?: (input: Input) => JSONValue | Promise<JSONValue>;
+  metadata: // if user input any, we cannot check the schema
+  Input extends Known ? ToolMetadata<JSONSchemaType<Input>> : ToolMetadata;
+}
+
+export type BaseToolWithCall<Input = any> = Omit<BaseTool<Input>, "call"> & {
+  call: NonNullable<Pick<BaseTool<Input>, "call">["call"]>;
+};
+
+export type ToolOutput = {
+  tool: BaseTool | undefined;
+  // all of existing function calling LLMs only support object input
+  input: JSONObject;
+  output: JSONValue;
+  isError: boolean;
+};

packages/core/src/schema/zod.ts

@@ -1,5 +1,7 @@
 import { z } from "zod";
 
+export const anyFunctionSchema = z.function(z.tuple([]).rest(z.any()), z.any());
+
 export const toolMetadataSchema = z.object({
   description: z.string(),
   name: z.string(),
@@ -7,7 +9,7 @@ export const toolMetadataSchema = z.object({
 });
 
 export const baseToolSchema = z.object({
-  call: z.optional(z.function()),
+  call: anyFunctionSchema.optional(),
   metadata: toolMetadataSchema,
 });
 

packages/env/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/env
 
+## 0.1.8
+
+### Patch Changes
+
+- f326ab8: chore: bump version
+
 ## 0.1.7
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/experimental/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/experimental
 
+## 0.0.47
+
+### Patch Changes
+
+- Updated dependencies [f326ab8]
+  - llamaindex@0.4.12
+
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [8bf5b4a]
+  - llamaindex@0.4.11
+
+## 0.0.45
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.43
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.43",
+  "version": "0.0.47",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,33 @@
 # llamaindex
 
+## 0.4.12
+
+### Patch Changes
+
+- f326ab8: chore: bump version
+- Updated dependencies [f326ab8]
+  - @llamaindex/cloud@0.1.2
+  - @llamaindex/core@0.0.3
+  - @llamaindex/env@0.1.8
+
+## 0.4.11
+
+### Patch Changes
+
+- 8bf5b4a: fix: llama parse input spreadsheet
+
+## 0.4.10
+
+### Patch Changes
+
+- 7dce3d2: fix: disable External Filters for Gemini
+
+## 0.4.9
+
+### Patch Changes
+
+- 3a96a48: fix: anthroipic image input
+
 ## 0.4.8
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.31
+
+### Patch Changes
+
+- Updated dependencies [f326ab8]
+  - llamaindex@0.4.12
+
+## 0.0.30
+
+### Patch Changes
+
+- Updated dependencies [8bf5b4a]
+  - llamaindex@0.4.11
+
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.27
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.27",
+  "version": "0.0.31",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/next-agent-test
 
+## 0.1.31
+
+### Patch Changes
+
+- Updated dependencies [f326ab8]
+  - llamaindex@0.4.12
+
+## 0.1.30
+
+### Patch Changes
+
+- Updated dependencies [8bf5b4a]
+  - llamaindex@0.4.11
+
+## 0.1.29
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.1.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.1.27
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.27",
+  "version": "0.1.31",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-agent/src/actions/index.tsx

@@ -1,7 +1,7 @@
 "use server";
 import { createStreamableUI } from "ai/rsc";
+import type { ChatMessage } from "llamaindex";
 import { OpenAIAgent } from "llamaindex";
-import type { ChatMessage } from "llamaindex/llm/types";
 
 export async function chatWithAgent(
   question: string,

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,33 @@
 # test-edge-runtime
 
+## 0.1.30
+
+### Patch Changes
+
+- Updated dependencies [f326ab8]
+  - llamaindex@0.4.12
+
+## 0.1.29
+
+### Patch Changes
+
+- Updated dependencies [8bf5b4a]
+  - llamaindex@0.4.11
+
+## 0.1.28
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.1.27
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.1.26
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.26",
+  "version": "0.1.30",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [f326ab8]
+  - llamaindex@0.4.12
+
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [8bf5b4a]
+  - llamaindex@0.4.11
+
+## 0.0.10
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.9
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.8
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.8",
+  "version": "0.0.12",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.31
+
+### Patch Changes
+
+- Updated dependencies [f326ab8]
+  - llamaindex@0.4.12
+
+## 0.0.30
+
+### Patch Changes
+
+- Updated dependencies [8bf5b4a]
+  - llamaindex@0.4.11
+
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [7dce3d2]
+  - llamaindex@0.4.10
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
 ## 0.0.27
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.27",
+  "version": "0.0.31",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/fixtures/llm/openai.ts

@@ -7,7 +7,7 @@ import type {
   LLMChatParamsStreaming,
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
-} from "llamaindex/llm/types";
+} from "llamaindex";
 import { extractText } from "llamaindex/llm/utils";
 import { deepStrictEqual, strictEqual } from "node:assert";
 import { llmCompleteMockStorage } from "../../node/utils.js";

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.4.8",
+  "version": "0.4.12",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/ChatHistory.ts

@@ -1,8 +1,8 @@
+import type { ChatMessage, LLM, MessageType } from "@llamaindex/core/llms";
 import { tokenizers, type Tokenizer } from "@llamaindex/env";
 import type { SummaryPrompt } from "./Prompt.js";
 import { defaultSummaryPrompt, messagesToHistoryStr } from "./Prompt.js";
 import { OpenAI } from "./llm/openai.js";
-import type { ChatMessage, LLM, MessageType } from "./llm/types.js";
 import { extractText } from "./llm/utils.js";
 
 /**

packages/llamaindex/src/EngineResponse.ts

@@ -1,9 +1,9 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
 import type {
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
-} from "./llm/types.js";
+} from "@llamaindex/core/llms";
+import type { NodeWithScore } from "@llamaindex/core/schema";
 import { extractText } from "./llm/utils.js";
 
 export class EngineResponse implements ChatResponse, ChatResponseChunk {

packages/llamaindex/src/Prompt.ts

@@ -1,6 +1,5 @@
+import type { ChatMessage, ToolMetadata } from "@llamaindex/core/llms";
 import type { SubQuestion } from "./engines/query/types.js";
-import type { ChatMessage } from "./llm/types.js";
-import type { ToolMetadata } from "./types.js";
 
 /**
  * A SimplePrompt is a function that takes a dictionary of inputs and returns a string.

packages/llamaindex/src/QuestionGenerator.ts

@@ -1,3 +1,4 @@
+import type { LLM, ToolMetadata } from "@llamaindex/core/llms";
 import { SubQuestionOutputParser } from "./OutputParser.js";
 import type { SubQuestionPrompt } from "./Prompt.js";
 import { buildToolsText, defaultSubQuestionPrompt } from "./Prompt.js";
@@ -6,13 +7,8 @@ import type {
   SubQuestion,
 } from "./engines/query/types.js";
 import { OpenAI } from "./llm/openai.js";
-import type { LLM } from "./llm/types.js";
 import { PromptMixin } from "./prompts/index.js";
-import type {
-  BaseOutputParser,
-  StructuredOutput,
-  ToolMetadata,
-} from "./types.js";
+import type { BaseOutputParser, StructuredOutput } from "./types.js";
 
 /**
  * LLMQuestionGenerator uses the LLM to generate new questions for the LLM using tools and a user query.

packages/llamaindex/src/ServiceContext.ts

@@ -1,8 +1,8 @@
+import type { LLM } from "@llamaindex/core/llms";
 import { PromptHelper } from "./PromptHelper.js";
 import { OpenAIEmbedding } from "./embeddings/OpenAIEmbedding.js";
 import type { BaseEmbedding } from "./embeddings/types.js";
 import { OpenAI } from "./llm/openai.js";
-import type { LLM } from "./llm/types.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 import type { NodeParser } from "./nodeParsers/types.js";
 

packages/llamaindex/src/Settings.ts

@@ -5,6 +5,7 @@ import { OpenAI } from "./llm/openai.js";
 import { PromptHelper } from "./PromptHelper.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 
+import type { LLM } from "@llamaindex/core/llms";
 import { AsyncLocalStorage, getEnv } from "@llamaindex/env";
 import type { ServiceContext } from "./ServiceContext.js";
 import type { BaseEmbedding } from "./embeddings/types.js";
@@ -18,7 +19,6 @@ import {
   setEmbeddedModel,
   withEmbeddedModel,
 } from "./internal/settings/EmbedModel.js";
-import type { LLM } from "./llm/types.js";
 import type { NodeParser } from "./nodeParsers/types.js";
 
 export type PromptConfig = {

packages/llamaindex/src/agent/base.ts

@@ -1,3 +1,10 @@
+import type {
+  BaseToolWithCall,
+  ChatMessage,
+  LLM,
+  MessageContent,
+  ToolOutput,
+} from "@llamaindex/core/llms";
 import { ReadableStream, TransformStream, randomUUID } from "@llamaindex/env";
 import { ChatHistory } from "../ChatHistory.js";
 import { EngineResponse } from "../EngineResponse.js";
@@ -11,9 +18,7 @@ import { wrapEventCaller } from "../internal/context/EventCaller.js";
 import { consoleLogger, emptyLogger } from "../internal/logger.js";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import { isAsyncIterable } from "../internal/utils.js";
-import type { ChatMessage, LLM, MessageContent } from "../llm/index.js";
 import { ObjectRetriever } from "../objects/index.js";
-import type { BaseToolWithCall, ToolOutput } from "../types.js";
 import type {
   AgentTaskContext,
   TaskHandler,
@@ -229,13 +234,12 @@ export abstract class AgentRunner<
     const { llm, getTools, stream } = step.context;
     const lastMessage = step.context.store.messages.at(-1)!.content;
     const tools = await getTools(lastMessage);
-    const response = await llm.chat({
-      // @ts-expect-error
-      stream,
-      tools,
-      messages: [...step.context.store.messages],
-    });
     if (!stream) {
+      const response = await llm.chat({
+        stream,
+        tools,
+        messages: [...step.context.store.messages],
+      });
       await stepTools<LLM>({
         response,
         tools,
@@ -243,6 +247,11 @@ export abstract class AgentRunner<
         enqueueOutput,
       });
     } else {
+      const response = await llm.chat({
+        stream,
+        tools,
+        messages: [...step.context.store.messages],
+      });
       await stepToolsStreaming<LLM>({
         response,
         tools,

packages/llamaindex/src/agent/llm.ts

@@ -1,7 +1,6 @@
-import type { LLM } from "../llm/index.js";
+import type { BaseToolWithCall, LLM } from "@llamaindex/core/llms";
 import { ObjectRetriever } from "../objects/index.js";
 import { Settings } from "../Settings.js";
-import type { BaseToolWithCall } from "../types.js";
 import { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
 import { validateAgentParams } from "./utils.js";
 

packages/llamaindex/src/agent/react.ts

@@ -1,18 +1,19 @@
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLM,
+} from "@llamaindex/core/llms";
 import { randomUUID, ReadableStream } from "@llamaindex/env";
 import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
 import {
   isAsyncIterable,
   stringifyJSONToMessageContent,
 } from "../internal/utils.js";
-import {
-  type ChatMessage,
-  type ChatResponse,
-  type ChatResponseChunk,
-  type LLM,
-} from "../llm/index.js";
 import { extractText } from "../llm/utils.js";
 import { Settings } from "../Settings.js";
-import type { BaseTool, JSONObject, JSONValue } from "../types.js";
+import type { JSONObject, JSONValue } from "../types.js";
 import { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
 import type { TaskHandler } from "./types.js";
 import {

packages/llamaindex/src/agent/types.ts

@@ -1,14 +1,16 @@
-import { ReadableStream } from "@llamaindex/env";
-import type { Logger } from "../internal/logger.js";
-import type { BaseEvent } from "../internal/type.js";
 import type {
+  BaseToolWithCall,
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLM,
   MessageContent,
-} from "../llm/types.js";
-import type { BaseToolWithCall, ToolOutput, UUID } from "../types.js";
+  ToolOutput,
+} from "@llamaindex/core/llms";
+import { ReadableStream } from "@llamaindex/env";
+import type { Logger } from "../internal/logger.js";
+import type { BaseEvent } from "../internal/type.js";
+import type { UUID } from "../types.js";
 
 export type AgentTaskContext<
   Model extends LLM,

packages/llamaindex/src/agent/utils.ts

@@ -1,3 +1,15 @@
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLM,
+  PartialToolCall,
+  TextChatMessage,
+  ToolCall,
+  ToolCallLLMMessageOptions,
+  ToolOutput,
+} from "@llamaindex/core/llms";
 import { baseToolWithCallSchema } from "@llamaindex/core/schema";
 import { ReadableStream } from "@llamaindex/env";
 import { z } from "zod";
@@ -8,17 +20,7 @@ import {
   prettifyError,
   stringifyJSONToMessageContent,
 } from "../internal/utils.js";
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-  LLM,
-  PartialToolCall,
-  TextChatMessage,
-  ToolCall,
-  ToolCallLLMMessageOptions,
-} from "../llm/index.js";
-import type { BaseTool, JSONObject, JSONValue, ToolOutput } from "../types.js";
+import type { JSONObject, JSONValue } from "../types.js";
 import type { AgentParamsBase } from "./base.js";
 import type { TaskHandler } from "./types.js";
 
@@ -31,10 +33,12 @@ type StepToolsResponseParams<Model extends LLM> = {
   >[1];
 };
 
-type StepToolsStreamingResponseParams<Model extends LLM> =
-  StepToolsResponseParams<Model> & {
-    response: AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>;
-  };
+type StepToolsStreamingResponseParams<Model extends LLM> = Omit<
+  StepToolsResponseParams<Model>,
+  "response"
+> & {
+  response: AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>;
+};
 
 // #TODO stepTools and stepToolsStreaming should be moved to a better abstraction
 
@@ -83,7 +87,7 @@ export async function stepToolsStreaming<Model extends LLM>({
       }
     }
 
-    // If there are toolCalls but they didn't get read into the stream, used for Gemini
+    // If there are toolCalls, but they didn't get read into the stream, used for Gemini
     if (!toolCalls.size && value.options && "toolCall" in value.options) {
       value.options.toolCall.forEach((toolCall) => {
         toolCalls.set(toolCall.id, toolCall);

packages/llamaindex/src/callbacks/CallbackManager.ts

@@ -1,4 +1,5 @@
 import type { Anthropic } from "@anthropic-ai/sdk";
+import type { MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import { CustomEvent } from "@llamaindex/env";
 import type { AgentEndEvent, AgentStartEvent } from "../agent/types.js";
@@ -12,7 +13,6 @@ import type {
   LLMStreamEvent,
   LLMToolCallEvent,
   LLMToolResultEvent,
-  MessageContent,
   RetrievalEndEvent,
   RetrievalStartEvent,
 } from "../llm/types.js";

packages/llamaindex/src/embeddings/DeepInfraEmbedding.ts

@@ -1,5 +1,5 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import { getEnv } from "@llamaindex/env";
-import type { MessageContentDetail } from "../llm/index.js";
 import { extractSingleText } from "../llm/utils.js";
 import { BaseEmbedding } from "./types.js";
 

packages/llamaindex/src/embeddings/MultiModalEmbedding.ts

@@ -1,3 +1,4 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import {
   ImageNode,
   MetadataMode,
@@ -6,7 +7,6 @@ import {
   type BaseNode,
   type ImageType,
 } from "@llamaindex/core/schema";
-import type { MessageContentDetail } from "../llm/types.js";
 import { extractImage, extractSingleText } from "../llm/utils.js";
 import { BaseEmbedding, batchEmbeddings } from "./types.js";
 

packages/llamaindex/src/embeddings/types.ts

@@ -1,8 +1,8 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import type { BaseNode } from "@llamaindex/core/schema";
 import { MetadataMode } from "@llamaindex/core/schema";
 import { type Tokenizers } from "@llamaindex/env";
 import type { TransformComponent } from "../ingestion/types.js";
-import type { MessageContentDetail } from "../llm/types.js";
 import { extractSingleText } from "../llm/utils.js";
 import { truncateMaxTokens } from "./tokenizer.js";
 import { SimilarityType, similarity } from "./utils.js";

packages/llamaindex/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -1,3 +1,4 @@
+import type { ChatMessage, LLM } from "@llamaindex/core/llms";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import type { EngineResponse } from "../../EngineResponse.js";
@@ -9,7 +10,6 @@ import {
 import type { ServiceContext } from "../../ServiceContext.js";
 import { llmFromSettingsOrContext } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import type { ChatMessage, LLM } from "../../llm/index.js";
 import { extractText, streamReducer } from "../../llm/utils.js";
 import { PromptMixin } from "../../prompts/index.js";
 import type { QueryEngine } from "../../types.js";

packages/llamaindex/src/engines/chat/ContextChatEngine.ts

@@ -1,3 +1,9 @@
+import type {
+  ChatMessage,
+  LLM,
+  MessageContent,
+  MessageType,
+} from "@llamaindex/core/llms";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import { EngineResponse } from "../../EngineResponse.js";
@@ -5,8 +11,6 @@ import type { ContextSystemPrompt } from "../../Prompt.js";
 import type { BaseRetriever } from "../../Retriever.js";
 import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import type { ChatMessage, LLM } from "../../llm/index.js";
-import type { MessageContent, MessageType } from "../../llm/types.js";
 import {
   extractText,
   streamConverter,

packages/llamaindex/src/engines/chat/DefaultContextGenerator.ts

@@ -1,10 +1,10 @@
+import type { MessageContent, MessageType } from "@llamaindex/core/llms";
 import { type NodeWithScore } from "@llamaindex/core/schema";
+import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import { defaultContextSystemPrompt } from "../../Prompt.js";
-import type { BaseRetriever } from "../../Retriever.js";
-import type { MessageContent, MessageType } from "../../llm/types.js";
-import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import { PromptMixin } from "../../prompts/index.js";
+import type { BaseRetriever } from "../../Retriever.js";
 import { createMessageContent } from "../../synthesizers/utils.js";
 import type { Context, ContextGenerator } from "./types.js";
 

packages/llamaindex/src/engines/chat/SimpleChatEngine.ts

@@ -1,9 +1,9 @@
+import type { LLM } from "@llamaindex/core/llms";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import { EngineResponse } from "../../EngineResponse.js";
 import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import type { LLM } from "../../llm/index.js";
 import { streamConverter, streamReducer } from "../../llm/utils.js";
 import type {
   ChatEngine,

packages/llamaindex/src/engines/chat/types.ts

@@ -1,8 +1,7 @@
+import type { ChatMessage, MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import type { ChatHistory } from "../../ChatHistory.js";
 import type { EngineResponse } from "../../EngineResponse.js";
-import type { ChatMessage } from "../../llm/index.js";
-import type { MessageContent } from "../../llm/types.js";
 
 /**
  * Represents the base parameters for ChatEngine.

packages/llamaindex/src/engines/query/SubQuestionQueryEngine.ts

@@ -11,13 +11,12 @@ import {
 } from "../../synthesizers/index.js";
 
 import type {
-  BaseTool,
   QueryEngine,
   QueryEngineParamsNonStreaming,
   QueryEngineParamsStreaming,
-  ToolMetadata,
 } from "../../types.js";
 
+import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import type { BaseQuestionGenerator, SubQuestion } from "./types.js";
 

packages/llamaindex/src/engines/query/types.ts

@@ -1,4 +1,4 @@
-import type { ToolMetadata } from "../../types.js";
+import type { ToolMetadata } from "@llamaindex/core/llms";
 
 /**
  * QuestionGenerators generate new questions for the LLM using tools and a user query.

packages/llamaindex/src/evaluation/Correctness.ts

@@ -1,9 +1,9 @@
+import type { ChatMessage, LLM } from "@llamaindex/core/llms";
 import { MetadataMode } from "@llamaindex/core/schema";
-import type { ServiceContext } from "../ServiceContext.js";
-import { llmFromSettingsOrContext } from "../Settings.js";
-import type { ChatMessage, LLM } from "../llm/types.js";
 import { extractText } from "../llm/utils.js";
 import { PromptMixin } from "../prompts/Mixin.js";
+import type { ServiceContext } from "../ServiceContext.js";
+import { llmFromSettingsOrContext } from "../Settings.js";
 import type { CorrectnessSystemPrompt } from "./prompts.js";
 import {
   defaultCorrectnessSystemPrompt,

packages/llamaindex/src/extractors/MetadataExtractors.ts

@@ -1,6 +1,6 @@
+import type { LLM } from "@llamaindex/core/llms";
 import type { BaseNode } from "@llamaindex/core/schema";
 import { MetadataMode, TextNode } from "@llamaindex/core/schema";
-import type { LLM } from "../llm/index.js";
 import { OpenAI } from "../llm/index.js";
 import {
   defaultKeywordExtractorPromptTemplate,

packages/llamaindex/src/index.edge.ts

@@ -1,3 +1,4 @@
+export * from "@llamaindex/core/llms";
 export * from "@llamaindex/core/schema";
 export * from "./agent/index.js";
 export * from "./callbacks/CallbackManager.js";

packages/llamaindex/src/indices/keyword/index.ts

@@ -31,8 +31,8 @@ import {
   simpleExtractKeywords,
 } from "./utils.js";
 
+import type { LLM } from "@llamaindex/core/llms";
 import { llmFromSettingsOrContext } from "../../Settings.js";
-import type { LLM } from "../../llm/types.js";
 import { extractText } from "../../llm/utils.js";
 
 export interface KeywordIndexOptions {

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -1,3 +1,4 @@
+import type { MessageContent } from "@llamaindex/core/llms";
 import {
   ImageNode,
   ModalityType,
@@ -23,7 +24,6 @@ import {
 } from "../../ingestion/strategies/index.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
-import type { MessageContent } from "../../llm/types.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/types.js";
 import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";

packages/llamaindex/src/internal/prompt/react.ts

@@ -1,4 +1,4 @@
-import type { BaseTool } from "../../types.js";
+import type { BaseTool } from "@llamaindex/core/llms";
 
 export const getReACTAgentSystemHeader = (tools: BaseTool[]) => {
   const description = tools

packages/llamaindex/src/llm/anthropic.ts

@@ -13,19 +13,22 @@ import type {
   TextBlock,
   TextBlockParam,
 } from "@anthropic-ai/sdk/resources/index";
-import type { MessageParam } from "@anthropic-ai/sdk/resources/messages";
-import { getEnv } from "@llamaindex/env";
-import _ from "lodash";
-import type { BaseTool } from "../types.js";
-import { ToolCallLLM } from "./base.js";
 import type {
+  ImageBlockParam,
+  MessageParam,
+} from "@anthropic-ai/sdk/resources/messages";
+import type {
+  BaseTool,
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
   ToolCallLLMMessageOptions,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import _ from "lodash";
+import { ToolCallLLM } from "./base.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 export class AnthropicSession {
@@ -214,7 +217,32 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
         }
 
         return {
-          content: extractText(message.content),
+          content:
+            typeof message.content === "string"
+              ? message.content
+              : message.content.map(
+                  (content): TextBlockParam | ImageBlockParam =>
+                    content.type === "text"
+                      ? {
+                          type: "text",
+                          text: content.text,
+                        }
+                      : {
+                          type: "image",
+                          source: {
+                            data: content.image_url.url.substring(
+                              content.image_url.url.indexOf(",") + 1,
+                            ),
+                            media_type:
+                              `image/${content.image_url.url.substring("data:image/".length, content.image_url.url.indexOf(";base64"))}` as
+                                | "image/jpeg"
+                                | "image/png"
+                                | "image/gif"
+                                | "image/webp",
+                            type: "base64",
+                          },
+                        },
+                ),
           role: message.role as "user" | "assistant",
         } satisfies MessageParam;
       });

packages/llamaindex/src/llm/base.ts

@@ -9,7 +9,7 @@ import type {
   LLMCompletionParamsStreaming,
   LLMMetadata,
   ToolCallLLMMessageOptions,
-} from "./types.js";
+} from "@llamaindex/core/llms";
 import { extractText, streamConverter } from "./utils.js";
 
 export abstract class BaseLLM<

packages/llamaindex/src/llm/gemini/base.ts

@@ -7,8 +7,6 @@ import {
   type GenerateContentStreamResult as GoogleStreamGenerateContentResult,
 } from "@google/generative-ai";
 
-import { getEnv, randomUUID } from "@llamaindex/env";
-import { ToolCallLLM } from "../base.js";
 import type {
   CompletionResponse,
   LLMCompletionParamsNonStreaming,
@@ -16,7 +14,9 @@ import type {
   LLMMetadata,
   ToolCall,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
+import { getEnv, randomUUID } from "@llamaindex/env";
+import { ToolCallLLM } from "../base.js";
 import { streamConverter, wrapLLMEvent } from "../utils.js";
 import {
   GEMINI_BACKENDS,
@@ -33,6 +33,7 @@ import {
   type IGeminiSession,
 } from "./types.js";
 import {
+  DEFAULT_SAFETY_SETTINGS,
   GeminiHelper,
   getChatContext,
   getPartsText,
@@ -87,7 +88,10 @@ export class GeminiSession implements IGeminiSession {
   }
 
   getGenerativeModel(metadata: GoogleModelParams): GoogleGenerativeModel {
-    return this.gemini.getGenerativeModel(metadata);
+    return this.gemini.getGenerativeModel({
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
+      ...metadata,
+    });
   }
 
   getResponseText(response: EnhancedGenerateContentResponse): string {
@@ -143,8 +147,9 @@ export class GeminiSessionStore {
   }> = [];
 
   private static getSessionId(options: GeminiSessionOptions): string {
-    if (options.backend === GEMINI_BACKENDS.GOOGLE)
+    if (options.backend === GEMINI_BACKENDS.GOOGLE) {
       return options?.apiKey ?? "";
+    }
     return "";
   }
   private static sessionMatched(
@@ -223,6 +228,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
           ),
         },
       ],
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
     });
     const { response } = await chat.sendMessage(context.message);
     const topCandidate = response.candidates![0];
@@ -258,6 +264,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
           ),
         },
       ],
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
     });
     const result = await chat.sendMessageStream(context.message);
     yield* this.session.getChatStream(result);

packages/llamaindex/src/llm/gemini/types.ts

@@ -33,7 +33,7 @@ import type {
   LLMChatParamsStreaming,
   ToolCall,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
 
 export enum GEMINI_BACKENDS {
   GOOGLE = "google",

packages/llamaindex/src/llm/gemini/utils.ts

@@ -1,17 +1,20 @@
 import {
   type FunctionCall,
   type Content as GeminiMessageContent,
+  HarmBlockThreshold,
+  HarmCategory,
+  type SafetySetting,
 } from "@google/generative-ai";
 
 import { type GenerateContentResponse } from "@google-cloud/vertexai";
-import type { BaseTool } from "../../types.js";
 import type {
+  BaseTool,
   ChatMessage,
   MessageContentImageDetail,
   MessageContentTextDetail,
   MessageType,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
 import { extractDataUrlComponents } from "../utils.js";
 import type {
   ChatContext,
@@ -53,10 +56,13 @@ const getImageParts = (
     const { mimeType, base64: data } = extractDataUrlComponents(
       message.image_url.url,
     );
-    if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+    if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType)) {
       throw new Error(
-        `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+        `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join(
+          "\n",
+        )}`,
       );
+    }
     return {
       inlineData: {
         mimeType,
@@ -65,10 +71,13 @@ const getImageParts = (
     };
   }
   const mimeType = getFileURLMimeType(message.image_url.url);
-  if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+  if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType)) {
     throw new Error(
-      `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+      `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join(
+        "\n",
+      )}`,
     );
+  }
   return {
     fileData: { mimeType, fileUri: message.image_url.url },
   };
@@ -124,10 +133,11 @@ export const getChatContext = (
   // 2. Parts that have empty text
   const fnMap = params.messages.reduce(
     (result, message) => {
-      if (message.options && "toolCall" in message.options)
+      if (message.options && "toolCall" in message.options) {
         message.options.toolCall.forEach((call) => {
           result[call.id] = call.name;
         });
+      }
 
       return result;
     },
@@ -224,10 +234,11 @@ export class GeminiHelper {
     if (options && "toolResult" in options) {
       if (!fnMap) throw Error("fnMap must be set");
       const name = fnMap[options.toolResult.id];
-      if (!name)
+      if (!name) {
         throw Error(
           `Could not find the name for fn call with id ${options.toolResult.id}`,
         );
+      }
 
       return [
         {
@@ -299,3 +310,26 @@ export function getFunctionCalls(
     return undefined;
   }
 }
+
+/**
+ * Safety settings to disable external filters
+ * Documentation: https://ai.google.dev/gemini-api/docs/safety-settings
+ */
+export const DEFAULT_SAFETY_SETTINGS: SafetySetting[] = [
+  {
+    category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_HARASSMENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+];

packages/llamaindex/src/llm/gemini/vertex.ts

@@ -1,8 +1,8 @@
 import {
+  type GenerateContentResponse,
   VertexAI,
   GenerativeModel as VertexGenerativeModel,
   GenerativeModelPreview as VertexGenerativeModelPreview,
-  type GenerateContentResponse,
   type ModelParams as VertexModelParams,
   type StreamGenerateContentResult as VertexStreamGenerateContentResult,
 } from "@google-cloud/vertexai";
@@ -14,14 +14,14 @@ import type {
 } from "./types.js";
 
 import type { FunctionCall } from "@google/generative-ai";
-import { getEnv, randomUUID } from "@llamaindex/env";
 import type {
   CompletionResponse,
   ToolCall,
   ToolCallLLMMessageOptions,
-} from "../types.js";
+} from "@llamaindex/core/llms";
+import { getEnv, randomUUID } from "@llamaindex/env";
 import { streamConverter } from "../utils.js";
-import { getFunctionCalls, getText } from "./utils.js";
+import { DEFAULT_SAFETY_SETTINGS, getFunctionCalls, getText } from "./utils.js";
 
 /* To use Google's Vertex AI backend, it doesn't use api key authentication.
  *
@@ -59,8 +59,16 @@ export class GeminiVertexSession implements IGeminiSession {
   getGenerativeModel(
     metadata: VertexModelParams,
   ): VertexGenerativeModelPreview | VertexGenerativeModel {
-    if (this.preview) return this.vertex.preview.getGenerativeModel(metadata);
-    return this.vertex.getGenerativeModel(metadata);
+    if (this.preview) {
+      return this.vertex.preview.getGenerativeModel({
+        safetySettings: DEFAULT_SAFETY_SETTINGS,
+        ...metadata,
+      });
+    }
+    return this.vertex.getGenerativeModel({
+      safetySettings: DEFAULT_SAFETY_SETTINGS,
+      ...metadata,
+    });
   }
 
   getResponseText(response: GenerateContentResponse): string {

packages/llamaindex/src/llm/huggingface.ts

@@ -1,11 +1,4 @@
 import { HfInference } from "@huggingface/inference";
-import type {
-  PreTrainedModel,
-  PreTrainedTokenizer,
-  Tensor,
-} from "@xenova/transformers";
-import { lazyLoadTransformers } from "../internal/deps/transformers.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -14,7 +7,14 @@ import type {
   LLMChatParamsStreaming,
   LLMMetadata,
   ToolCallLLMMessageOptions,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import type {
+  PreTrainedModel,
+  PreTrainedTokenizer,
+  Tensor,
+} from "@xenova/transformers";
+import { lazyLoadTransformers } from "../internal/deps/transformers.js";
+import { BaseLLM } from "./base.js";
 import { streamConverter, wrapLLMEvent } from "./utils.js";
 
 // TODO workaround issue with @huggingface/inference@2.7.0

packages/llamaindex/src/llm/mistral.ts

@@ -1,14 +1,14 @@
-import { getEnv } from "@llamaindex/env";
-import { Settings } from "../Settings.js";
-import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
+import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
+import { BaseLLM } from "./base.js";
 
 export const ALL_AVAILABLE_MISTRAL_MODELS = {
   "mistral-tiny": { contextWindow: 32000 },

packages/llamaindex/src/llm/ollama.ts

@@ -1,3 +1,14 @@
+import type {
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+} from "@llamaindex/core/llms";
 import { BaseEmbedding } from "../embeddings/types.js";
 import {
   Ollama as OllamaBase,
@@ -19,17 +30,6 @@ import {
   type ShowResponse,
   type StatusResponse,
 } from "../internal/deps/ollama.js";
-import type {
-  ChatResponse,
-  ChatResponseChunk,
-  CompletionResponse,
-  LLM,
-  LLMChatParamsNonStreaming,
-  LLMChatParamsStreaming,
-  LLMCompletionParamsNonStreaming,
-  LLMCompletionParamsStreaming,
-  LLMMetadata,
-} from "./types.js";
 import { extractText, streamConverter } from "./utils.js";
 
 const messageAccessor = (part: OllamaChatResponse): ChatResponseChunk => {

packages/llamaindex/src/llm/openai.ts

@@ -7,6 +7,19 @@ import type {
 } from "openai";
 import { AzureOpenAI, OpenAI as OrigOpenAI } from "openai";
 
+import type {
+  BaseTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMMetadata,
+  MessageType,
+  PartialToolCall,
+  ToolCallLLMMessageOptions,
+} from "@llamaindex/core/llms";
 import { Tokenizers } from "@llamaindex/env";
 import type {
   ChatCompletionAssistantMessageParam,
@@ -20,7 +33,6 @@ import type {
 import type { ChatCompletionMessageParam } from "openai/resources/index.js";
 import { wrapEventCaller } from "../internal/context/EventCaller.js";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import type { BaseTool } from "../types.js";
 import type { AzureOpenAIConfig } from "./azure.js";
 import {
   getAzureConfigFromEnv,
@@ -28,18 +40,6 @@ import {
   shouldUseAzure,
 } from "./azure.js";
 import { ToolCallLLM } from "./base.js";
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-  LLM,
-  LLMChatParamsNonStreaming,
-  LLMChatParamsStreaming,
-  LLMMetadata,
-  MessageType,
-  PartialToolCall,
-  ToolCallLLMMessageOptions,
-} from "./types.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 export class OpenAISession {

packages/llamaindex/src/llm/portkey.ts

@@ -1,10 +1,3 @@
-import { getEnv } from "@llamaindex/env";
-import _ from "lodash";
-import type { LLMOptions } from "portkey-ai";
-import { Portkey as OrigPortKey } from "portkey-ai";
-import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -13,7 +6,14 @@ import type {
   LLMChatParamsStreaming,
   LLMMetadata,
   MessageType,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import _ from "lodash";
+import type { LLMOptions } from "portkey-ai";
+import { Portkey as OrigPortKey } from "portkey-ai";
+import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { BaseLLM } from "./base.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 interface PortkeyOptions {

packages/llamaindex/src/llm/replicate_ai.ts

@@ -1,6 +1,3 @@
-import { getEnv } from "@llamaindex/env";
-import Replicate from "../internal/deps/replicate.js";
-import { BaseLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -8,7 +5,10 @@ import type {
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
   MessageType,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import { getEnv } from "@llamaindex/env";
+import Replicate from "../internal/deps/replicate.js";
+import { BaseLLM } from "./base.js";
 import {
   extractText,
   streamCallbacks,

packages/llamaindex/src/llm/types.ts

@@ -1,7 +1,14 @@
+import type {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  MessageContent,
+  ToolCall,
+  ToolOutput,
+} from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
-import type { Tokenizers } from "@llamaindex/env";
 import type { BaseEvent } from "../internal/type.js";
-import type { BaseTool, JSONObject, ToolOutput, UUID } from "../types.js";
+import type { UUID } from "../types.js";
 
 export type RetrievalStartEvent = BaseEvent<{
   query: MessageContent;
@@ -29,197 +36,3 @@ export type LLMStreamEvent = BaseEvent<{
   id: UUID;
   chunk: ChatResponseChunk;
 }>;
-
-/**
- * @internal
- */
-export interface LLMChat<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> {
-  chat(
-    params:
-      | LLMChatParamsStreaming<AdditionalChatOptions>
-      | LLMChatParamsNonStreaming<AdditionalChatOptions>,
-  ): Promise<
-    | ChatResponse<AdditionalMessageOptions>
-    | AsyncIterable<ChatResponseChunk<AdditionalMessageOptions>>
-  >;
-}
-
-/**
- * Unified language model interface
- */
-export interface LLM<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> extends LLMChat<AdditionalChatOptions> {
-  metadata: LLMMetadata;
-  /**
-   * Get a chat response from the LLM
-   */
-  chat(
-    params: LLMChatParamsStreaming<
-      AdditionalChatOptions,
-      AdditionalMessageOptions
-    >,
-  ): Promise<AsyncIterable<ChatResponseChunk>>;
-  chat(
-    params: LLMChatParamsNonStreaming<
-      AdditionalChatOptions,
-      AdditionalMessageOptions
-    >,
-  ): Promise<ChatResponse<AdditionalMessageOptions>>;
-
-  /**
-   * Get a prompt completion from the LLM
-   */
-  complete(
-    params: LLMCompletionParamsStreaming,
-  ): Promise<AsyncIterable<CompletionResponse>>;
-  complete(
-    params: LLMCompletionParamsNonStreaming,
-  ): Promise<CompletionResponse>;
-}
-
-export type MessageType = "user" | "assistant" | "system" | "memory";
-
-export type TextChatMessage<AdditionalMessageOptions extends object = object> =
-  {
-    content: string;
-    role: MessageType;
-    options?: undefined | AdditionalMessageOptions;
-  };
-
-export type ChatMessage<AdditionalMessageOptions extends object = object> = {
-  content: MessageContent;
-  role: MessageType;
-  options?: undefined | AdditionalMessageOptions;
-};
-
-export interface ChatResponse<
-  AdditionalMessageOptions extends object = object,
-> {
-  message: ChatMessage<AdditionalMessageOptions>;
-  /**
-   * Raw response from the LLM
-   *
-   * If LLM response an iterable of chunks, this will be an array of those chunks
-   */
-  raw: object | null;
-}
-
-export type ChatResponseChunk<
-  AdditionalMessageOptions extends object = object,
-> = {
-  raw: object | null;
-  delta: string;
-  options?: undefined | AdditionalMessageOptions;
-};
-
-export interface CompletionResponse {
-  text: string;
-  /**
-   * Raw response from the LLM
-   *
-   * It's possible that this is `null` if the LLM response an iterable of chunks
-   */
-  raw: object | null;
-}
-
-export type LLMMetadata = {
-  model: string;
-  temperature: number;
-  topP: number;
-  maxTokens?: number;
-  contextWindow: number;
-  tokenizer: Tokenizers | undefined;
-};
-
-export interface LLMChatParamsBase<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> {
-  messages: ChatMessage<AdditionalMessageOptions>[];
-  additionalChatOptions?: AdditionalChatOptions;
-  tools?: BaseTool[];
-}
-
-export interface LLMChatParamsStreaming<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
-  stream: true;
-}
-
-export interface LLMChatParamsNonStreaming<
-  AdditionalChatOptions extends object = object,
-  AdditionalMessageOptions extends object = object,
-> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
-  stream?: false;
-}
-
-export interface LLMCompletionParamsBase {
-  prompt: MessageContent;
-}
-
-export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
-  stream: true;
-}
-
-export interface LLMCompletionParamsNonStreaming
-  extends LLMCompletionParamsBase {
-  stream?: false | null;
-}
-
-export type MessageContentTextDetail = {
-  type: "text";
-  text: string;
-};
-
-export type MessageContentImageDetail = {
-  type: "image_url";
-  image_url: { url: string };
-};
-
-export type MessageContentDetail =
-  | MessageContentTextDetail
-  | MessageContentImageDetail;
-
-/**
- * Extended type for the content of a message that allows for multi-modal messages.
- */
-export type MessageContent = string | MessageContentDetail[];
-
-export type ToolCall = {
-  name: string;
-  input: JSONObject;
-  id: string;
-};
-
-// happened in streaming response, the tool call is not ready yet
-export type PartialToolCall = {
-  name: string;
-  id: string;
-  // input is not ready yet, JSON.parse(input) will throw an error
-  input: string;
-};
-
-export type ToolResult = {
-  id: string;
-  result: string;
-  isError: boolean;
-};
-
-export type ToolCallOptions = {
-  toolCall: (ToolCall | PartialToolCall)[];
-};
-
-export type ToolResultOptions = {
-  toolResult: ToolResult;
-};
-
-export type ToolCallLLMMessageOptions =
-  | ToolResultOptions
-  | ToolCallOptions
-  | {};

packages/llamaindex/src/llm/utils.ts

@@ -1,6 +1,3 @@
-import type { ImageType } from "@llamaindex/core/schema";
-import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import type {
   ChatResponse,
   ChatResponseChunk,
@@ -9,7 +6,10 @@ import type {
   MessageContent,
   MessageContentDetail,
   MessageContentTextDetail,
-} from "./types.js";
+} from "@llamaindex/core/llms";
+import type { ImageType } from "@llamaindex/core/schema";
+import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 
 export async function* streamConverter<S, D>(
   stream: AsyncIterable<S>,

packages/llamaindex/src/memory/ChatMemoryBuffer.ts

@@ -1,5 +1,5 @@
+import type { ChatMessage, LLM } from "@llamaindex/core/llms";
 import type { ChatHistory } from "../ChatHistory.js";
-import type { ChatMessage, LLM } from "../llm/index.js";
 import { SimpleChatStore } from "../storage/chatStore/SimpleChatStore.js";
 import type { BaseChatStore } from "../storage/chatStore/types.js";
 import type { BaseMemory } from "./types.js";

packages/llamaindex/src/memory/types.ts

@@ -1,4 +1,4 @@
-import type { ChatMessage } from "../llm/index.js";
+import type { ChatMessage } from "@llamaindex/core/llms";
 
 export interface BaseMemory<AdditionalMessageOptions extends object = object> {
   tokenLimit: number;

packages/llamaindex/src/objects/base.ts

@@ -1,10 +1,9 @@
+import type { BaseTool, MessageContent } from "@llamaindex/core/llms";
 import type { BaseNode, Metadata } from "@llamaindex/core/schema";
 import { TextNode } from "@llamaindex/core/schema";
 import type { BaseRetriever } from "../Retriever.js";
 import type { VectorStoreIndex } from "../indices/vectorStore/index.js";
-import type { MessageContent } from "../llm/index.js";
 import { extractText } from "../llm/utils.js";
-import type { BaseTool } from "../types.js";
 
 // Assuming that necessary interfaces and classes (like OT, TextNode, BaseNode, etc.) are defined elsewhere
 // Import statements (e.g., for TextNode, BaseNode) should be added based on your project's structure

packages/llamaindex/src/postprocessors/rerankers/CohereRerank.ts

@@ -1,8 +1,8 @@
 import { CohereClient } from "cohere-ai";
 
+import type { MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import { MetadataMode } from "@llamaindex/core/schema";
-import type { MessageContent } from "../../llm/types.js";
 import { extractText } from "../../llm/utils.js";
 import type { BaseNodePostprocessor } from "../types.js";
 

packages/llamaindex/src/postprocessors/rerankers/JinaAIReranker.ts

@@ -1,7 +1,7 @@
+import type { MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import { MetadataMode } from "@llamaindex/core/schema";
 import { getEnv } from "@llamaindex/env";
-import type { MessageContent } from "../../llm/types.js";
 import { extractText } from "../../llm/utils.js";
 import type { BaseNodePostprocessor } from "../types.js";
 

packages/llamaindex/src/postprocessors/types.ts

@@ -1,5 +1,5 @@
+import type { MessageContent } from "@llamaindex/core/llms";
 import type { NodeWithScore } from "@llamaindex/core/schema";
-import type { MessageContent } from "../llm/types.js";
 
 export interface BaseNodePostprocessor {
   /**

packages/llamaindex/src/readers/LlamaParseReader.ts

@@ -1,105 +1,100 @@
 import { Document } from "@llamaindex/core/schema";
 import { fs, getEnv } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
+import { filetypeinfo } from "magic-bytes.js";
 import { FileReader, type Language, type ResultType } from "./type.js";
 
-const SupportedFiles: { [key: string]: string } = {
-  ".pdf": "application/pdf",
-  // Documents and Presentations
-  ".602": "application/x-t602",
-  ".abw": "application/x-abiword",
-  ".cgm": "image/cgm",
-  ".cwk": "application/x-cwk",
-  ".doc": "application/msword",
-  ".docx":
-    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-  ".docm": "application/vnd.ms-word.document.macroEnabled.12",
-  ".dot": "application/msword",
-  ".dotm": "application/vnd.ms-word.template.macroEnabled.12",
-  ".dotx":
-    "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
-  ".hwp": "application/x-hwp",
-  ".key": "application/x-iwork-keynote-sffkey",
-  ".lwp": "application/vnd.lotus-wordpro",
-  ".mw": "application/macwriteii",
-  ".mcw": "application/macwriteii",
-  ".pages": "application/x-iwork-pages-sffpages",
-  ".pbd": "application/x-pagemaker",
-  ".ppt": "application/vnd.ms-powerpoint",
-  ".pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
-  ".pptx":
-    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-  ".pot": "application/vnd.ms-powerpoint",
-  ".potm": "application/vnd.ms-powerpoint.template.macroEnabled.12",
-  ".potx":
-    "application/vnd.openxmlformats-officedocument.presentationml.template",
-  ".rtf": "application/rtf",
-  ".sda": "application/vnd.stardivision.draw",
-  ".sdd": "application/vnd.stardivision.impress",
-  ".sdp": "application/sdp",
-  ".sdw": "application/vnd.stardivision.writer",
-  ".sgl": "application/vnd.stardivision.writer",
-  ".sti": "application/vnd.sun.xml.impress.template",
-  ".sxi": "application/vnd.sun.xml.impress",
-  ".sxw": "application/vnd.sun.xml.writer",
-  ".stw": "application/vnd.sun.xml.writer.template",
-  ".sxg": "application/vnd.sun.xml.writer.global",
-  ".txt": "text/plain",
-  ".uof": "application/vnd.uoml+xml",
-  ".uop": "application/vnd.openofficeorg.presentation",
-  ".uot": "application/x-uo",
-  ".vor": "application/vnd.stardivision.writer",
-  ".wpd": "application/wordperfect",
-  ".wps": "application/vnd.ms-works",
-  ".xml": "application/xml",
-  ".zabw": "application/x-abiword",
-  // Images
-  ".epub": "application/epub+zip",
-  ".jpg": "image/jpeg",
-  ".jpeg": "image/jpeg",
-  ".png": "image/png",
-  ".gif": "image/gif",
-  ".bmp": "image/bmp",
-  ".svg": "image/svg+xml",
-  ".tiff": "image/tiff",
-  ".webp": "image/webp",
-  // Web
-  ".htm": "text/html",
-  ".html": "text/html",
-  // Spreadsheets
-  ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-  ".xls": "application/vnd.ms-excel",
-  ".xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12",
-  ".xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
-  ".xlw": "application/vnd.ms-excel",
-  ".csv": "text/csv",
-  ".dif": "application/x-dif",
-  ".sylk": "text/vnd.sylk",
-  ".slk": "text/vnd.sylk",
-  ".prn": "application/x-prn",
-  ".numbers": "application/x-iwork-numbers-sffnumbers",
-  ".et": "application/vnd.ms-excel",
-  ".ods": "application/vnd.oasis.opendocument.spreadsheet",
-  ".fods": "application/vnd.oasis.opendocument.spreadsheet",
-  ".uos1": "application/vnd.uoml+xml",
-  ".uos2": "application/vnd.uoml+xml",
-  ".dbf": "application/vnd.dbf",
-  ".wk1": "application/vnd.lotus-1-2-3",
-  ".wk2": "application/vnd.lotus-1-2-3",
-  ".wk3": "application/vnd.lotus-1-2-3",
-  ".wk4": "application/vnd.lotus-1-2-3",
-  ".wks": "application/vnd.lotus-1-2-3",
-  ".123": "application/vnd.lotus-1-2-3",
-  ".wq1": "application/x-lotus",
-  ".wq2": "application/x-lotus",
-  ".wb1": "application/x-quattro-pro",
-  ".wb2": "application/x-quattro-pro",
-  ".wb3": "application/x-quattro-pro",
-  ".qpw": "application/x-quattro-pro",
-  ".xlr": "application/vnd.ms-works",
-  ".eth": "application/ethos",
-  ".tsv": "text/tab-separated-values",
-};
+const SUPPORT_FILE_EXT: string[] = [
+  ".pdf",
+  // document and presentations
+  ".602",
+  ".abw",
+  ".cgm",
+  ".cwk",
+  ".doc",
+  ".docx",
+  ".docm",
+  ".dot",
+  ".dotm",
+  ".hwp",
+  ".key",
+  ".lwp",
+  ".mw",
+  ".mcw",
+  ".pages",
+  ".pbd",
+  ".ppt",
+  ".pptm",
+  ".pptx",
+  ".pot",
+  ".potm",
+  ".potx",
+  ".rtf",
+  ".sda",
+  ".sdd",
+  ".sdp",
+  ".sdw",
+  ".sgl",
+  ".sti",
+  ".sxi",
+  ".sxw",
+  ".stw",
+  ".sxg",
+  ".txt",
+  ".uof",
+  ".uop",
+  ".uot",
+  ".vor",
+  ".wpd",
+  ".wps",
+  ".xml",
+  ".zabw",
+  ".epub",
+  // images
+  ".jpg",
+  ".jpeg",
+  ".png",
+  ".gif",
+  ".bmp",
+  ".svg",
+  ".tiff",
+  ".webp",
+  // web
+  ".htm",
+  ".html",
+  // spreadsheets
+  ".xlsx",
+  ".xls",
+  ".xlsm",
+  ".xlsb",
+  ".xlw",
+  ".csv",
+  ".dif",
+  ".sylk",
+  ".slk",
+  ".prn",
+  ".numbers",
+  ".et",
+  ".ods",
+  ".fods",
+  ".uos1",
+  ".uos2",
+  ".dbf",
+  ".wk1",
+  ".wk2",
+  ".wk3",
+  ".wk4",
+  ".wks",
+  ".123",
+  ".wq1",
+  ".wq2",
+  ".wb1",
+  ".wb2",
+  ".wb3",
+  ".qpw",
+  ".xlr",
+  ".eth",
+  ".tsv",
+];
 
 /**
  * Represents a reader for parsing files using the LlamaParse API.
@@ -165,7 +160,7 @@ export class LlamaParseReader extends FileReader {
     fileName?: string,
   ): Promise<string> {
     // Load data, set the mime type
-    const { mimeType, extension } = await this.getMimeType(data);
+    const { mime, extension } = await LlamaParseReader.getMimeType(data);
 
     if (this.verbose) {
       const name = fileName ? fileName : extension;
@@ -173,7 +168,7 @@ export class LlamaParseReader extends FileReader {
     }
 
     const body = new FormData();
-    body.set("file", new Blob([data], { type: mimeType }), fileName);
+    body.set("file", new Blob([data], { type: mime }), fileName);
 
     const LlamaParseBodyParams = {
       language: this.language,
@@ -378,19 +373,23 @@ export class LlamaParseReader extends FileReader {
     return images;
   }
 
-  private async getMimeType(
+  static async getMimeType(
     data: Uint8Array,
-  ): Promise<{ mimeType: string; extension: string }> {
-    const mimes = filetypemime(data); // Get an array of possible MIME types
-    const extension = Object.keys(SupportedFiles).find(
-      (ext) => SupportedFiles[ext] === mimes[0],
-    ); // Find the extension for the first MIME type
-    if (!extension) {
-      const supportedExtensions = Object.keys(SupportedFiles).join(", ");
+  ): Promise<{ mime: string; extension: string }> {
+    const typeinfos = filetypeinfo(data);
+    // find the first type info that matches the supported MIME types
+    // It could be happened that docx file is recognized as zip file, so we need to check the mime type
+    const info = typeinfos.find((info) => {
+      if (info.extension && SUPPORT_FILE_EXT.includes(`.${info.extension}`)) {
+        return info;
+      }
+    });
+    if (!info || !info.mime || !info.extension) {
+      const ext = SUPPORT_FILE_EXT.join(", ");
       throw new Error(
-        `File has type "${mimes[0]}" which does not match supported MIME Types. Supported formats include: ${supportedExtensions}`,
+        `File has type which does not match supported MIME Types. Supported formats include: ${ext}`,
       );
     }
-    return { mimeType: mimes[0], extension }; // Return the first MIME type and its corresponding extension
+    return { mime: info.mime, extension: info.extension };
   }
 }

packages/llamaindex/src/selectors/llmSelectors.ts

@@ -1,4 +1,4 @@
-import type { LLM } from "../llm/index.js";
+import type { LLM } from "@llamaindex/core/llms";
 import type { Answer } from "../outputParsers/selectors.js";
 import { SelectionOutputParser } from "../outputParsers/selectors.js";
 import type {

packages/llamaindex/src/storage/chatStore/SimpleChatStore.ts

@@ -1,4 +1,4 @@
-import type { ChatMessage } from "../../llm/index.js";
+import type { ChatMessage } from "@llamaindex/core/llms";
 import type { BaseChatStore } from "./types.js";
 
 /**

packages/llamaindex/src/storage/chatStore/types.ts

@@ -1,4 +1,4 @@
-import type { ChatMessage } from "../../llm/index.js";
+import type { ChatMessage } from "@llamaindex/core/llms";
 
 export interface BaseChatStore<
   AdditionalMessageOptions extends object = object,

packages/llamaindex/src/synthesizers/builders.ts

@@ -1,4 +1,4 @@
-import type { LLM } from "../llm/index.js";
+import type { LLM } from "@llamaindex/core/llms";
 import { streamConverter } from "../llm/utils.js";
 import type {
   RefinePrompt,

packages/llamaindex/src/synthesizers/utils.ts

@@ -1,3 +1,4 @@
+import type { MessageContentDetail } from "@llamaindex/core/llms";
 import {
   ImageNode,
   MetadataMode,
@@ -7,7 +8,6 @@ import {
 } from "@llamaindex/core/schema";
 import type { SimplePrompt } from "../Prompt.js";
 import { imageToDataUrl } from "../embeddings/utils.js";
-import type { MessageContentDetail } from "../llm/types.js";
 
 export async function createMessageContent(
   prompt: SimplePrompt,

packages/llamaindex/src/tools/AzureDynamicSessionTool.node.ts

@@ -2,6 +2,7 @@ import {
   DefaultAzureCredential,
   getBearerTokenProvider,
 } from "@azure/identity";
+import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
 import {
   Readable,
   createWriteStream,
@@ -11,7 +12,6 @@ import {
   path,
   randomUUID,
 } from "@llamaindex/env";
-import type { BaseTool, ToolMetadata } from "../types.js";
 export type InterpreterParameter = {
   code: string;
 };