Release 0.5.14 (#1103)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.eslintrc.cjs

@@ -31,7 +31,12 @@ module.exports = {
     "@typescript-eslint/ban-types": "off",
     "no-array-constructor": "off",
     "@typescript-eslint/no-array-constructor": "off",
-    "@typescript-eslint/no-base-to-string": "off",
+    "@typescript-eslint/no-base-to-string": [
+      "error",
+      {
+        ignoredTypeNames: ["Error", "RegExp", "URL", "URLSearchParams"],
+      },
+    ],
     "@typescript-eslint/no-duplicate-enum-values": "off",
     "@typescript-eslint/no-duplicate-type-constituents": "off",
     "@typescript-eslint/no-explicit-any": "off",

README.md

@@ -164,7 +164,7 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
 
 - [Node](/packages/llamaindex/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
 
-- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
+- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that question. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
 
 - [Indices](/packages/llamaindex/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
 

apps/docs/CHANGELOG.md

@@ -1,5 +1,34 @@
 # docs
 
+## 0.0.55
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.54
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
+## 0.0.52
+
+### Patch Changes
+
+- 376d29a: feat: added tool calling and agent support for llama3.1 504B
+  - llamaindex@0.5.11
+
 ## 0.0.51
 
 ### Patch Changes

apps/docs/docs/modules/agent/index.md

@@ -15,6 +15,7 @@ LlamaIndex.TS comes with a few built-in agents, but you can also create your own
 - Anthropic Agent both via Anthropic and Bedrock (in `@llamaIndex/community`)
 - Gemini Agent
 - ReACT Agent
+- Meta3.1 504B via Bedrock (in `@llamaIndex/community`)
 
 ## Examples
 

apps/docs/docs/modules/data_loaders/llama_parse/index.mdx

@@ -48,6 +48,7 @@ They can be divided into two groups.
 - `gpt4oApiKey?` Deprecated. Use vendorMultimodal params. Optional. Set the GPT-4o API key. Lowers the cost of parsing by using your own API key. Your OpenAI account will be charged. Can also be set in the environment variable `LLAMA_CLOUD_GPT4O_API_KEY`.
 - `boundingBox?` Optional. Specify an area of the document to parse. Expects the bounding box margins as a string in clockwise order, e.g. `boundingBox = "0.1,0,0,0"` to not parse the top 10% of the document.
 - `targetPages?` Optional. Specify which pages to parse by specifying them as a comma-separated list. First page is `0`.
+- `splitByPage` Wether to split the results, creating one document per page. Uses the set `pageSeparator` or `\n---\n` as fallback. Default is true.
 - `useVendorMultimodalModel` set to true to use a multimodal model. Default is `false`.
 - `vendorMultimodalModel?` Optional. Specify which multimodal model to use. Default is GPT4o. See [here](https://docs.cloud.llamaindex.ai/llamaparse/features/multimodal) for a list of available models and cost.
 - `vendorMultimodalApiKey?` Optional. Set the multimodal model API key. Can also be set in the environment variable `LLAMA_CLOUD_VENDOR_MULTIMODAL_API_KEY`.

apps/docs/docs/modules/llms/available_llms/bedrock.md

@@ -31,7 +31,7 @@ META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0";
 META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0";
 META_LLAMA3_1_8B_INSTRUCT = "meta.llama3-1-8b-instruct-v1:0"; // available on us-west-2
 META_LLAMA3_1_70B_INSTRUCT = "meta.llama3-1-70b-instruct-v1:0"; // available on us-west-2
-META_LLAMA3_1_405B_INSTRUCT = "meta.llama3-1-405b-instruct-v1:0"; // preview only, available on us-west-2
+META_LLAMA3_1_405B_INSTRUCT = "meta.llama3-1-405b-instruct-v1:0"; // preview only, available on us-west-2, tool calling supported
 ```
 
 Sonnet, Haiku and Opus are multimodal, image_url only supports base64 data url format, e.g. `data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==`
@@ -67,3 +67,72 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## Agent Example
+
+```ts
+import { BEDROCK_MODELS, Bedrock } from "@llamaindex/community";
+import { FunctionTool, LLMAgent } from "llamaindex";
+
+const sumNumbers = FunctionTool.from(
+  ({ a, b }: { a: number; b: number }) => `${a + b}`,
+  {
+    name: "sumNumbers",
+    description: "Use this function to sum two numbers",
+    parameters: {
+      type: "object",
+      properties: {
+        a: {
+          type: "number",
+          description: "The first number",
+        },
+        b: {
+          type: "number",
+          description: "The second number",
+        },
+      },
+      required: ["a", "b"],
+    },
+  },
+);
+
+const divideNumbers = FunctionTool.from(
+  ({ a, b }: { a: number; b: number }) => `${a / b}`,
+  {
+    name: "divideNumbers",
+    description: "Use this function to divide two numbers",
+    parameters: {
+      type: "object",
+      properties: {
+        a: {
+          type: "number",
+          description: "The dividend a to divide",
+        },
+        b: {
+          type: "number",
+          description: "The divisor b to divide by",
+        },
+      },
+      required: ["a", "b"],
+    },
+  },
+);
+
+const bedrock = new Bedrock({
+  model: BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT,
+  ...
+});
+
+async function main() {
+  const agent = new LLMAgent({
+    llm: bedrock,
+    tools: [sumNumbers, divideNumbers],
+  });
+
+  const response = await agent.chat({
+    message: "How much is 5 + 5? then divide by 2",
+  });
+
+  console.log(response.message);
+}
+```

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.51",
+  "version": "0.0.55",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.39
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.38
+
+### Patch Changes
+
+- llamaindex@0.5.13
+- @llamaindex/autotool@2.0.0
+
+## 0.1.37
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.36
+
+### Patch Changes
+
+- llamaindex@0.5.11
+- @llamaindex/autotool@2.0.0
+
 ## 0.1.35
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.35",
+  "version": "0.1.39",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.5.10",
+    "llamaindex": "^0.5.14",
     "openai": "^4",
     "typescript": "^4"
   },
@@ -70,7 +70,7 @@
     "@swc/types": "^0.1.8",
     "@types/json-schema": "^7.0.15",
     "@types/node": "^20.12.11",
-    "bunchee": "5.3.0-beta.0",
+    "bunchee": "5.3.1",
     "llamaindex": "workspace:*",
     "next": "14.2.5",
     "rollup": "^4.18.0",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/cloud
 
+## 0.2.1
+
+### Patch Changes
+
+- 1f680d7: chore: bump llamacloud api
+
 ## 0.2.0
 
 ### Minor Changes

packages/cloud/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "type": "module",
   "license": "MIT",
   "scripts": {
-    "generate": "pnpm dlx @hey-api/openapi-ts",
+    "generate": "pnpm dlx @hey-api/openapi-ts@0.49.0",
     "build": "pnpm run generate && bunchee"
   },
   "files": [
@@ -35,6 +35,6 @@
   },
   "devDependencies": {
     "@hey-api/openapi-ts": "^0.48.0",
-    "bunchee": "5.3.0-beta.0"
+    "bunchee": "5.3.1"
   }
 }

packages/community/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @llamaindex/community
 
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [04b2f8e]
+  - @llamaindex/core@0.1.7
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [0452af9]
+  - @llamaindex/core@0.1.6
+
+## 0.0.26
+
+### Patch Changes
+
+- 224d507: fix: prevent tool calling getting mixed with conversation
+- 376d29a: feat: added tool calling and agent support for llama3.1 504B
+
 ## 0.0.25
 
 ### Patch Changes

packages/community/README.md

@@ -6,6 +6,7 @@
 
 - Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
 - Bedrock support for the Meta LLama 2, 3 and 3.1 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
+- Meta LLama3.1 405b tool call support
 
 ## LICENSE
 

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.25",
+  "version": "0.0.28",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -19,11 +19,11 @@
     "./llm/bedrock": {
       "import": {
         "types": "./dist/type/llm/bedrock.d.ts",
-        "default": "./dist/llm/bedrock/base.js"
+        "default": "./dist/llm/bedrock/index.js"
       },
       "require": {
         "types": "./dist/type/llm/bedrock.d.ts",
-        "default": "./dist/llm/bedrock/base.cjs"
+        "default": "./dist/llm/bedrock/index.cjs"
       }
     }
   },
@@ -43,10 +43,11 @@
   },
   "devDependencies": {
     "@types/node": "^20.14.2",
-    "bunchee": "5.3.0-beta.0"
+    "bunchee": "5.3.1"
   },
   "dependencies": {
     "@aws-sdk/client-bedrock-runtime": "^3.613.0",
-    "@llamaindex/core": "workspace:*"
+    "@llamaindex/core": "workspace:*",
+    "@llamaindex/env": "workspace:*"
   }
 }

packages/community/src/index.ts

@@ -2,4 +2,4 @@ export {
   BEDROCK_MODELS,
   BEDROCK_MODEL_MAX_TOKENS,
   Bedrock,
-} from "./llm/bedrock/base.js";
+} from "./llm/bedrock/index.js";

packages/community/src/llm/bedrock/anthropic/provider.ts

@@ -16,17 +16,18 @@ import {
   type BedrockChatStreamResponse,
   Provider,
 } from "../provider";
+import { toUtf8 } from "../utils";
 import type {
   AnthropicNoneStreamingResponse,
   AnthropicStreamEvent,
   AnthropicTextContent,
   ToolBlock,
-} from "../types";
+} from "./types";
+
 import {
   mapBaseToolsToAnthropicTools,
   mapChatMessagesToAnthropicMessages,
-  toUtf8,
-} from "../utils";
+} from "./utils";
 
 export class AnthropicProvider extends Provider<AnthropicStreamEvent> {
   getResultFromResponse(
@@ -69,6 +70,7 @@ export class AnthropicProvider extends Provider<AnthropicStreamEvent> {
     let tool: ToolBlock | undefined = undefined;
     // #TODO this should be broken down into a separate consumer
     for await (const response of stream) {
+      const delta = this.getTextFromStreamResponse(response);
       const event = this.getStreamingEventResponse(response);
       if (
         event?.type === "content_block_start" &&
@@ -114,11 +116,10 @@ export class AnthropicProvider extends Provider<AnthropicStreamEvent> {
           };
         }
       }
-      const delta = this.getTextFromStreamResponse(response);
       if (!delta && !options) continue;
 
       yield {
-        delta,
+        delta: options ? "" : delta,
         options,
         raw: response,
       };

packages/community/src/llm/bedrock/anthropic/types.ts

@@ -0,0 +1,142 @@
+import type { ToolMetadata } from "@llamaindex/core/llms";
+import type { InvocationMetrics } from "../types";
+
+type Usage = {
+  input_tokens: number;
+  output_tokens: number;
+};
+
+type Message = {
+  id: string;
+  type: string;
+  role: string;
+  content: string[];
+  model: string;
+  stop_reason: string | null;
+  stop_sequence: string | null;
+  usage: Usage;
+};
+
+export type ToolBlock = {
+  id: string;
+  input: unknown;
+  name: string;
+  type: "tool_use";
+};
+
+export type TextBlock = {
+  type: "text";
+  text: string;
+};
+
+type ContentBlockStart = {
+  type: "content_block_start";
+  index: number;
+  content_block: ToolBlock | TextBlock;
+};
+
+type Delta =
+  | {
+      type: "text_delta";
+      text: string;
+    }
+  | {
+      type: "input_json_delta";
+      partial_json: string;
+    };
+
+type ContentBlockDelta = {
+  type: "content_block_delta";
+  index: number;
+  delta: Delta;
+};
+
+type ContentBlockStop = {
+  type: "content_block_stop";
+  index: number;
+};
+
+type MessageDelta = {
+  type: "message_delta";
+  delta: {
+    stop_reason: string;
+    stop_sequence: string | null;
+  };
+  usage: Usage;
+};
+
+export type MessageStop = {
+  type: "message_stop";
+  "amazon-bedrock-invocationMetrics": InvocationMetrics;
+};
+
+export type AnthropicStreamEvent =
+  | { type: "message_start"; message: Message }
+  | ContentBlockStart
+  | ContentBlockDelta
+  | ContentBlockStop
+  | MessageDelta
+  | MessageStop;
+
+export type AnthropicContent =
+  | AnthropicTextContent
+  | AnthropicImageContent
+  | AnthropicToolContent
+  | AnthropicToolResultContent;
+
+export type AnthropicTextContent = {
+  type: "text";
+  text: string;
+};
+
+export type AnthropicToolContent = {
+  type: "tool_use";
+  id: string;
+  name: string;
+  input: Record<string, unknown>;
+};
+
+export type AnthropicToolResultContent = {
+  type: "tool_result";
+  tool_use_id: string;
+  content: string;
+};
+
+export type AnthropicMediaTypes =
+  | "image/jpeg"
+  | "image/png"
+  | "image/webp"
+  | "image/gif";
+
+export type AnthropicImageSource = {
+  type: "base64";
+  media_type: AnthropicMediaTypes;
+  data: string; // base64 encoded image bytes
+};
+
+export type AnthropicImageContent = {
+  type: "image";
+  source: AnthropicImageSource;
+};
+
+export type AnthropicMessage = {
+  role: "user" | "assistant";
+  content: AnthropicContent[];
+};
+
+export type AnthropicNoneStreamingResponse = {
+  id: string;
+  type: "message";
+  role: "assistant";
+  content: AnthropicContent[];
+  model: string;
+  stop_reason: "end_turn" | "max_tokens" | "stop_sequence";
+  stop_sequence?: string;
+  usage: { input_tokens: number; output_tokens: number };
+};
+
+export type AnthropicTool = {
+  name: string;
+  description: string;
+  input_schema: ToolMetadata["parameters"];
+};

packages/community/src/llm/bedrock/anthropic/utils.ts

@@ -0,0 +1,186 @@
+import type { JSONObject } from "@llamaindex/core/global";
+import type {
+  BaseTool,
+  ChatMessage,
+  MessageContent,
+  MessageContentDetail,
+  ToolCallLLMMessageOptions,
+} from "@llamaindex/core/llms";
+import { mapMessageContentToMessageContentDetails } from "../utils";
+import type {
+  AnthropicContent,
+  AnthropicImageContent,
+  AnthropicMediaTypes,
+  AnthropicMessage,
+  AnthropicTextContent,
+  AnthropicTool,
+} from "./types.js";
+
+const ACCEPTED_IMAGE_MIME_TYPES = [
+  "image/jpeg",
+  "image/png",
+  "image/webp",
+  "image/gif",
+];
+
+export const mergeNeighboringSameRoleMessages = (
+  messages: AnthropicMessage[],
+): AnthropicMessage[] => {
+  return messages.reduce(
+    (result: AnthropicMessage[], current: AnthropicMessage, index: number) => {
+      if (index > 0 && messages[index - 1].role === current.role) {
+        result[result.length - 1].content = [
+          ...result[result.length - 1].content,
+          ...current.content,
+        ];
+      } else {
+        result.push(current);
+      }
+      return result;
+    },
+    [],
+  );
+};
+
+export const mapMessageContentDetailToAnthropicContent = <
+  T extends MessageContentDetail,
+>(
+  detail: T,
+): AnthropicContent => {
+  let content: AnthropicContent;
+
+  if (detail.type === "text") {
+    content = mapTextContent(detail.text);
+  } else if (detail.type === "image_url") {
+    content = mapImageContent(detail.image_url.url);
+  } else {
+    throw new Error("Unsupported content detail type");
+  }
+  return content;
+};
+
+export const mapMessageContentToAnthropicContent = <T extends MessageContent>(
+  content: T,
+): AnthropicContent[] => {
+  return mapMessageContentToMessageContentDetails(content).map(
+    mapMessageContentDetailToAnthropicContent,
+  );
+};
+
+export const mapBaseToolsToAnthropicTools = (
+  tools?: BaseTool[],
+): AnthropicTool[] => {
+  if (!tools) return [];
+  return tools.map((tool: BaseTool) => {
+    const {
+      metadata: { parameters, ...options },
+    } = tool;
+    return {
+      ...options,
+      input_schema: parameters,
+    };
+  });
+};
+
+export const mapChatMessagesToAnthropicMessages = <
+  T extends ChatMessage<ToolCallLLMMessageOptions>,
+>(
+  messages: T[],
+): AnthropicMessage[] => {
+  const mapped = messages
+    .flatMap((msg: T): AnthropicMessage[] => {
+      if (msg.options && "toolCall" in msg.options) {
+        return [
+          {
+            role: "assistant",
+            content: msg.options.toolCall.map((call) => ({
+              type: "tool_use",
+              id: call.id,
+              name: call.name,
+              input: call.input as JSONObject,
+            })),
+          },
+        ];
+      }
+      if (msg.options && "toolResult" in msg.options) {
+        return [
+          {
+            role: "user",
+            content: [
+              {
+                type: "tool_result",
+                tool_use_id: msg.options.toolResult.id,
+                content: msg.options.toolResult.result,
+              },
+            ],
+          },
+        ];
+      }
+      return mapMessageContentToMessageContentDetails(msg.content).map(
+        (detail: MessageContentDetail): AnthropicMessage => {
+          const content = mapMessageContentDetailToAnthropicContent(detail);
+
+          return {
+            role: msg.role === "assistant" ? "assistant" : "user",
+            content: [content],
+          };
+        },
+      );
+    })
+    .filter((message: AnthropicMessage) => {
+      const content = message.content[0];
+      if (content.type === "text" && !content.text) return false;
+      if (content.type === "image" && !content.source.data) return false;
+      if (content.type === "image" && message.role === "assistant")
+        return false;
+      return true;
+    });
+
+  return mergeNeighboringSameRoleMessages(mapped);
+};
+
+export const mapTextContent = (text: string): AnthropicTextContent => {
+  return { type: "text", text };
+};
+
+export const extractDataUrlComponents = (
+  dataUrl: string,
+): {
+  mimeType: string;
+  base64: string;
+} => {
+  const parts = dataUrl.split(";base64,");
+
+  if (parts.length !== 2 || !parts[0].startsWith("data:")) {
+    throw new Error("Invalid data URL");
+  }
+
+  const mimeType = parts[0].slice(5);
+  const base64 = parts[1];
+
+  return {
+    mimeType,
+    base64,
+  };
+};
+
+export const mapImageContent = (imageUrl: string): AnthropicImageContent => {
+  if (!imageUrl.startsWith("data:"))
+    throw new Error(
+      "For Anthropic please only use base64 data url, e.g.: data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==",
+    );
+  const { mimeType, base64: data } = extractDataUrlComponents(imageUrl);
+  if (!ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+    throw new Error(
+      `Anthropic only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+    );
+
+  return {
+    type: "image",
+    source: {
+      type: "base64",
+      media_type: mimeType as AnthropicMediaTypes,
+      data,
+    },
+  };
+};

packages/community/src/llm/bedrock/index.ts

@@ -22,8 +22,16 @@ import {
   type BedrockChatStreamResponse,
   Provider,
 } from "./provider";
-import { PROVIDERS } from "./providers";
-import { mapMessageContentToMessageContentDetails } from "./utils.js";
+import { mapMessageContentToMessageContentDetails } from "./utils";
+
+import { AnthropicProvider } from "./anthropic/provider";
+import { MetaProvider } from "./meta/provider";
+
+// Other providers should go here
+export const PROVIDERS: { [key: string]: Provider } = {
+  anthropic: new AnthropicProvider(),
+  meta: new MetaProvider(),
+};
 
 export type BedrockChatParamsStreaming = LLMChatParamsStreaming<
   BedrockAdditionalChatOptions,
@@ -140,6 +148,7 @@ export const TOOL_CALL_MODELS = [
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
+  BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT,
 ];
 
 const getProvider = (model: string): Provider => {

packages/community/src/llm/bedrock/meta/constants.ts

@@ -0,0 +1,3 @@
+export const TOKENS = {
+  TOOL_CALL: "<|python_tag|>",
+};

packages/community/src/llm/bedrock/meta/provider.ts

@@ -0,0 +1,136 @@
+import type {
+  InvokeModelCommandInput,
+  InvokeModelWithResponseStreamCommandInput,
+  ResponseStream,
+} from "@aws-sdk/client-bedrock-runtime";
+import type {
+  BaseTool,
+  ChatMessage,
+  LLMMetadata,
+  ToolCall,
+  ToolCallLLMMessageOptions,
+} from "@llamaindex/core/llms";
+import { toUtf8 } from "../utils";
+import type { MetaNoneStreamingResponse, MetaStreamEvent } from "./types";
+
+import { randomUUID } from "@llamaindex/env";
+import { Provider, type BedrockChatStreamResponse } from "../provider";
+import { TOKENS } from "./constants";
+import {
+  mapChatMessagesToMetaLlama2Messages,
+  mapChatMessagesToMetaLlama3Messages,
+} from "./utils";
+
+export class MetaProvider extends Provider<MetaStreamEvent> {
+  getResultFromResponse(
+    response: Record<string, any>,
+  ): MetaNoneStreamingResponse {
+    return JSON.parse(toUtf8(response.body));
+  }
+
+  getToolsFromResponse<ToolContent>(
+    response: Record<string, any>,
+  ): ToolContent[] {
+    const result = this.getResultFromResponse(response);
+    if (!result.generation.trim().startsWith(TOKENS.TOOL_CALL)) return [];
+    const tool = JSON.parse(
+      result.generation.trim().split(TOKENS.TOOL_CALL)[1],
+    );
+    return [
+      {
+        id: randomUUID(),
+        name: tool.name,
+        input: tool.parameters,
+      } as ToolContent,
+    ];
+  }
+
+  getTextFromResponse(response: Record<string, any>): string {
+    const result = this.getResultFromResponse(response);
+    if (result.generation.trim().startsWith(TOKENS.TOOL_CALL)) return "";
+    return result.generation;
+  }
+
+  getTextFromStreamResponse(response: Record<string, any>): string {
+    const event = this.getStreamingEventResponse(response);
+    if (event?.generation) {
+      return event.generation;
+    }
+    return "";
+  }
+
+  async *reduceStream(
+    stream: AsyncIterable<ResponseStream>,
+  ): BedrockChatStreamResponse {
+    const collecting: string[] = [];
+    let toolId: string | undefined = undefined;
+    for await (const response of stream) {
+      const event = this.getStreamingEventResponse(response);
+      const delta = this.getTextFromStreamResponse(response);
+      // odd quirk of llama3.1, start token is \n\n
+      if (
+        !event?.generation.trim() &&
+        event?.generation_token_count === 1 &&
+        event.prompt_token_count !== null
+      )
+        continue;
+
+      if (delta === TOKENS.TOOL_CALL) {
+        toolId = randomUUID();
+        continue;
+      }
+
+      let options: undefined | ToolCallLLMMessageOptions = undefined;
+      if (toolId && event?.stop_reason === "stop") {
+        const tool = JSON.parse(collecting.join(""));
+        options = {
+          toolCall: [
+            {
+              id: toolId,
+              name: tool.name,
+              input: tool.parameters,
+            } as ToolCall,
+          ],
+        };
+      } else if (toolId && !event?.stop_reason) {
+        collecting.push(delta);
+        continue;
+      }
+
+      if (!delta && !options) continue;
+
+      yield {
+        delta: options ? "" : delta,
+        options,
+        raw: response,
+      };
+    }
+  }
+
+  getRequestBody<T extends ChatMessage>(
+    metadata: LLMMetadata,
+    messages: T[],
+    tools?: BaseTool[],
+  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
+    let prompt: string = "";
+    if (metadata.model.startsWith("meta.llama3")) {
+      prompt = mapChatMessagesToMetaLlama3Messages(messages, tools);
+    } else if (metadata.model.startsWith("meta.llama2")) {
+      prompt = mapChatMessagesToMetaLlama2Messages(messages);
+    } else {
+      throw new Error(`Meta model ${metadata.model} is not supported`);
+    }
+
+    return {
+      modelId: metadata.model,
+      contentType: "application/json",
+      accept: "application/json",
+      body: JSON.stringify({
+        prompt,
+        max_gen_len: metadata.maxTokens,
+        temperature: metadata.temperature,
+        top_p: metadata.topP,
+      }),
+    };
+  }
+}

packages/community/src/llm/bedrock/meta/types.ts

@@ -0,0 +1,21 @@
+import type { InvocationMetrics } from "../types";
+
+export type MetaTextContent = string;
+
+export type MetaMessage = {
+  role: "user" | "assistant" | "system" | "ipython";
+  content: MetaTextContent;
+};
+
+type MetaResponse = {
+  generation: string;
+  prompt_token_count: number;
+  generation_token_count: number;
+  stop_reason: "stop" | "length";
+};
+
+export type MetaStreamEvent = MetaResponse & {
+  "amazon-bedrock-invocationMetrics": InvocationMetrics;
+};
+
+export type MetaNoneStreamingResponse = MetaResponse;

packages/community/src/llm/bedrock/meta/utils.ts

@@ -0,0 +1,198 @@
+import type {
+  BaseTool,
+  ChatMessage,
+  MessageContentTextDetail,
+  ToolCallLLMMessageOptions,
+} from "@llamaindex/core/llms";
+import type { MetaMessage } from "./types";
+
+const getToolCallInstructionString = (tool: BaseTool): string => {
+  return `Use the function '${tool.metadata.name}' to '${tool.metadata.description}'`;
+};
+
+const getToolCallParametersString = (tool: BaseTool): string => {
+  return JSON.stringify({
+    name: tool.metadata.name,
+    description: tool.metadata.description,
+    parameters: tool.metadata.parameters
+      ? Object.entries(tool.metadata.parameters.properties).map(
+          ([name, definition]) => ({ [name]: definition }),
+        )
+      : {},
+  });
+};
+
+// ported from https://github.com/meta-llama/llama-agentic-system/blob/main/llama_agentic_system/system_prompt.py
+// NOTE: using json instead of the above xml style tool calling works more reliability
+export const getToolsPrompt = (tools?: BaseTool[]) => {
+  if (!tools?.length) return "";
+
+  const customToolParams = tools.map((tool) => {
+    return [
+      getToolCallInstructionString(tool),
+      getToolCallParametersString(tool),
+    ].join("\n\n");
+  });
+
+  return `
+Environment: node
+
+# Tool Instructions
+- Never use ipython, always use javascript in node
+
+Cutting Knowledge Date: December 2023
+Today Date: ${new Date().toLocaleString("en-US", { year: "numeric", month: "long" })} 
+
+You have access to the following functions:
+
+${customToolParams}
+
+Think very carefully before calling functions.
+
+If a you choose to call a function ONLY reply in the following json format:
+{
+  "name": function_name,
+  "parameters": parameters,
+}
+where
+
+{
+  "name": function_name,
+  "parameters": parameters, => a JSON dict with the function argument name as key and function argument value as value.
+}
+
+Here is an example,
+
+{
+  "name": "example_function_name",
+  "parameters": {"example_name": "example_value"}
+}
+
+Reminder:
+- Function calls MUST follow the specified format
+- Required parameters MUST be specified
+- Only call one function at a time
+- Put the entire function call reply on one line
+- Always add your sources when using search results to answer the user query
+  `;
+};
+
+export const mapChatRoleToMetaRole = (
+  role: ChatMessage["role"],
+): MetaMessage["role"] => {
+  if (role === "assistant") return "assistant";
+  if (role === "user") return "user";
+  return "system";
+};
+
+export const mapChatMessagesToMetaMessages = <
+  T extends ChatMessage<ToolCallLLMMessageOptions>,
+>(
+  messages: T[],
+): MetaMessage[] => {
+  return messages.flatMap((msg) => {
+    if (msg.options && "toolCall" in msg.options) {
+      return msg.options.toolCall.map((call) => ({
+        role: "assistant",
+        content: JSON.stringify({
+          id: call.id,
+          name: call.name,
+          parameters: call.input,
+        }),
+      }));
+    }
+
+    if (msg.options && "toolResult" in msg.options) {
+      return {
+        role: "ipython",
+        content: JSON.stringify(msg.options.toolResult),
+      };
+    }
+
+    let content: string = "";
+    if (typeof msg.content === "string") {
+      content = msg.content;
+    } else if (msg.content.length) {
+      content = (msg.content[0] as MessageContentTextDetail).text;
+    }
+    return {
+      role: mapChatRoleToMetaRole(msg.role),
+      content,
+    };
+  });
+};
+
+/**
+ * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
+ */
+export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>(
+  messages: T[],
+  tools?: BaseTool[],
+): string => {
+  const parts: string[] = [];
+  if (tools?.length) {
+    parts.push(
+      "<|begin_of_text|>",
+      "<|start_header_id|>system<|end_header_id|>",
+      getToolsPrompt(tools),
+      "<|eot_id|>",
+    );
+  }
+
+  const mapped = mapChatMessagesToMetaMessages(messages).map((message) => {
+    return [
+      "<|start_header_id|>",
+      message.role,
+      "<|end_header_id|>",
+      message.content,
+      "<|eot_id|>",
+    ].join("\n");
+  });
+
+  parts.push(
+    "<|begin_of_text|>",
+    ...mapped,
+    "<|start_header_id|>assistant<|end_header_id|>",
+  );
+  return parts.join("\n");
+};
+
+/**
+ * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-2
+ */
+export const mapChatMessagesToMetaLlama2Messages = <T extends ChatMessage>(
+  messages: T[],
+): string => {
+  const mapped = mapChatMessagesToMetaMessages(messages);
+  let output = "<s>";
+  let insideInst = false;
+  let needsStartAgain = false;
+  for (const message of mapped) {
+    if (needsStartAgain) {
+      output += "<s>";
+      needsStartAgain = false;
+    }
+    const text = message.content;
+    if (message.role === "system") {
+      if (!insideInst) {
+        output += "[INST] ";
+        insideInst = true;
+      }
+      output += `<<SYS>>\n${text}\n<</SYS>>\n`;
+    } else if (message.role === "user") {
+      output += text;
+      if (insideInst) {
+        output += " [/INST]";
+        insideInst = false;
+      }
+    } else if (message.role === "assistant") {
+      if (insideInst) {
+        output += " [/INST]";
+        insideInst = false;
+      }
+      output += ` ${text} </s>\n`;
+      needsStartAgain = true;
+    }
+  }
+  return output;
+};

packages/community/src/llm/bedrock/provider.ts

@@ -23,6 +23,7 @@ export type BedrockChatStreamResponse = AsyncIterable<
 export abstract class Provider<ProviderStreamEvent extends {} = {}> {
   abstract getTextFromResponse(response: Record<string, any>): string;
 
+  // Return tool calls from none streaming calls
   abstract getToolsFromResponse<T extends {} = {}>(
     response: Record<string, any>,
   ): T[];

packages/community/src/llm/bedrock/providers/index.ts

@@ -1,9 +0,0 @@
-import { Provider } from "../provider";
-import { AnthropicProvider } from "./anthropic";
-import { MetaProvider } from "./meta";
-
-// Other providers should go here
-export const PROVIDERS: { [key: string]: Provider } = {
-  anthropic: new AnthropicProvider(),
-  meta: new MetaProvider(),
-};

packages/community/src/llm/bedrock/providers/meta.ts

@@ -1,69 +0,0 @@
-import type {
-  InvokeModelCommandInput,
-  InvokeModelWithResponseStreamCommandInput,
-} from "@aws-sdk/client-bedrock-runtime";
-import type { ChatMessage, LLMMetadata } from "@llamaindex/core/llms";
-import type { MetaNoneStreamingResponse, MetaStreamEvent } from "../types";
-import {
-  mapChatMessagesToMetaLlama2Messages,
-  mapChatMessagesToMetaLlama3Messages,
-  toUtf8,
-} from "../utils";
-
-import { Provider } from "../provider";
-
-export class MetaProvider extends Provider<MetaStreamEvent> {
-  constructor() {
-    super();
-  }
-
-  getResultFromResponse(
-    response: Record<string, any>,
-  ): MetaNoneStreamingResponse {
-    return JSON.parse(toUtf8(response.body));
-  }
-
-  getToolsFromResponse(_response: Record<string, any>): never {
-    throw new Error("Not supported by this provider.");
-  }
-
-  getTextFromResponse(response: Record<string, any>): string {
-    const result = this.getResultFromResponse(response);
-    return result.generation;
-  }
-
-  getTextFromStreamResponse(response: Record<string, any>): string {
-    const event = this.getStreamingEventResponse(response);
-    if (event?.generation) {
-      return event.generation;
-    }
-    return "";
-  }
-
-  getRequestBody<T extends ChatMessage>(
-    metadata: LLMMetadata,
-    messages: T[],
-  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
-    let promptFunction: (messages: ChatMessage[]) => string;
-
-    if (metadata.model.startsWith("meta.llama3")) {
-      promptFunction = mapChatMessagesToMetaLlama3Messages;
-    } else if (metadata.model.startsWith("meta.llama2")) {
-      promptFunction = mapChatMessagesToMetaLlama2Messages;
-    } else {
-      throw new Error(`Meta model ${metadata.model} is not supported`);
-    }
-
-    return {
-      modelId: metadata.model,
-      contentType: "application/json",
-      accept: "application/json",
-      body: JSON.stringify({
-        prompt: promptFunction(messages),
-        max_gen_len: metadata.maxTokens,
-        temperature: metadata.temperature,
-        top_p: metadata.topP,
-      }),
-    };
-  }
-}

packages/community/src/llm/bedrock/types.ts

@@ -1,165 +1,11 @@
-type Usage = {
-  input_tokens: number;
-  output_tokens: number;
-};
-
-type Message = {
-  id: string;
-  type: string;
-  role: string;
-  content: string[];
-  model: string;
-  stop_reason: string | null;
-  stop_sequence: string | null;
-  usage: Usage;
-};
-
-export type ToolBlock = {
-  id: string;
-  input: unknown;
-  name: string;
-  type: "tool_use";
-};
-
-export type TextBlock = {
-  type: "text";
-  text: string;
-};
-
-type ContentBlockStart = {
-  type: "content_block_start";
-  index: number;
-  content_block: ToolBlock | TextBlock;
-};
-
-type Delta =
-  | {
-      type: "text_delta";
-      text: string;
-    }
-  | {
-      type: "input_json_delta";
-      partial_json: string;
-    };
-
-type ContentBlockDelta = {
-  type: "content_block_delta";
-  index: number;
-  delta: Delta;
-};
-
-type ContentBlockStop = {
-  type: "content_block_stop";
-  index: number;
-};
-
-type MessageDelta = {
-  type: "message_delta";
-  delta: {
-    stop_reason: string;
-    stop_sequence: string | null;
-  };
-  usage: Usage;
-};
-
-type InvocationMetrics = {
+export type InvocationMetrics = {
   inputTokenCount: number;
   outputTokenCount: number;
   invocationLatency: number;
   firstByteLatency: number;
 };
 
-type MessageStop = {
-  type: "message_stop";
-  "amazon-bedrock-invocationMetrics": InvocationMetrics;
-};
-
 export type ToolChoice =
   | { type: "any" }
   | { type: "auto" }
   | { type: "tool"; name: string };
-
-export type AnthropicStreamEvent =
-  | { type: "message_start"; message: Message }
-  | ContentBlockStart
-  | ContentBlockDelta
-  | ContentBlockStop
-  | MessageDelta
-  | MessageStop;
-
-export type AnthropicContent =
-  | AnthropicTextContent
-  | AnthropicImageContent
-  | AnthropicToolContent
-  | AnthropicToolResultContent;
-
-export type MetaTextContent = string;
-
-export type AnthropicTextContent = {
-  type: "text";
-  text: string;
-};
-
-export type AnthropicToolContent = {
-  type: "tool_use";
-  id: string;
-  name: string;
-  input: Record<string, unknown>;
-};
-
-export type AnthropicToolResultContent = {
-  type: "tool_result";
-  tool_use_id: string;
-  content: string;
-};
-
-export type AnthropicMediaTypes =
-  | "image/jpeg"
-  | "image/png"
-  | "image/webp"
-  | "image/gif";
-
-export type AnthropicImageSource = {
-  type: "base64";
-  media_type: AnthropicMediaTypes;
-  data: string; // base64 encoded image bytes
-};
-
-export type AnthropicImageContent = {
-  type: "image";
-  source: AnthropicImageSource;
-};
-
-export type AnthropicMessage = {
-  role: "user" | "assistant";
-  content: AnthropicContent[];
-};
-
-export type MetaMessage = {
-  role: "user" | "assistant" | "system";
-  content: MetaTextContent;
-};
-
-export type AnthropicNoneStreamingResponse = {
-  id: string;
-  type: "message";
-  role: "assistant";
-  content: AnthropicContent[];
-  model: string;
-  stop_reason: "end_turn" | "max_tokens" | "stop_sequence";
-  stop_sequence?: string;
-  usage: { input_tokens: number; output_tokens: number };
-};
-
-type MetaResponse = {
-  generation: string;
-  prompt_token_count: number;
-  generation_token_count: number;
-  stop_reason: "stop" | "length";
-};
-
-export type MetaStreamEvent = MetaResponse & {
-  "amazon-bedrock-invocationMetrics": InvocationMetrics;
-};
-
-export type MetaNoneStreamingResponse = MetaResponse;

packages/community/src/llm/bedrock/utils.ts

@@ -1,28 +1,7 @@
-import type { JSONObject } from "@llamaindex/core/global";
 import type {
-  BaseTool,
-  ChatMessage,
   MessageContent,
   MessageContentDetail,
-  MessageContentTextDetail,
-  ToolCallLLMMessageOptions,
-  ToolMetadata,
 } from "@llamaindex/core/llms";
-import type {
-  AnthropicContent,
-  AnthropicImageContent,
-  AnthropicMediaTypes,
-  AnthropicMessage,
-  AnthropicTextContent,
-  MetaMessage,
-} from "./types.js";
-
-const ACCEPTED_IMAGE_MIME_TYPES = [
-  "image/jpeg",
-  "image/png",
-  "image/webp",
-  "image/gif",
-];
 
 export const mapMessageContentToMessageContentDetails = (
   content: MessageContent,
@@ -30,250 +9,5 @@ export const mapMessageContentToMessageContentDetails = (
   return Array.isArray(content) ? content : [{ type: "text", text: content }];
 };
 
-export const mergeNeighboringSameRoleMessages = (
-  messages: AnthropicMessage[],
-): AnthropicMessage[] => {
-  return messages.reduce(
-    (result: AnthropicMessage[], current: AnthropicMessage, index: number) => {
-      if (index > 0 && messages[index - 1].role === current.role) {
-        result[result.length - 1].content = [
-          ...result[result.length - 1].content,
-          ...current.content,
-        ];
-      } else {
-        result.push(current);
-      }
-      return result;
-    },
-    [],
-  );
-};
-
-export const mapMessageContentDetailToAnthropicContent = <
-  T extends MessageContentDetail,
->(
-  detail: T,
-): AnthropicContent => {
-  let content: AnthropicContent;
-
-  if (detail.type === "text") {
-    content = mapTextContent(detail.text);
-  } else if (detail.type === "image_url") {
-    content = mapImageContent(detail.image_url.url);
-  } else {
-    throw new Error("Unsupported content detail type");
-  }
-  return content;
-};
-
-export const mapMessageContentToAnthropicContent = <T extends MessageContent>(
-  content: T,
-): AnthropicContent[] => {
-  return mapMessageContentToMessageContentDetails(content).map(
-    mapMessageContentDetailToAnthropicContent,
-  );
-};
-
-type AnthropicTool = {
-  name: string;
-  description: string;
-  input_schema: ToolMetadata["parameters"];
-};
-
-export const mapBaseToolsToAnthropicTools = (
-  tools?: BaseTool[],
-): AnthropicTool[] => {
-  if (!tools) return [];
-  return tools.map((tool: BaseTool) => {
-    const {
-      metadata: { parameters, ...options },
-    } = tool;
-    return {
-      ...options,
-      input_schema: parameters,
-    };
-  });
-};
-
-export const mapChatMessagesToAnthropicMessages = <
-  T extends ChatMessage<ToolCallLLMMessageOptions>,
->(
-  messages: T[],
-): AnthropicMessage[] => {
-  const mapped = messages
-    .flatMap((msg: T): AnthropicMessage[] => {
-      if (msg.options && "toolCall" in msg.options) {
-        return [
-          {
-            role: "assistant",
-            content: msg.options.toolCall.map((call) => ({
-              type: "tool_use",
-              id: call.id,
-              name: call.name,
-              input: call.input as JSONObject,
-            })),
-          },
-        ];
-      }
-      if (msg.options && "toolResult" in msg.options) {
-        return [
-          {
-            role: "user",
-            content: [
-              {
-                type: "tool_result",
-                tool_use_id: msg.options.toolResult.id,
-                content: msg.options.toolResult.result,
-              },
-            ],
-          },
-        ];
-      }
-      return mapMessageContentToMessageContentDetails(msg.content).map(
-        (detail: MessageContentDetail): AnthropicMessage => {
-          const content = mapMessageContentDetailToAnthropicContent(detail);
-
-          return {
-            role: msg.role === "assistant" ? "assistant" : "user",
-            content: [content],
-          };
-        },
-      );
-    })
-    .filter((message: AnthropicMessage) => {
-      const content = message.content[0];
-      if (content.type === "text" && !content.text) return false;
-      if (content.type === "image" && !content.source.data) return false;
-      return true;
-    });
-
-  return mergeNeighboringSameRoleMessages(mapped);
-};
-
-export const mapChatMessagesToMetaMessages = <T extends ChatMessage>(
-  messages: T[],
-): MetaMessage[] => {
-  return messages.map((msg) => {
-    let content: string = "";
-    if (typeof msg.content === "string") {
-      content = msg.content;
-    } else if (msg.content.length) {
-      content = (msg.content[0] as MessageContentTextDetail).text;
-    }
-    return {
-      role:
-        msg.role === "assistant"
-          ? "assistant"
-          : msg.role === "user"
-            ? "user"
-            : "system",
-      content,
-    };
-  });
-};
-
-/**
- * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
- */
-export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>(
-  messages: T[],
-): string => {
-  const mapped = mapChatMessagesToMetaMessages(messages).map((message) => {
-    const text = message.content;
-    return `<|start_header_id|>${message.role}<|end_header_id|>\n${text}\n<|eot_id|>\n`;
-  });
-  return (
-    "<|begin_of_text|>" +
-    mapped.join("\n") +
-    "\n<|start_header_id|>assistant<|end_header_id|>\n"
-  );
-};
-
-/**
- * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-2
- */
-export const mapChatMessagesToMetaLlama2Messages = <T extends ChatMessage>(
-  messages: T[],
-): string => {
-  const mapped = mapChatMessagesToMetaMessages(messages);
-  let output = "<s>";
-  let insideInst = false;
-  let needsStartAgain = false;
-  for (const message of mapped) {
-    if (needsStartAgain) {
-      output += "<s>";
-      needsStartAgain = false;
-    }
-    const text = message.content;
-    if (message.role === "system") {
-      if (!insideInst) {
-        output += "[INST] ";
-        insideInst = true;
-      }
-      output += `<<SYS>>\n${text}\n<</SYS>>\n`;
-    } else if (message.role === "user") {
-      output += text;
-      if (insideInst) {
-        output += " [/INST]";
-        insideInst = false;
-      }
-    } else if (message.role === "assistant") {
-      if (insideInst) {
-        output += " [/INST]";
-        insideInst = false;
-      }
-      output += ` ${text} </s>\n`;
-      needsStartAgain = true;
-    }
-  }
-  return output;
-};
-
-export const mapTextContent = (text: string): AnthropicTextContent => {
-  return { type: "text", text };
-};
-
-export const extractDataUrlComponents = (
-  dataUrl: string,
-): {
-  mimeType: string;
-  base64: string;
-} => {
-  const parts = dataUrl.split(";base64,");
-
-  if (parts.length !== 2 || !parts[0].startsWith("data:")) {
-    throw new Error("Invalid data URL");
-  }
-
-  const mimeType = parts[0].slice(5);
-  const base64 = parts[1];
-
-  return {
-    mimeType,
-    base64,
-  };
-};
-
-export const mapImageContent = (imageUrl: string): AnthropicImageContent => {
-  if (!imageUrl.startsWith("data:"))
-    throw new Error(
-      "For Anthropic please only use base64 data url, e.g.: data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==",
-    );
-  const { mimeType, base64: data } = extractDataUrlComponents(imageUrl);
-  if (!ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
-    throw new Error(
-      `Anthropic only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
-    );
-
-  return {
-    type: "image",
-    source: {
-      type: "base64",
-      media_type: mimeType as AnthropicMediaTypes,
-      data,
-    },
-  };
-};
-
 export const toUtf8 = (input: Uint8Array): string =>
   new TextDecoder("utf-8").decode(input);

packages/core/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @llamaindex/core
 
+## 0.1.7
+
+### Patch Changes
+
+- 04b2f8e: Fix issue with metadata included after sentence splitter
+
+## 0.1.6
+
+### Patch Changes
+
+- 0452af9: fix: handling errors in splitBySentenceTokenizer
+
 ## 0.1.5
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./node-parser": {
@@ -131,7 +131,7 @@
   },
   "devDependencies": {
     "ajv": "^8.16.0",
-    "bunchee": "5.3.0-beta.0",
+    "bunchee": "5.3.1",
     "natural": "^7.1.0"
   },
   "dependencies": {

packages/core/src/node-parser/base.ts

@@ -140,7 +140,7 @@ export abstract class MetadataAwareTextSplitter extends TextSplitter {
     return nodes.reduce<TextNode[]>((allNodes, node) => {
       const metadataStr = this.getMetadataString(node);
       const splits = this.splitTextMetadataAware(
-        node.getContent(MetadataMode.ALL),
+        node.getContent(MetadataMode.NONE),
         metadataStr,
       );
       return allNodes.concat(buildNodeFromSplits(splits, node));

packages/core/src/node-parser/utils.ts

@@ -39,7 +39,11 @@ export const splitBySentenceTokenizer = (): TextSplitterFn => {
   }
   const tokenizer = sentenceTokenizer;
   return (text: string) => {
-    return tokenizer.tokenize(text);
+    try {
+      return tokenizer.tokenize(text);
+    } catch {
+      return [text];
+    }
   };
 };
 

packages/core/tests/node-parser/text-splitter.test.ts

@@ -1,7 +1,10 @@
-import { SentenceSplitter } from "@llamaindex/core/node-parser";
+import {
+  SentenceSplitter,
+  splitBySentenceTokenizer,
+} from "@llamaindex/core/node-parser";
 import { describe, expect, test } from "vitest";
 
-describe("SentenceSplitter", () => {
+describe("sentence splitter", () => {
   test("initializes", () => {
     const sentenceSplitter = new SentenceSplitter();
     expect(sentenceSplitter).toBeDefined();
@@ -105,4 +108,11 @@ describe("SentenceSplitter", () => {
       "因为他照了人类，连我都在内。",
     ]);
   });
+
+  test("issue 1087 - edge case when input with brackets", () => {
+    const text =
+      "A card must be of uniform thickness and made of unfolded and uncreased paper or cardstock of approximately the quality and weight of a stamped card (i.e., a card available from USPS).";
+    const split = splitBySentenceTokenizer();
+    expect(split(text)).toEqual([text]);
+  });
 });

packages/experimental/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/experimental
 
+## 0.0.64
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.63
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.62
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
+## 0.0.61
+
+### Patch Changes
+
+- llamaindex@0.5.11
+
 ## 0.0.60
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.60",
+  "version": "0.0.64",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/experimental/src/engines/query/JSONQueryEngine.ts

@@ -162,18 +162,18 @@ export class JSONQueryEngine implements QueryEngine {
 
     const schema = this.getSchemaContext();
 
-    const jsonPathResponseStr = await this.serviceContext.llm.complete({
+    const { text: jsonPathResponse } = await this.serviceContext.llm.complete({
       prompt: this.jsonPathPrompt({ query, schema }),
     });
 
     if (this.verbose) {
       console.log(
-        `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponseStr}\n\`\`\`\n`,
+        `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponse}\n\`\`\`\n`,
       );
     }
 
     const jsonPathOutput = await this.outputProcessor({
-      llmOutput: jsonPathResponseStr.text,
+      llmOutput: jsonPathResponse,
       jsonValue: this.jsonValue,
     });
 
@@ -188,7 +188,7 @@ export class JSONQueryEngine implements QueryEngine {
         prompt: this.responseSynthesisPrompt({
           query,
           jsonSchema: schema,
-          jsonPath: jsonPathResponseStr.text,
+          jsonPath: jsonPathResponse,
           jsonPathValue: JSON.stringify(jsonPathOutput),
         }),
       });
@@ -199,7 +199,7 @@ export class JSONQueryEngine implements QueryEngine {
     }
 
     const responseMetadata = {
-      jsonPathResponseStr,
+      jsonPathResponse,
     };
 
     const response = EngineResponse.fromResponse(responseStr, false);

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,35 @@
 # llamaindex
 
+## 0.5.14
+
+### Patch Changes
+
+- c825a2f: Add gpt-4o-mini to Azure. Add 2024-06-01 API version for Azure
+
+## 0.5.13
+
+### Patch Changes
+
+- Updated dependencies [04b2f8e]
+  - @llamaindex/core@0.1.7
+
+## 0.5.12
+
+### Patch Changes
+
+- 345300f: feat: add splitByPage mode to LlamaParseReader
+- da5cfc4: Add metadatafilter options to retriever constructors
+- da5cfc4: Fix system prompt not used in ContextChatEngine
+- Updated dependencies [0452af9]
+  - @llamaindex/core@0.1.6
+
+## 0.5.11
+
+### Patch Changes
+
+- Updated dependencies [1f680d7]
+  - @llamaindex/cloud@0.2.1
+
 ## 0.5.10
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.47
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
+## 0.0.45
+
+### Patch Changes
+
+- llamaindex@0.5.11
+
 ## 0.0.44
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.44",
+  "version": "0.0.48",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/next-agent-test
 
+## 0.1.48
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.1.47
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.1.46
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
+## 0.1.45
+
+### Patch Changes
+
+- llamaindex@0.5.11
+
 ## 0.1.44
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.44",
+  "version": "0.1.48",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,33 @@
 # test-edge-runtime
 
+## 0.1.47
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.1.46
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.1.45
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
+## 0.1.44
+
+### Patch Changes
+
+- llamaindex@0.5.11
+
 ## 0.1.43
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.43",
+  "version": "0.1.47",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.28
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
+## 0.0.26
+
+### Patch Changes
+
+- llamaindex@0.5.11
+
 ## 0.0.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.25",
+  "version": "0.0.29",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.47
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
+## 0.0.45
+
+### Patch Changes
+
+- llamaindex@0.5.11
+
 ## 0.0.44
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.44",
+  "version": "0.0.48",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.5.10",
+  "version": "0.5.14",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/cloud/LlamaCloudRetriever.ts

@@ -15,8 +15,8 @@ import { initService } from "./utils.js";
 
 export type CloudRetrieveParams = Omit<
   RetrievalParams,
-  "query" | "searchFilters" | "className" | "denseSimilarityTopK"
-> & { similarityTopK?: number };
+  "query" | "search_filters" | "dense_similarity_top_k"
+> & { similarityTopK?: number; filters?: MetadataFilters };
 
 export class LlamaCloudRetriever implements BaseRetriever {
   clientParams: ClientParams;
@@ -84,7 +84,9 @@ export class LlamaCloudRetriever implements BaseRetriever {
         requestBody: {
           ...this.retrieveParams,
           query: extractText(query),
-          search_filters: preFilters as MetadataFilters,
+          search_filters:
+            this.retrieveParams.filters ?? (preFilters as MetadataFilters),
+          dense_similarity_top_k: this.retrieveParams.similarityTopK,
         },
       });
 

packages/llamaindex/src/embeddings/JinaAIEmbedding.ts

@@ -79,7 +79,7 @@ export class JinaAIEmbedding extends MultiModalEmbedding {
   private async getImageInput(
     image: ImageType,
   ): Promise<{ bytes: string } | { url: string }> {
-    if (isLocal(image)) {
+    if (isLocal(image) || image instanceof Blob) {
       const base64 = await imageToDataUrl(image);
       const bytes = base64.split(",")[1];
       return { bytes };

packages/llamaindex/src/engines/chat/ContextChatEngine.ts

@@ -126,7 +126,7 @@ export class ContextChatEngine extends PromptMixin implements ChatEngine {
     if (!this.systemPrompt) return message;
     return {
       ...message,
-      content: this.systemPrompt.trim() + "\n" + message.content,
+      content: this.systemPrompt.trim() + "\n" + extractText(message.content),
     };
   }
 }

packages/llamaindex/src/engines/query/RouterQueryEngine.ts

@@ -133,7 +133,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
       const responses: EngineResponse[] = [];
       for (let i = 0; i < result.selections.length; i++) {
         const engineInd = result.selections[i];
-        const logStr = `Selecting query engine ${engineInd}: ${result.selections[i]}.`;
+        const logStr = `Selecting query engine ${engineInd.index}: ${result.selections[i].index}.`;
 
         if (this.verbose) {
           console.log(logStr + "\n");

packages/llamaindex/src/engines/query/SubQuestionQueryEngine.ts

@@ -119,15 +119,15 @@ export class SubQuestionQueryEngine
         return null;
       }
 
-      const responseText = await queryEngine?.call?.({
+      const responseValue = await queryEngine?.call?.({
         query: question,
       });
 
-      if (!responseText) {
+      if (responseValue == null) {
         return null;
       }
 
-      const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
+      const nodeText = `Sub question: ${question}\nResponse: ${typeof responseValue === "string" ? responseValue : JSON.stringify(responseValue)}`;
       const node = new TextNode({ text: nodeText });
       return { node, score: 0 };
     } catch (error) {

packages/llamaindex/src/evaluation/Relevancy.ts

@@ -78,7 +78,7 @@ export class RelevancyEvaluator extends PromptMixin implements BaseEvaluator {
       serviceContext: this.serviceContext,
     });
 
-    const queryResponse = `Question: ${query}\nResponse: ${response}`;
+    const queryResponse = `Question: ${extractText(query)}\nResponse: ${response}`;
 
     const queryEngine = index.asQueryEngine();
 

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -386,6 +386,7 @@ export type VectorIndexRetrieverOptions = {
   index: VectorStoreIndex;
   similarityTopK?: number;
   topK?: TopKMap;
+  filters?: MetadataFilters;
 };
 
 export class VectorIndexRetriever implements BaseRetriever {
@@ -393,14 +394,21 @@ export class VectorIndexRetriever implements BaseRetriever {
   topK: TopKMap;
 
   serviceContext?: ServiceContext;
+  filters?: MetadataFilters;
 
-  constructor({ index, similarityTopK, topK }: VectorIndexRetrieverOptions) {
+  constructor({
+    index,
+    similarityTopK,
+    topK,
+    filters,
+  }: VectorIndexRetrieverOptions) {
     this.index = index;
     this.serviceContext = this.index.serviceContext;
     this.topK = topK ?? {
       [ModalityType.TEXT]: similarityTopK ?? DEFAULT_SIMILARITY_TOP_K,
       [ModalityType.IMAGE]: DEFAULT_SIMILARITY_TOP_K,
     };
+    this.filters = filters;
   }
 
   /**
@@ -443,7 +451,7 @@ export class VectorIndexRetriever implements BaseRetriever {
     query: MessageContent,
     type: ModalityType,
     vectorStore: VectorStore,
-    preFilters?: MetadataFilters,
+    filters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
     // convert string message to multi-modal format
     if (typeof query === "string") {
@@ -460,7 +468,7 @@ export class VectorIndexRetriever implements BaseRetriever {
           queryEmbedding,
           mode: VectorStoreQueryMode.DEFAULT,
           similarityTopK: this.topK[type],
-          filters: preFilters ?? undefined,
+          filters: this.filters ?? filters ?? undefined,
         });
         nodes = nodes.concat(this.buildNodeListFromQueryResult(result));
       }

packages/llamaindex/src/llm/azure.ts

@@ -18,6 +18,7 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     openAIModel: "gpt-3.5-turbo-16k",
   },
   "gpt-4o": { contextWindow: 128000, openAIModel: "gpt-4o" },
+  "gpt-4o-mini": { contextWindow: 128000, openAIModel: "gpt-4o-mini" },
   "gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
   "gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
   "gpt-4-turbo": {
@@ -40,6 +41,10 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     contextWindow: 128000,
     openAIModel: "gpt-4o-2024-05-13",
   },
+  "gpt-4o-mini-2024-07-18": {
+    contextWindow: 128000,
+    openAIModel: "gpt-4o-mini-2024-07-18",
+  },
 };
 
 const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
@@ -73,6 +78,7 @@ const ALL_AZURE_API_VERSIONS = [
   "2024-03-01-preview",
   "2024-04-01-preview",
   "2024-05-01-preview",
+  "2024-06-01",
 ];
 
 const DEFAULT_API_VERSION = "2023-05-15";

packages/llamaindex/src/readers/JSONReader.ts

@@ -185,7 +185,7 @@ export class JSONReader<T extends JSONValue> extends FileReader {
       return jsonStr;
     } catch (e) {
       throw new JSONStringifyError(
-        `Error stringifying JSON: ${e} in "${data}"`,
+        `Error stringifying JSON: ${e} in "${JSON.stringify(data)}"`,
       );
     }
   }

packages/llamaindex/src/readers/LlamaParseReader.ts

@@ -143,6 +143,8 @@ export class LlamaParseReader extends FileReader {
   targetPages?: string;
   // Whether or not to ignore and skip errors raised during parsing.
   ignoreErrors: boolean = true;
+  // Whether to split by page using the pageSeparator or '\n---\n' as default.
+  splitByPage: boolean = true;
   // Whether to use the vendor multimodal API.
   useVendorMultimodalModel: boolean = false;
   // The model name for the vendor multimodal API
@@ -326,10 +328,17 @@ export class LlamaParseReader extends FileReader {
       }
 
       // Return results as Document objects
-      const resultJson = await this.getJobResult(jobId, this.resultType);
+      const jobResults = await this.getJobResult(jobId, this.resultType);
+      const resultText = jobResults[this.resultType];
+
+      // Split the text by separator if splitByPage is true
+      if (this.splitByPage) {
+        return this.splitTextBySeparator(resultText);
+      }
+
       return [
         new Document({
-          text: resultJson[this.resultType],
+          text: resultText,
         }),
       ];
     } catch (e) {
@@ -485,6 +494,17 @@ export class LlamaParseReader extends FileReader {
     return filteredParams;
   }
 
+  private splitTextBySeparator(text: string): Document[] {
+    const separator = this.pageSeparator ?? "\n---\n";
+    const textChunks = text.split(separator);
+    return textChunks.map(
+      (docChunk: string) =>
+        new Document({
+          text: docChunk,
+        }),
+    );
+  }
+
   static async getMimeType(
     data: Uint8Array,
   ): Promise<{ mime: string; extension: string }> {

packages/llamaindex/src/readers/PDFReader.ts

@@ -6,6 +6,10 @@ import { FileReader } from "./type.js";
  */
 export class PDFReader extends FileReader {
   async loadDataAsContent(content: Uint8Array): Promise<Document[]> {
+    // XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error if a Buffer passed
+    if (content instanceof Buffer) {
+      content = new Uint8Array(content);
+    }
     const { totalPages, text } = await readPDF(content);
     return text.map((text, page) => {
       const metadata = {

packages/llamaindex/src/readers/type.ts

@@ -18,8 +18,7 @@ export abstract class FileReader implements BaseReader {
   ): Promise<Document[]>;
 
   async loadData(filePath: string): Promise<Document[]> {
-    // XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error in PDFReader
-    const fileContent = new Uint8Array(await fs.readFile(filePath));
+    const fileContent = await fs.readFile(filePath);
     const fileName = path.basename(filePath);
     const docs = await this.loadDataAsContent(fileContent, fileName);
     docs.forEach(FileReader.addMetaData(filePath));