feat: add large tool call with string example

.changeset/fresh-bobcats-joke.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/env": patch
+"llamaindex": patch
+---
+
+Add support for azure dynamic session tool

.changeset/honest-monkeys-cross.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/cloud": patch
+"llamaindex": patch
+---
+
+fix: generate api as class

.changeset/tame-otters-heal.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+fix: throw error when no pipeline found

apps/docs/CHANGELOG.md

@@ -1,21 +1,5 @@
 # docs
 
-## 0.0.34
-
-### Patch Changes
-
-- Updated dependencies [83ebdfb]
-  - llamaindex@0.4.8
-
-## 0.0.33
-
-### Patch Changes
-
-- Updated dependencies [41fe871]
-- Updated dependencies [321c39d]
-- Updated dependencies [f7f1af0]
-  - llamaindex@0.4.7
-
 ## 0.0.32
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.34",
+  "version": "0.0.32",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/agent/large_toolcall_with_string.ts

@@ -0,0 +1,77 @@
+import { FunctionTool, OpenAI, ToolCallOptions } from "llamaindex";
+
+(async () => {
+  // The tool call will generate a partial JSON for `gpt-4-turbo`
+  // See thread: https://community.openai.com/t/gpt-4o-doesnt-consistently-respect-json-schema-on-tool-use/751125/7
+
+  const models = ["gpt-4o", "gpt-4-turbo"];
+  for (const model of models) {
+    const validJSON = await callLLM({ model });
+    console.log(
+      `LLM call resulting in large tool input with '${model}': LLM generates ${validJSON ? "valid" : "invalid"} JSON.`,
+    );
+  }
+})();
+
+async function callLLM(init: Partial<OpenAI>) {
+  const csvData = `Country,Average Height (cm)\n${"Netherlands,156\n".repeat(
+    50,
+  )}`;
+
+  const userQuestion = "Describe data in this csv";
+
+  // fake code interpreter tool
+  const interpreterTool = FunctionTool.from(
+    ({ data }: { data: string }) => data,
+    {
+      name: "interpreter",
+      description:
+        "Analyze csv raw data from a CSV file and return detail information.",
+      parameters: {
+        type: "object",
+        properties: {
+          data: {
+            type: "string",
+            description: "The data to analyze",
+          },
+        },
+        required: ["data"],
+      },
+    },
+  );
+
+  const systemPrompt =
+    "You are a Python interpreter.\n- You are given tasks to complete and you run python code to solve them.\n- The python code runs in a Jupyter notebook. Every time you call $(interpreter) tool, the python code is executed in a separate cell. It's okay to make multiple calls to $(interpreter).\n- Display visualizations using matplotlib or any other visualization library directly in the notebook. Shouldn't save the visualizations to a file, just return the base64 encoded data.\n- You can install any pip package (if it exists) if you need to but the usual packages for data analysis are already preinstalled.\n- You can run any python code you want in a secure environment.";
+
+  const llm = new OpenAI(init);
+  const response = await llm.chat({
+    tools: [interpreterTool],
+    messages: [
+      { role: "system", content: systemPrompt },
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: userQuestion,
+          },
+          {
+            type: "text",
+            text: `Use data from following CSV raw contents:\n${csvData}`,
+          },
+        ],
+      },
+    ],
+  });
+
+  const options = response.message?.options as ToolCallOptions;
+  const input = options.toolCall[0].input as string;
+
+  console.log({ input });
+  try {
+    JSON.parse(input);
+    return true;
+  } catch {
+    return false;
+  }
+}

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,23 +1,5 @@
 # @llamaindex/autotool-02-next-example
 
-## 0.1.18
-
-### Patch Changes
-
-- Updated dependencies [83ebdfb]
-  - llamaindex@0.4.8
-  - @llamaindex/autotool@1.0.0
-
-## 0.1.17
-
-### Patch Changes
-
-- Updated dependencies [41fe871]
-- Updated dependencies [321c39d]
-- Updated dependencies [f7f1af0]
-  - llamaindex@0.4.7
-  - @llamaindex/autotool@1.0.0
-
 ## 0.1.16
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.18",
+  "version": "0.1.16",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.4.8",
+    "llamaindex": "^0.4.6",
     "openai": "^4",
     "typescript": "^4"
   },

packages/cloud/CHANGELOG.md

@@ -1,7 +0,0 @@
-# @llamaindex/cloud
-
-## 0.1.1
-
-### Patch Changes
-
-- 321c39d: fix: generate api as class

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.1.1",
+  "version": "0.1.0",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,21 +1,5 @@
 # @llamaindex/community
 
-## 0.0.12
-
-### Patch Changes
-
-- Updated dependencies [83ebdfb]
-  - llamaindex@0.4.8
-
-## 0.0.11
-
-### Patch Changes
-
-- Updated dependencies [41fe871]
-- Updated dependencies [321c39d]
-- Updated dependencies [f7f1af0]
-  - llamaindex@0.4.7
-
 ## 0.0.10
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.12",
+  "version": "0.0.10",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -1,9 +0,0 @@
-# @llamaindex/core
-
-## 0.0.2
-
-### Patch Changes
-
-- f10b41d: fix: release files
-- Updated dependencies [41fe871]
-  - @llamaindex/env@0.1.7

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.0.2",
+  "version": "0.0.1",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./decorator": {
@@ -47,9 +47,6 @@
       }
     }
   },
-  "files": [
-    "dist"
-  ],
   "scripts": {
     "dev": "bunchee --watch",
     "build": "bunchee"

packages/env/CHANGELOG.md

@@ -1,11 +1,5 @@
 # @llamaindex/env
 
-## 0.1.7
-
-### Patch Changes
-
-- 41fe871: Add support for azure dynamic session tool
-
 ## 0.1.6
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.7",
+  "version": "0.1.6",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/experimental/CHANGELOG.md

@@ -1,21 +1,5 @@
 # @llamaindex/experimental
 
-## 0.0.43
-
-### Patch Changes
-
-- Updated dependencies [83ebdfb]
-  - llamaindex@0.4.8
-
-## 0.0.42
-
-### Patch Changes
-
-- Updated dependencies [41fe871]
-- Updated dependencies [321c39d]
-- Updated dependencies [f7f1af0]
-  - llamaindex@0.4.7
-
 ## 0.0.41
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.43",
+  "version": "0.0.41",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,25 +1,5 @@
 # llamaindex
 
-## 0.4.8
-
-### Patch Changes
-
-- 83ebdfb: fix: next.js build error
-
-## 0.4.7
-
-### Patch Changes
-
-- 41fe871: Add support for azure dynamic session tool
-- 321c39d: fix: generate api as class
-- f7f1af0: fix: throw error when no pipeline found
-- Updated dependencies [41fe871]
-- Updated dependencies [f10b41d]
-- Updated dependencies [321c39d]
-  - @llamaindex/env@0.1.7
-  - @llamaindex/core@0.0.2
-  - @llamaindex/cloud@0.1.1
-
 ## 0.4.6
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,21 +1,5 @@
 # @llamaindex/cloudflare-worker-agent-test
 
-## 0.0.27
-
-### Patch Changes
-
-- Updated dependencies [83ebdfb]
-  - llamaindex@0.4.8
-
-## 0.0.26
-
-### Patch Changes
-
-- Updated dependencies [41fe871]
-- Updated dependencies [321c39d]
-- Updated dependencies [f7f1af0]
-  - llamaindex@0.4.7
-
 ## 0.0.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.27",
+  "version": "0.0.25",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,21 +1,5 @@
 # @llamaindex/next-agent-test
 
-## 0.1.27
-
-### Patch Changes
-
-- Updated dependencies [83ebdfb]
-  - llamaindex@0.4.8
-
-## 0.1.26
-
-### Patch Changes
-
-- Updated dependencies [41fe871]
-- Updated dependencies [321c39d]
-- Updated dependencies [f7f1af0]
-  - llamaindex@0.4.7
-
 ## 0.1.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.27",
+  "version": "0.1.25",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,21 +1,5 @@
 # test-edge-runtime
 
-## 0.1.26
-
-### Patch Changes
-
-- Updated dependencies [83ebdfb]
-  - llamaindex@0.4.8
-
-## 0.1.25
-
-### Patch Changes
-
-- Updated dependencies [41fe871]
-- Updated dependencies [321c39d]
-- Updated dependencies [f7f1af0]
-  - llamaindex@0.4.7
-
 ## 0.1.24
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.26",
+  "version": "0.1.24",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,22 +1,5 @@
 # @llamaindex/next-node-runtime
 
-## 0.0.8
-
-### Patch Changes
-
-- 83ebdfb: fix: next.js build error
-- Updated dependencies [83ebdfb]
-  - llamaindex@0.4.8
-
-## 0.0.7
-
-### Patch Changes
-
-- Updated dependencies [41fe871]
-- Updated dependencies [321c39d]
-- Updated dependencies [f7f1af0]
-  - llamaindex@0.4.7
-
 ## 0.0.6
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.8",
+  "version": "0.0.6",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/src/actions/openai.ts

@@ -1,68 +0,0 @@
-"use server";
-import {
-  OpenAI,
-  OpenAIAgent,
-  QueryEngineTool,
-  Settings,
-  VectorStoreIndex,
-} from "llamaindex";
-import { HuggingFaceEmbedding } from "llamaindex/embeddings/HuggingFaceEmbedding";
-import { SimpleDirectoryReader } from "llamaindex/readers/SimpleDirectoryReader";
-
-Settings.llm = new OpenAI({
-  // eslint-disable-next-line turbo/no-undeclared-env-vars
-  apiKey: process.env.NEXT_PUBLIC_OPENAI_KEY ?? "FAKE_KEY_TO_PASS_TESTS",
-  model: "gpt-4o",
-});
-Settings.embedModel = new HuggingFaceEmbedding({
-  modelType: "BAAI/bge-small-en-v1.5",
-  quantized: false,
-});
-Settings.callbackManager.on("llm-tool-call", (event) => {
-  console.log(event.detail.payload);
-});
-Settings.callbackManager.on("llm-tool-result", (event) => {
-  console.log(event.detail.payload);
-});
-
-export async function getOpenAIModelRequest(query: string) {
-  try {
-    const currentDir = __dirname;
-
-    // load our data and create a query engine
-    const reader = new SimpleDirectoryReader();
-    const documents = await reader.loadData(currentDir);
-    const index = await VectorStoreIndex.fromDocuments(documents);
-    const retriever = index.asRetriever({
-      similarityTopK: 10,
-    });
-    const queryEngine = index.asQueryEngine({
-      retriever,
-    });
-
-    // define the query engine as a tool
-    const tools = [
-      new QueryEngineTool({
-        queryEngine: queryEngine,
-        metadata: {
-          name: "deployment_details_per_env",
-          description: `This tool can answer detailed questions about deployments happened in various environments.`,
-        },
-      }),
-    ];
-    // create the agent
-    const agent = new OpenAIAgent({ tools });
-
-    const { response } = await agent.chat({
-      message: query,
-    });
-    return {
-      message: response,
-    };
-  } catch (err) {
-    console.error(err);
-    return {
-      errors: "Error Calling OpenAI Model",
-    };
-  }
-}

packages/llamaindex/e2e/examples/nextjs-node-runtime/src/app/api/openai/route.ts

@@ -1,10 +0,0 @@
-import { getOpenAIModelRequest } from "@/actions/openai";
-import { NextRequest, NextResponse } from "next/server";
-
-// POST /api/openai
-export async function POST(request: NextRequest) {
-  const body = await request.json();
-  const content = await getOpenAIModelRequest(body.query);
-
-  return NextResponse.json(content, { status: 200 });
-}

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,21 +1,5 @@
 # @llamaindex/waku-query-engine-test
 
-## 0.0.27
-
-### Patch Changes
-
-- Updated dependencies [83ebdfb]
-  - llamaindex@0.4.8
-
-## 0.0.26
-
-### Patch Changes
-
-- Updated dependencies [41fe871]
-- Updated dependencies [321c39d]
-- Updated dependencies [f7f1af0]
-  - llamaindex@0.4.7
-
 ## 0.0.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.27",
+  "version": "0.0.25",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.4.8",
+  "version": "0.4.6",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/next.ts

@@ -16,23 +16,23 @@
  * @module
  */
 export default function withLlamaIndex(config: any) {
-  config.experimental = config.experimental ?? {};
-  config.experimental.serverComponentsExternalPackages =
-    config.experimental.serverComponentsExternalPackages ?? [];
-  config.experimental.serverComponentsExternalPackages.push(
-    "@xenova/transformers",
-  );
   const userWebpack = config.webpack;
+  //#region hack for `@xenova/transformers`
+  // Ignore node-specific modules when bundling for the browser
+  // See https://webpack.js.org/configuration/resolve/#resolvealias
   config.webpack = function (webpackConfig: any) {
     if (userWebpack) {
       webpackConfig = userWebpack(webpackConfig);
     }
     webpackConfig.resolve.alias = {
       ...webpackConfig.resolve.alias,
+      sharp$: false,
+      "onnxruntime-node$": false,
       "@google-cloud/vertexai": false,
       "groq-sdk": false,
     };
     return webpackConfig;
   };
+  //#endregion
   return config;
 }