Release 0.4.9 (#1001)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

README.md

@@ -76,7 +76,7 @@ main();
 node --import tsx ./main.ts
 ```
 
-### Next.js
+### React Server Component (Next.js, Waku, Redwood.JS...)
 
 First, you will need to add a llamaindex plugin to your Next.js project.
 
@@ -154,40 +154,6 @@ export async function chatWithAgent(
 }
 ```
 
-### Cloudflare Workers
-
-```ts
-// src/index.ts
-export default {
-  async fetch(
-    request: Request,
-    env: Env,
-    ctx: ExecutionContext,
-  ): Promise<Response> {
-    const { setEnvs } = await import("@llamaindex/env");
-    // set environment variables so that the OpenAIAgent can use them
-    setEnvs(env);
-    const { OpenAIAgent } = await import("llamaindex");
-    const agent = new OpenAIAgent({
-      tools: [],
-    });
-    const responseStream = await agent.chat({
-      stream: true,
-      message: "Hello? What is the weather today?",
-    });
-    const textEncoder = new TextEncoder();
-    const response = responseStream.pipeThrough(
-      new TransformStream({
-        transform: (chunk, controller) => {
-          controller.enqueue(textEncoder.encode(chunk.response.delta));
-        },
-      }),
-    );
-    return new Response(response);
-  },
-};
-```
-
 ## Playground
 
 Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground

apps/docs/CHANGELOG.md

@@ -1,5 +1,28 @@
 # docs
 
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.34
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.33
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.32
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.32",
+  "version": "0.0.35",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/README.md

@@ -9,7 +9,7 @@ make sure you have basic knowledge of the [LlamaIndexTS](https://ts.llamaindex.a
 # export your API key
 export OPENAI_API_KEY="sk-..."
 
-npx ts-node ./chatEngine.ts
+npx tsx ./chatEngine.ts
 ```
 
 ## Build your own RAG app

examples/agent/azure_dynamic_session.ts

@@ -0,0 +1,54 @@
+import "dotenv/config";
+
+import {
+  DefaultAzureCredential,
+  getBearerTokenProvider,
+} from "@azure/identity";
+import { AzureDynamicSessionTool, OpenAI, ReActAgent } from "llamaindex";
+
+async function main() {
+  const credential = new DefaultAzureCredential();
+  const azureADTokenProvider = getBearerTokenProvider(
+    credential,
+    "https://cognitiveservices.azure.com/.default",
+  );
+
+  const azure = {
+    azureADTokenProvider,
+    deployment: process.env.AZURE_OPENAI_DEPLOYMENT ?? "gpt-35-turbo",
+  };
+
+  // configure LLM model
+  const llm = new OpenAI({
+    azure,
+  });
+
+  const azureDynamicSession = new AzureDynamicSessionTool();
+
+  // Create an ReActAgent with the azure dynamic session tool
+  const agent = new ReActAgent({
+    llm,
+    tools: [azureDynamicSession],
+    // verbose: true,
+    systemPrompt: `You are a Python interpreter.
+      - You are given tasks to complete and you run python code to solve them.
+      - The python code runs by the python runtime. Every time you call $(interpreter) tool, the python code is executed in a separate cell. It's okay to make multiple calls to $(interpreter).
+      - You can run any python code you want in a secure environment.
+      - For images, return the full URL, not the base64 data.
+      - Return any image content as an HTML tag with the src attribute set to the URL of the image.`,
+  });
+
+  // Chat with the agent
+  const response = await agent.chat({
+    message:
+      "plot a chart of 5 random numbers and save it to /mnt/data/chart.png",
+    stream: false,
+  });
+
+  // Print the response
+  console.log({ response });
+}
+
+void main().then(() => {
+  console.log("Done");
+});

examples/astradb/README.md

@@ -24,7 +24,7 @@ Here are two sample scripts which work well with the sample data in the Astra Po
 
 Loads and queries a simple vectorstore with some documents about Astra DB
 
-run `ts-node astradb/example`
+run `tsx astradb/example`
 
 ## Movie Reviews Example
 
@@ -32,10 +32,10 @@ run `ts-node astradb/example`
 
 This sample loads the same dataset of movie reviews as the Astra Portal sample dataset. (Feel free to load the data in your the Astra Data Explorer to compare)
 
-run `npx ts-node astradb/load`
+run `npx tsx astradb/load`
 
 ### Use RAG to Query the data
 
 Check out your data in the Astra Data Explorer and change the sample query as you see fit.
 
-run `npx ts-node astradb/query`
+run `npx tsx astradb/query`

examples/chromadb/README.md

@@ -6,7 +6,7 @@ Export your OpenAI API Key using `export OPEN_API_KEY=insert your api key here`
 
 If you haven't installed chromadb, run `pip install chromadb`. Start the server using `chroma run`.
 
-Now, open a new terminal window and inside `examples`, run `pnpm dlx ts-node chromadb/test.ts`.
+Now, open a new terminal window and inside `examples`, run `pnpm dlx tsx chromadb/test.ts`.
 
 Here's the output for the input query `Tell me about Godfrey Cheshire's rating of La Sapienza.`:
 

examples/cloud/README.md

@@ -21,7 +21,7 @@ export LLAMA_CLOUD_BASE_URL="https://api.staging.llamaindex.ai"
 This example is using the managed index named `test` from the project `default` to create a chat engine.
 
 ```shell
-pnpx ts-node cloud/chat.ts
+pnpx tsx cloud/chat.ts
 ```
 
 ## Query Engine
@@ -29,7 +29,7 @@ pnpx ts-node cloud/chat.ts
 This example shows how to use the managed index with a query engine.
 
 ```shell
-pnpx ts-node cloud/query.ts
+pnpx tsx cloud/query.ts
 ```
 
 ## Pipeline
@@ -37,5 +37,5 @@ pnpx ts-node cloud/query.ts
 This example shows how to create a managed index with a pipeline.
 
 ```shell
-pnpx ts-node cloud/pipeline.ts
+pnpx tsx cloud/pipeline.ts
 ```

examples/cloud/chat.ts

@@ -6,7 +6,7 @@ import { ContextChatEngine, LlamaCloudIndex } from "llamaindex";
 async function main() {
   const index = new LlamaCloudIndex({
     name: "test",
-    projectName: "default",
+    projectName: "Default",
     baseUrl: process.env.LLAMA_CLOUD_BASE_URL,
     apiKey: process.env.LLAMA_CLOUD_API_KEY,
   });
@@ -19,10 +19,10 @@ async function main() {
   while (true) {
     const query = await rl.question("User: ");
     const stream = await chatEngine.chat({ message: query, stream: true });
-    console.log();
     for await (const chunk of stream) {
       process.stdout.write(chunk.response);
     }
+    process.stdout.write("\n");
   }
 }
 

examples/milvus/README.md

@@ -25,10 +25,10 @@ Here are two sample scripts which work with loading and querying data from a Mil
 
 This sample loads the same dataset of movie reviews as sample dataset. You can install https://github.com/zilliztech/attu to inspect the loaded data.
 
-run `npx ts-node milvus/load`
+run `npx tsx milvus/load`
 
 ## Use RAG to Query the data
 
 Check out your data in Attu and change the sample query as you see fit.
 
-run `npx ts-node milvus/query`
+run `npx tsx milvus/query`

examples/mongodb/README.md

@@ -34,7 +34,7 @@ MONGODB_COLLECTION=tiny_tweets_collection
 You are now ready to import our ready-made data set into Mongo. This is the file `tinytweets.json`, a selection of approximately 1000 tweets from @seldo on Twitter in mid-2019. With your environment set up you can do this by running
 
 ```
-npx ts-node mongodb/1_import.ts
+npx tsx mongodb/1_import.ts
 ```
 
 If you don't want to use tweets, you can replace `json_file` with any other array of JSON objects, but you will need to modify some code later to make sure the correct field gets indexed. There is no LlamaIndex-specific code here; you can load your data into Mongo any way you want to.
@@ -59,7 +59,7 @@ MONGODB_VECTOR_INDEX=tiny_tweets_vector_index
 If the data you're indexing is the tweets we gave you, you're ready to go:
 
 ```bash
-npx ts-node mongodb/2_load_and_index.ts
+npx tsx mongodb/2_load_and_index.ts
 ```
 
 > Note: this script is running a couple of minutes and currently doesn't show any progress.
@@ -112,7 +112,7 @@ Now you're ready to query your data!
 You can do this by running
 
 ```bash
-npx ts-node mongodb/3_query.ts
+npx tsx mongodb/3_query.ts
 ```
 
 This sets up a connection to Atlas just like `2_load_and_index.ts` did, then it creates a [query engine](https://docs.llamaindex.ai/en/stable/understanding/querying/querying.html#getting-started) and runs a query against it.

examples/package.json

@@ -4,6 +4,7 @@
   "version": "0.0.6",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
+    "@azure/identity": "^4.2.1",
     "@datastax/astra-db-ts": "^1.2.1",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^2.2.2",
@@ -18,7 +19,6 @@
   },
   "devDependencies": {
     "@types/node": "^20.14.1",
-    "ts-node": "^10.9.2",
     "tsx": "^4.15.6",
     "typescript": "^5.5.2"
   },

examples/pg-vector-store/README.md

@@ -37,7 +37,7 @@ Read and follow the instructions in the README.md file located one directory up
 
 To import documents and save the embedding vectors to your database:
 
-> `npx ts-node pg-vector-store/load-docs.ts data`
+> `npx tsx pg-vector-store/load-docs.ts data`
 
 where data is the directory containing your input files. Using the `data` directory in the example above will read all of the files in that directory using the LlamaIndexTS default readers for each file type.
 
@@ -45,6 +45,6 @@ where data is the directory containing your input files. Using the `data` direct
 
 To query using the resulting vector store:
 
-> `npx ts-node pg-vector-store/query.ts`
+> `npx tsx pg-vector-store/query.ts`
 
 The script will prompt for a question, then process and present the answer using the PGVectorStore data and your OpenAI API key. It will continue to prompt until you enter `q`, `quit` or `exit` as the next query.

examples/pinecone-vector-store/README.md

@@ -19,7 +19,7 @@ Read and follow the instructions in the README.md file located one directory up
 
 To import documents and save the embedding vectors to your database:
 
-> `npx ts-node pinecone-vector-store/load-docs.ts data`
+> `npx tsx pinecone-vector-store/load-docs.ts data`
 
 where data is the directory containing your input files. Using the _data_ directory in the example above will read all of the files in that directory using the llamaindexTS default readers for each file type.
 
@@ -29,6 +29,6 @@ where data is the directory containing your input files. Using the _data_ direct
 
 To query using the resulting vector store:
 
-> `npx ts-node pinecone-vector-store/query.ts`
+> `npx tsx pinecone-vector-store/query.ts`
 
 The script will prompt for a question, then process and present the answer using the PineconeVectorStore data and your OpenAI API key. It will continue to prompt until you enter `q`, `quit` or `exit` as the next query.

examples/qdrantdb/README.md

@@ -8,4 +8,4 @@ Add your OpenAI API Key into a file called `.env` in the parent folder of this d
 OPEN_API_KEY=sk-you-key
 ```
 
-Now, open a new terminal window and inside `examples`, run `npx ts-node qdrantdb/preFilters.ts`.
+Now, open a new terminal window and inside `examples`, run `npx tsx qdrantdb/preFilters.ts`.

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,31 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.19
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.18
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.17
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+  - @llamaindex/autotool@1.0.0
+
 ## 0.1.16
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.16",
+  "version": "0.1.19",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.4.6",
+    "llamaindex": "^0.4.9",
     "openai": "^4",
     "typescript": "^4"
   },

packages/cloud/CHANGELOG.md

@@ -0,0 +1,7 @@
+# @llamaindex/cloud
+
+## 0.1.1
+
+### Patch Changes
+
+- 321c39d: fix: generate api as class

packages/cloud/README.md

@@ -5,7 +5,7 @@
 ## Usage
 
 ```ts
-import { OpenAPI, Service } from "@llamaindex/cloud/api";
+import { OpenAPI } from "@llamaindex/cloud/api";
 OpenAPI.TOKEN = "YOUR_API_KEY";
 OpenAPI.BASE = "https://api.cloud.llamaindex.ai/";
 // ...

packages/cloud/openapi-ts.config.ts

@@ -9,6 +9,9 @@ export default defineConfig({
     format: "prettier",
     lint: "eslint",
   },
+  services: {
+    asClass: true,
+  },
   types: {
     enums: "javascript",
   },

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/cloud/src/api.ts

@@ -1,4 +1 @@
-import * as Service from "./client/services.gen";
-
 export * from "./client";
-export { Service };

packages/community/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @llamaindex/community
 
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.10
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.10",
+  "version": "0.0.13",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -0,0 +1,9 @@
+# @llamaindex/core
+
+## 0.0.2
+
+### Patch Changes
+
+- f10b41d: fix: release files
+- Updated dependencies [41fe871]
+  - @llamaindex/env@0.1.7

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./decorator": {
@@ -47,6 +47,9 @@
       }
     }
   },
+  "files": [
+    "dist"
+  ],
   "scripts": {
     "dev": "bunchee --watch",
     "build": "bunchee"

packages/env/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/env
 
+## 0.1.7
+
+### Patch Changes
+
+- 41fe871: Add support for azure dynamic session tool
+
 ## 0.1.6
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/env/src/fs/node.ts

@@ -5,6 +5,7 @@
  *
  * @module
  */
+import { createWriteStream } from "node:fs";
 import fs from "node:fs/promises";
 
-export { fs };
+export { createWriteStream, fs };

packages/env/src/index.ts

@@ -15,12 +15,14 @@ import { ok } from "node:assert";
 import { createHash, randomUUID } from "node:crypto";
 import { EOL } from "node:os";
 import path from "node:path";
+import { Readable } from "node:stream";
 import {
   ReadableStream,
   TransformStream,
   WritableStream,
 } from "node:stream/web";
-import { fs } from "./fs/node.js";
+import { fileURLToPath } from "node:url";
+import { createWriteStream, fs } from "./fs/node.js";
 import type { SHA256 } from "./polyfill.js";
 
 export function createSHA256(): SHA256 {
@@ -38,11 +40,14 @@ export function createSHA256(): SHA256 {
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js";
 export { AsyncLocalStorage, CustomEvent, getEnv, setEnvs } from "./utils.js";
 export {
+  createWriteStream,
   EOL,
+  fileURLToPath,
   fs,
   ok,
   path,
   randomUUID,
+  Readable,
   ReadableStream,
   TransformStream,
   WritableStream,

packages/experimental/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @llamaindex/experimental
 
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.43
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.42
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.41
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.41",
+  "version": "0.0.44",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,31 @@
 # llamaindex
 
+## 0.4.9
+
+### Patch Changes
+
+- 3a96a48: fix: anthroipic image input
+
+## 0.4.8
+
+### Patch Changes
+
+- 83ebdfb: fix: next.js build error
+
+## 0.4.7
+
+### Patch Changes
+
+- 41fe871: Add support for azure dynamic session tool
+- 321c39d: fix: generate api as class
+- f7f1af0: fix: throw error when no pipeline found
+- Updated dependencies [41fe871]
+- Updated dependencies [f10b41d]
+- Updated dependencies [321c39d]
+  - @llamaindex/env@0.1.7
+  - @llamaindex/core@0.0.2
+  - @llamaindex/cloud@0.1.1
+
 ## 0.4.6
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.25",
+  "version": "0.0.28",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @llamaindex/next-agent-test
 
+## 0.1.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.1.27
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.1.26
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.1.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.25",
+  "version": "0.1.28",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,28 @@
 # test-edge-runtime
 
+## 0.1.27
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.1.26
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.1.25
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.1.24
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.24",
+  "version": "0.1.27",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,29 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.9
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.8
+
+### Patch Changes
+
+- 83ebdfb: fix: next.js build error
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.7
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.6
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.6",
+  "version": "0.0.9",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/src/actions/openai.ts

@@ -0,0 +1,68 @@
+"use server";
+import {
+  OpenAI,
+  OpenAIAgent,
+  QueryEngineTool,
+  Settings,
+  VectorStoreIndex,
+} from "llamaindex";
+import { HuggingFaceEmbedding } from "llamaindex/embeddings/HuggingFaceEmbedding";
+import { SimpleDirectoryReader } from "llamaindex/readers/SimpleDirectoryReader";
+
+Settings.llm = new OpenAI({
+  // eslint-disable-next-line turbo/no-undeclared-env-vars
+  apiKey: process.env.NEXT_PUBLIC_OPENAI_KEY ?? "FAKE_KEY_TO_PASS_TESTS",
+  model: "gpt-4o",
+});
+Settings.embedModel = new HuggingFaceEmbedding({
+  modelType: "BAAI/bge-small-en-v1.5",
+  quantized: false,
+});
+Settings.callbackManager.on("llm-tool-call", (event) => {
+  console.log(event.detail.payload);
+});
+Settings.callbackManager.on("llm-tool-result", (event) => {
+  console.log(event.detail.payload);
+});
+
+export async function getOpenAIModelRequest(query: string) {
+  try {
+    const currentDir = __dirname;
+
+    // load our data and create a query engine
+    const reader = new SimpleDirectoryReader();
+    const documents = await reader.loadData(currentDir);
+    const index = await VectorStoreIndex.fromDocuments(documents);
+    const retriever = index.asRetriever({
+      similarityTopK: 10,
+    });
+    const queryEngine = index.asQueryEngine({
+      retriever,
+    });
+
+    // define the query engine as a tool
+    const tools = [
+      new QueryEngineTool({
+        queryEngine: queryEngine,
+        metadata: {
+          name: "deployment_details_per_env",
+          description: `This tool can answer detailed questions about deployments happened in various environments.`,
+        },
+      }),
+    ];
+    // create the agent
+    const agent = new OpenAIAgent({ tools });
+
+    const { response } = await agent.chat({
+      message: query,
+    });
+    return {
+      message: response,
+    };
+  } catch (err) {
+    console.error(err);
+    return {
+      errors: "Error Calling OpenAI Model",
+    };
+  }
+}

packages/llamaindex/e2e/examples/nextjs-node-runtime/src/app/api/openai/route.ts

@@ -0,0 +1,10 @@
+import { getOpenAIModelRequest } from "@/actions/openai";
+import { NextRequest, NextResponse } from "next/server";
+
+// POST /api/openai
+export async function POST(request: NextRequest) {
+  const body = await request.json();
+  const content = await getOpenAIModelRequest(body.query);
+
+  return NextResponse.json(content, { status: 200 });
+}

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [3a96a48]
+  - llamaindex@0.4.9
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [83ebdfb]
+  - llamaindex@0.4.8
+
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [41fe871]
+- Updated dependencies [321c39d]
+- Updated dependencies [f7f1af0]
+  - llamaindex@0.4.7
+
 ## 0.0.25
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.25",
+  "version": "0.0.28",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.4.6",
+  "version": "0.4.9",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -22,6 +22,7 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.21.1",
     "@aws-crypto/sha256-js": "^5.2.0",
+    "@azure/identity": "^4.2.1",
     "@datastax/astra-db-ts": "^1.2.1",
     "@google-cloud/vertexai": "^1.2.0",
     "@google/generative-ai": "^0.12.0",
@@ -33,7 +34,8 @@
     "@mistralai/mistralai": "^0.4.0",
     "@pinecone-database/pinecone": "^2.2.2",
     "@qdrant/js-client-rest": "^1.9.0",
-    "@types/lodash": "^4.17.5",
+    "@types/lodash": "^4.17.4",
+    "@types/node": "^20.14.5",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.6",
     "@xenova/transformers": "^2.17.2",

packages/llamaindex/src/cloud/LlamaCloudIndex.ts

@@ -11,8 +11,9 @@ import { getPipelineCreate } from "./config.js";
 import type { CloudConstructorParams } from "./constants.js";
 import { getAppBaseUrl, initService } from "./utils.js";
 
-import { Service } from "@llamaindex/cloud/api";
+import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api";
 import { getEnv } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
 import { OpenAIEmbedding } from "../embeddings/OpenAIEmbedding.js";
 import { SimpleNodeParser } from "../nodeParsers/SimpleNodeParser.js";
 
@@ -25,7 +26,7 @@ export class LlamaCloudIndex {
   }
 
   private async waitForPipelineIngestion(
-    verbose = false,
+    verbose = Settings.debug,
     raiseOnError = false,
   ): Promise<void> {
     const pipelineId = await this.getPipelineId(
@@ -39,9 +40,11 @@ export class LlamaCloudIndex {
 
     while (true) {
       const pipelineStatus =
-        await Service.getPipelineStatusApiV1PipelinesPipelineIdStatusGet({
+        await PipelinesService.getPipelineStatusApiV1PipelinesPipelineIdStatusGet(
-          pipelineId,
+          {
-        });
+            pipelineId,
+          },
+        );
 
       if (pipelineStatus.status === "SUCCESS") {
         if (verbose) {
@@ -70,7 +73,7 @@ export class LlamaCloudIndex {
 
   private async waitForDocumentIngestion(
     docIds: string[],
-    verbose = false,
+    verbose = Settings.debug,
     raiseOnError = false,
   ): Promise<void> {
     const pipelineId = await this.getPipelineId(
@@ -78,11 +81,6 @@ export class LlamaCloudIndex {
       this.params.projectName,
     );
 
-    const client = await initService({
-      ...this.params,
-      baseUrl: this.params.baseUrl,
-    });
-
     if (verbose) {
       console.log("Loading data: ");
     }
@@ -94,7 +92,7 @@ export class LlamaCloudIndex {
 
       for (const doc of pendingDocs) {
         const { status } =
-          await Service.getPipelineDocumentStatusApiV1PipelinesPipelineIdDocumentsDocumentIdStatusGet(
+          await PipelinesService.getPipelineDocumentStatusApiV1PipelinesPipelineIdDocumentsDocumentIdStatusGet(
             { pipelineId, documentId: doc },
           );
 
@@ -139,7 +137,7 @@ export class LlamaCloudIndex {
     name: string,
     projectName: string,
   ): Promise<string> {
-    const pipelines = await Service.searchPipelinesApiV1PipelinesGet({
+    const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
       projectName,
       pipelineName: name,
     });
@@ -160,10 +158,7 @@ export class LlamaCloudIndex {
         apiKey: getEnv("OPENAI_API_KEY"),
       }),
     ];
-
+    const apiUrl = getAppBaseUrl();
-    const appUrl = getAppBaseUrl(params.baseUrl);
-
-    const client = await initService({ ...params, baseUrl: appUrl });
 
     const pipelineCreateParams = await getPipelineCreate({
       pipelineName: params.name,
@@ -172,7 +167,7 @@ export class LlamaCloudIndex {
       transformations: params.transformations ?? defaultTransformations,
     });
 
-    const project = await Service.upsertProjectApiV1ProjectsPut({
+    const project = await ProjectsService.upsertProjectApiV1ProjectsPut({
       requestBody: {
         name: params.projectName ?? "default",
       },
@@ -182,7 +177,7 @@ export class LlamaCloudIndex {
       throw new Error("Project ID should be defined");
     }
 
-    const pipeline = await Service.upsertPipelineApiV1PipelinesPut({
+    const pipeline = await PipelinesService.upsertPipelineApiV1PipelinesPut({
       projectId: project.id,
       requestBody: {
         name: params.name,
@@ -200,7 +195,7 @@ export class LlamaCloudIndex {
       console.log(`Created pipeline ${pipeline.id} with name ${params.name}`);
     }
 
-    await Service.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
+    await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
       {
         pipelineId: pipeline.id,
         requestBody: params.documents.map((doc) => ({
@@ -215,9 +210,11 @@ export class LlamaCloudIndex {
 
     while (true) {
       const pipelineStatus =
-        await Service.getPipelineStatusApiV1PipelinesPipelineIdStatusGet({
+        await PipelinesService.getPipelineStatusApiV1PipelinesPipelineIdStatusGet(
-          pipelineId: pipeline.id,
+          {
-        });
+            pipelineId: pipeline.id,
+          },
+        );
 
       if (pipelineStatus.status === "SUCCESS") {
         console.info(
@@ -228,14 +225,14 @@ export class LlamaCloudIndex {
 
       if (pipelineStatus.status === "ERROR") {
         console.error(
-          `Some documents failed to ingest, check your pipeline logs at ${appUrl}/project/${project.id}/deploy/${pipeline.id}`,
+          `Some documents failed to ingest, check your pipeline logs at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
         );
         throw new Error("Some documents failed to ingest");
       }
 
       if (pipelineStatus.status === "PARTIAL_SUCCESS") {
         console.info(
-          `Documents ingestion partially succeeded, to check a more complete status check your pipeline at ${appUrl}/project/${project.id}/deploy/${pipeline.id}`,
+          `Documents ingestion partially succeeded, to check a more complete status check your pipeline at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
         );
         break;
       }
@@ -249,7 +246,7 @@ export class LlamaCloudIndex {
 
     if (params.verbose) {
       console.info(
-        `Ingestion completed, find your index at ${appUrl}/project/${project.id}/deploy/${pipeline.id}`,
+        `Ingestion completed, find your index at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
       );
     }
 
@@ -280,10 +277,6 @@ export class LlamaCloudIndex {
   }
 
   async insert(document: Document) {
-    const appUrl = getAppBaseUrl(this.params.baseUrl);
-
-    const client = await initService({ ...this.params, baseUrl: appUrl });
-
     const pipelineId = await this.getPipelineId(
       this.params.name,
       this.params.projectName,
@@ -293,7 +286,7 @@ export class LlamaCloudIndex {
       throw new Error("We couldn't find the pipeline ID for the given name");
     }
 
-    await Service.createBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPost(
+    await PipelinesService.createBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPost(
       {
         pipelineId: pipelineId,
         requestBody: [
@@ -312,10 +305,6 @@ export class LlamaCloudIndex {
   }
 
   async delete(document: Document) {
-    const appUrl = getAppBaseUrl(this.params.baseUrl);
-
-    const client = await initService({ ...this.params, baseUrl: appUrl });
-
     const pipelineId = await this.getPipelineId(
       this.params.name,
       this.params.projectName,
@@ -325,7 +314,7 @@ export class LlamaCloudIndex {
       throw new Error("We couldn't find the pipeline ID for the given name");
     }
 
-    await Service.deletePipelineDocumentApiV1PipelinesPipelineIdDocumentsDocumentIdDelete(
+    await PipelinesService.deletePipelineDocumentApiV1PipelinesPipelineIdDocumentsDocumentIdDelete(
       {
         pipelineId,
         documentId: document.id_,
@@ -336,10 +325,6 @@ export class LlamaCloudIndex {
   }
 
   async refreshDoc(document: Document) {
-    const appUrl = getAppBaseUrl(this.params.baseUrl);
-
-    const client = await initService({ ...this.params, baseUrl: appUrl });
-
     const pipelineId = await this.getPipelineId(
       this.params.name,
       this.params.projectName,
@@ -349,7 +334,7 @@ export class LlamaCloudIndex {
       throw new Error("We couldn't find the pipeline ID for the given name");
     }
 
-    await Service.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
+    await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
       {
         pipelineId,
         requestBody: [

packages/llamaindex/src/cloud/LlamaCloudRetriever.ts

@@ -1,7 +1,7 @@
 import {
   type MetadataFilters,
+  PipelinesService,
   type RetrievalParams,
-  Service,
   type TextNodeWithScore,
 } from "@llamaindex/cloud/api";
 import type { NodeWithScore } from "@llamaindex/core/schema";
@@ -51,20 +51,21 @@ export class LlamaCloudRetriever implements BaseRetriever {
     query,
     preFilters,
   }: RetrieveParams): Promise<NodeWithScore[]> {
-    const pipelines = await Service.searchPipelinesApiV1PipelinesGet({
+    const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
       projectName: this.projectName,
       pipelineName: this.pipelineName,
     });
 
-    if (!pipelines) {
+    if (pipelines.length === 0 || !pipelines[0].id) {
       throw new Error(
         `No pipeline found with name ${this.pipelineName} in project ${this.projectName}`,
       );
     }
 
-    const pipeline = await Service.getPipelineApiV1PipelinesPipelineIdGet({
+    const pipeline =
-      pipelineId: pipelines[0].id,
+      await PipelinesService.getPipelineApiV1PipelinesPipelineIdGet({
-    });
+        pipelineId: pipelines[0].id,
+      });
 
     if (!pipeline) {
       throw new Error(
@@ -72,16 +73,15 @@ export class LlamaCloudRetriever implements BaseRetriever {
       );
     }
 
-    const results = await Service.runSearchApiV1PipelinesPipelineIdRetrievePost(
+    const results =
-      {
+      await PipelinesService.runSearchApiV1PipelinesPipelineIdRetrievePost({
         pipelineId: pipeline.id,
         requestBody: {
           ...this.retrieveParams,
           query: extractText(query),
           search_filters: preFilters as MetadataFilters,
         },
-      },
+      });
-    );
 
     return this.resultNodesToNodeWithScore(results.retrieval_nodes);
   }

packages/llamaindex/src/cloud/constants.ts

@@ -1,6 +1,5 @@
 import type { ServiceContext } from "../ServiceContext.js";
 
-export const DEFAULT_PIPELINE_NAME = "default";
 export const DEFAULT_PROJECT_NAME = "Default";
 export const DEFAULT_BASE_URL = "https://api.cloud.llamaindex.ai";
 

packages/llamaindex/src/cloud/utils.ts

@@ -1,4 +1,4 @@
-import { OpenAPI, Service } from "@llamaindex/cloud/api";
+import { OpenAPI } from "@llamaindex/cloud/api";
 import { getEnv } from "@llamaindex/env";
 import type { ClientParams } from "./constants.js";
 import { DEFAULT_BASE_URL } from "./constants.js";
@@ -7,14 +7,11 @@ function getBaseUrl(baseUrl?: string): string {
   return baseUrl ?? getEnv("LLAMA_CLOUD_BASE_URL") ?? DEFAULT_BASE_URL;
 }
 
-export function getAppBaseUrl(baseUrl?: string): string {
+export function getAppBaseUrl(): string {
-  return getBaseUrl(baseUrl).replace(/api\./, "");
+  return OpenAPI.BASE.replace(/api\./, "");
 }
 
-export function initService({
+export function initService({ apiKey, baseUrl }: ClientParams = {}) {
-  apiKey,
-  baseUrl,
-}: ClientParams = {}): typeof Service {
   OpenAPI.TOKEN = apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
   OpenAPI.BASE = getBaseUrl(baseUrl);
   if (!OpenAPI.TOKEN) {
@@ -22,6 +19,4 @@ export function initService({
       "API Key is required for LlamaCloudIndex. Please pass the apiKey parameter",
     );
   }
-
-  return Service;
 }

packages/llamaindex/src/index.ts

@@ -13,3 +13,6 @@ export {
 
 export { type VertexGeminiSessionOptions } from "./llm/gemini/types.js";
 export { GeminiVertexSession } from "./llm/gemini/vertex.js";
+
+// Expose AzureDynamicSessionTool for node.js runtime only
+export { AzureDynamicSessionTool } from "./tools/AzureDynamicSessionTool.node.js";

packages/llamaindex/src/llm/anthropic.ts

@@ -13,7 +13,10 @@ import type {
   TextBlock,
   TextBlockParam,
 } from "@anthropic-ai/sdk/resources/index";
-import type { MessageParam } from "@anthropic-ai/sdk/resources/messages";
+import type {
+  ImageBlockParam,
+  MessageParam,
+} from "@anthropic-ai/sdk/resources/messages";
 import { getEnv } from "@llamaindex/env";
 import _ from "lodash";
 import type { BaseTool } from "../types.js";
@@ -214,7 +217,32 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
         }
 
         return {
-          content: extractText(message.content),
+          content:
+            typeof message.content === "string"
+              ? message.content
+              : message.content.map(
+                  (content): TextBlockParam | ImageBlockParam =>
+                    content.type === "text"
+                      ? {
+                          type: "text",
+                          text: content.text,
+                        }
+                      : {
+                          type: "image",
+                          source: {
+                            data: content.image_url.url.substring(
+                              content.image_url.url.indexOf(",") + 1,
+                            ),
+                            media_type:
+                              `image/${content.image_url.url.substring("data:image/".length, content.image_url.url.indexOf(";base64"))}` as
+                                | "image/jpeg"
+                                | "image/png"
+                                | "image/gif"
+                                | "image/webp",
+                            type: "base64",
+                          },
+                        },
+                ),
           role: message.role as "user" | "assistant",
         } satisfies MessageParam;
       });

packages/llamaindex/src/next.ts

@@ -16,23 +16,23 @@
  * @module
  */
 export default function withLlamaIndex(config: any) {
+  config.experimental = config.experimental ?? {};
+  config.experimental.serverComponentsExternalPackages =
+    config.experimental.serverComponentsExternalPackages ?? [];
+  config.experimental.serverComponentsExternalPackages.push(
+    "@xenova/transformers",
+  );
   const userWebpack = config.webpack;
-  //#region hack for `@xenova/transformers`
-  // Ignore node-specific modules when bundling for the browser
-  // See https://webpack.js.org/configuration/resolve/#resolvealias
   config.webpack = function (webpackConfig: any) {
     if (userWebpack) {
       webpackConfig = userWebpack(webpackConfig);
     }
     webpackConfig.resolve.alias = {
       ...webpackConfig.resolve.alias,
-      sharp$: false,
-      "onnxruntime-node$": false,
       "@google-cloud/vertexai": false,
       "groq-sdk": false,
     };
     return webpackConfig;
   };
-  //#endregion
   return config;
 }

packages/llamaindex/src/tools/AzureDynamicSessionTool.node.ts

@@ -0,0 +1,457 @@
+import {
+  DefaultAzureCredential,
+  getBearerTokenProvider,
+} from "@azure/identity";
+import {
+  Readable,
+  createWriteStream,
+  fileURLToPath,
+  fs,
+  getEnv,
+  path,
+  randomUUID,
+} from "@llamaindex/env";
+import type { BaseTool, ToolMetadata } from "../types.js";
+export type InterpreterParameter = {
+  code: string;
+};
+
+export type InterpreterToolOutputImage = {
+  base64_data: string;
+  format: string;
+  type: "image";
+};
+export type InterpreterToolOutput = {
+  result: InterpreterToolOutputImage | string;
+  stdout: string;
+  stderr: string;
+};
+
+export type AzureDynamicSessionToolParams = {
+  code?: string;
+  metadata?: ToolMetadata<InterpreterParameter>;
+  /**
+   * The endpoint of the pool management service.
+   */
+  poolManagementEndpoint: string;
+
+  /**
+   * The session ID. If not provided, a new session ID will be generated.
+   */
+  sessionId?: string;
+
+  /**
+   * A function that returns the access token to be used for authentication.
+   * If not provided, a default implementation that uses the DefaultAzureCredential
+   * will be used.
+   *
+   * @returns The access token to be used for authentication.
+   */
+  azureADTokenProvider?: () => Promise<string>;
+};
+
+export interface RemoteFileMetadata {
+  /**
+   * The filename of the file.
+   */
+  filename: string;
+
+  /**
+   * The size of the file in bytes.
+   */
+  size: number;
+
+  /**
+   * The last modified time of the file.
+   */
+  last_modified_time: string;
+
+  /**
+   * The identifier of the file.
+   */
+  $id: string;
+}
+
+type DownloadFileMetadata = {
+  /**
+   * The path to download the file from, relative to `/mnt/data`.
+   * @example "file.txt"
+   * @example "folder/file.txt"
+   */
+  remoteFilename: string;
+
+  /**
+   * The path to save the downloaded file to.
+   * If not provided, the file is returned as a ReadableStream.
+   * @example "/path/to/file.txt"
+   */
+  localFilename?: string;
+};
+
+type UploadFileMetadata = {
+  /**
+   * The data to upload
+   */
+  data: Blob;
+
+  /**
+   * The path to the local file to upload
+   * @example "file.txt"
+   * @example "folder/file.txt"
+   */
+  remoteFilename: string;
+};
+
+let _userAgent = "";
+
+/**
+ * A utility function to generate the user agent in the format:
+ *
+ * `llamaIndex-azure-dynamic-sessions (Language=TypeScript; node.js/v14.17.0; darwin/x64)`
+ * @returns The user agent string.
+ */
+async function getuserAgentSuffix(): Promise<string> {
+  try {
+    //@ts-ignore
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
+
+    if (!_userAgent) {
+      const data = await fs.readFile(
+        path.join(__dirname, "..", "package.json"),
+        "utf8",
+      );
+      const json = await JSON.parse(data.toString());
+      _userAgent = `${json.name}/${json.version}`;
+    }
+  } catch (e) {
+    _userAgent = `llamaIndex-azure-dynamic-sessions`;
+  }
+  return `${_userAgent} (Language=TypeScript; node.js/${process.version}; ${process.platform}; ${process.arch})`;
+}
+
+function getAzureADTokenProvider() {
+  return getBearerTokenProvider(
+    new DefaultAzureCredential(),
+    "https://dynamicsessions.io/.default",
+  );
+}
+
+const DEFAULT_META_DATA: ToolMetadata = {
+  name: "code_interpreter",
+  description:
+    "A Python shell. Use this to execute python commands " +
+    "when you need to perform calculations or computations. " +
+    "Input should be a valid python command. " +
+    "Returns the result, stdout, and stderr. ",
+  parameters: {
+    type: "object",
+    properties: {
+      code: {
+        type: "string",
+        description: "The Python code to execute",
+      },
+    },
+    required: ["code"],
+  },
+};
+
+/**
+ * Azure Code Interpreter tool: A tool that allows you to interact with a dynamic session on Azure.
+ */
+export class AzureDynamicSessionTool
+  implements BaseTool<AzureDynamicSessionToolParams>
+{
+  private readonly outputDir = path.normalize("tool-output");
+
+  /**
+   * The metadata for the tool.
+   */
+  metadata: ToolMetadata;
+
+  /**
+   * The session ID to use for the session pool. Defaults to a random UUID.
+   */
+  private sessionId: string;
+
+  /**
+   * The endpoint of the Azure pool management service.
+   * This is where the tool will send requests to interact with the session pool.
+   * If not provided, the tool will use the value of the `AZURE_CONTAINER_APP_SESSION_POOL_MANAGEMENT_ENDPOINT` environment variable.
+   */
+  private poolManagementEndpoint: string;
+
+  /**
+   * A function that returns the access token to use for the session pool.
+   */
+  private azureADTokenProvider: () => Promise<string>;
+
+  constructor(params?: AzureDynamicSessionToolParams) {
+    this.metadata = params?.metadata || DEFAULT_META_DATA;
+    this.sessionId = params?.sessionId || randomUUID();
+    this.poolManagementEndpoint =
+      params?.poolManagementEndpoint ||
+      (getEnv("AZURE_CONTAINER_APP_SESSION_POOL_MANAGEMENT_ENDPOINT") ?? "");
+    this.azureADTokenProvider =
+      params?.azureADTokenProvider ?? getAzureADTokenProvider();
+
+    if (!this.poolManagementEndpoint) {
+      throw new Error(
+        "AZURE_CONTAINER_APP_SESSION_POOL_MANAGEMENT_ENDPOINT must be defined.",
+      );
+    }
+  }
+
+  _buildUrl(path: string) {
+    let url = `${this.poolManagementEndpoint}${
+      this.poolManagementEndpoint.endsWith("/") ? "" : "/"
+    }${path}`;
+    url += url.includes("?") ? "&" : "?";
+    url += `identifier=${encodeURIComponent(this.sessionId)}`;
+    url += `&api-version=2024-02-02-preview`;
+    return url;
+  }
+
+  /**
+   * Upload a file to the session under the path `/mnt/data`.
+   * @param params.data The data to upload
+   * @param params.remoteFilename The path to the local file to upload
+   * @returns The remote file object. The list of metadatas for the uploaded files.
+   */
+  async uploadFile(params: UploadFileMetadata): Promise<RemoteFileMetadata> {
+    const token = await this.azureADTokenProvider();
+    const apiUrl = this._buildUrl("files/upload");
+    const headers = {
+      Authorization: `Bearer ${token}`,
+      "User-Agent": await getuserAgentSuffix(),
+    };
+    const body = new FormData();
+    body.append("file", params.data, params.remoteFilename);
+
+    try {
+      const response = await fetch(apiUrl, {
+        method: "POST",
+        headers,
+        body,
+      });
+      const json: any = await response.json();
+      return json.value[0].properties as RemoteFileMetadata;
+    } catch (error) {
+      throw new Error(
+        `[AzureDynamicSessionTool.downloadFile] HTTP error! status: ${error}`,
+      );
+    }
+  }
+
+  /**
+   * Download a file from the session back to your local environment.
+   * @param params.remoteFilename The path to download the file from, relative to `/mnt/data`.
+   * @param params.localFilename The path to save the downloaded file to. If not provided, the file is returned as a BufferedReader.
+   * @returns The file as a ReadableStream if no localFilename is provided. Otherwise, the file is saved to the localFilename.
+   */
+  async downloadFile(
+    params: DownloadFileMetadata,
+  ): Promise<ReadableStream | void> {
+    const token = await this.azureADTokenProvider();
+    const apiUrl = this._buildUrl(`files/content/${params.remoteFilename}`);
+    const headers = {
+      Authorization: `Bearer ${token}`,
+      "User-Agent": await getuserAgentSuffix(),
+    };
+    try {
+      const response = await fetch(apiUrl, {
+        method: "GET",
+        headers,
+      });
+
+      if (response.body) {
+        // if localFilename is provided, save the file to the localFilename
+        if (params.localFilename) {
+          const writer = createWriteStream(path.resolve(params.localFilename));
+          const blob = await response.blob();
+          Readable.from(blob.stream()).pipe(writer);
+          return;
+        }
+
+        // if localFilename is not provided, return the file as a ReadableStream
+        return response.body as ReadableStream;
+      } else {
+        throw new Error(
+          `[AzureDynamicSessionTool.downloadFile] HTTP error! status: ${response.status}`,
+        );
+      }
+    } catch (error) {
+      throw new Error(
+        `[AzureDynamicSessionTool.downloadFile] HTTP error! status: ${error}`,
+      );
+    }
+  }
+
+  /**
+   * List the files in the session.
+   * @returns The metadata for the files in the session
+   */
+  async listFiles(): Promise<RemoteFileMetadata[]> {
+    const token = await this.azureADTokenProvider();
+    const apiUrl = this._buildUrl("files");
+    const headers = {
+      Authorization: `Bearer ${token}`,
+      "User-Agent": await getuserAgentSuffix(),
+    };
+
+    try {
+      const response = await fetch(apiUrl, {
+        method: "GET",
+        headers,
+      });
+      const json: any = await response.json();
+      const list = json.value.map(
+        (x: { properties: RemoteFileMetadata }) => x.properties,
+      );
+      return list as RemoteFileMetadata[];
+    } catch (error: unknown) {
+      throw new Error(
+        `[AzureDynamicSessionTool.listFiles] HTTP error! status: ${error}`,
+      );
+    }
+  }
+
+  /**
+   * This tool is used to execute python commands when you need to perform calculations or computations in a Session. Input should be a valid python command. The tool returns the result, stdout, and stderr.
+   * @param code Python code to be executed generated by llm.
+   * @returns The result, stdout, and stderr.
+   */
+  async call({
+    code,
+  }: Pick<
+    AzureDynamicSessionToolParams,
+    "code"
+  >): Promise<InterpreterToolOutput> {
+    const token = await this.azureADTokenProvider();
+    const apiUrl = this._buildUrl("python/execute");
+    const headers = {
+      Authorization: `Bearer ${token}`,
+      "Content-Type": "application/json",
+      "User-Agent": await getuserAgentSuffix(),
+    };
+    const payload = {
+      properties: {
+        identifier: this.sessionId,
+        codeInputType: "inline",
+        executionType: "synchronous",
+        pythonCode: code,
+      },
+    };
+
+    console.log("payload", { payload });
+
+    try {
+      const response = await fetch(apiUrl, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(payload),
+      });
+
+      const output = (await response.json()) as InterpreterToolOutput;
+      console.log({ output });
+
+      if (typeof output.result !== "string") {
+        const result = output.result as InterpreterToolOutputImage;
+
+        console.log("result", { result });
+
+        if (result.type === "image") {
+          const { outputPath, filename } = await this.saveToDisk(
+            (output.result as InterpreterToolOutputImage).base64_data,
+            result.format,
+          );
+          output.result = `${outputPath}/${filename}`;
+        }
+      }
+
+      return output;
+    } catch (error) {
+      return {
+        result: "",
+        stdout: "",
+        stderr: "Error: Failed to execute Python code. " + error,
+      };
+    }
+  }
+
+  /**
+   * Saves a base64 encoded file to the disk.
+   * @param base64Data The base64 encoded data to save.
+   * @param ext The file extension.
+   * @returns The path and filename to the saved file.
+   */
+  private async saveToDisk(
+    base64Data: string,
+    ext: string,
+  ): Promise<{
+    outputPath: string;
+    filename: string;
+  }> {
+    try {
+      const filename = `${randomUUID()}.${ext}`;
+      const buffer = Buffer.from(base64Data, "base64");
+      const outputPath = await this.getOutputPath(filename);
+      await fs.writeFile(outputPath, buffer);
+      console.log(
+        `[AzureDynamicSessionTool.saveToDisk] Saved file to ${outputPath}`,
+      );
+      return {
+        outputPath,
+        filename,
+      };
+    } catch (error) {
+      console.error(
+        `[AzureDynamicSessionTool.saveToDisk] Error saving file to disk: ${error}`,
+      );
+      return {
+        outputPath: "",
+        filename: "",
+      };
+    }
+  }
+
+  /**
+   * Get the output path for the file.
+   * @param filename The filename to save the file as.
+   * @returns The output path for the file.
+   */
+  private async getOutputPath(filename: string) {
+    if ((await this.exists(this.outputDir)) === false) {
+      try {
+        await fs.mkdir(this.outputDir, { recursive: true });
+        console.log(
+          "[AzureDynamicSessionTool.getOutputPath] Created output directory:",
+          this.outputDir,
+        );
+      } catch (e) {
+        throw new Error(
+          `[AzureDynamicSessionTool.getOutputPath] Failed to create output directory: ${this.outputDir}`,
+        );
+      }
+    }
+    return path.join(this.outputDir, filename);
+  }
+
+  /**
+   * Check if a file exists.
+   * @param file The file to check.
+   * @returns True if the file exists, false otherwise.
+   */
+  private async exists(file: string) {
+    try {
+      await fs.lstat(file);
+      console.log(`[AzureDynamicSessionTool.exists] File exists: ${file}`);
+      return true;
+    } catch {
+      console.log(
+        `[AzureDynamicSessionTool.exists] File does not exist: ${file}`,
+      );
+      return false;
+    }
+  }
+}

packages/llamaindex/tests/llm/index.test.ts

@@ -62,5 +62,51 @@ describe("Anthropic llm", () => {
         role: "user",
       },
     ]);
+
+    expect(
+      anthropic.formatMessages([
+        {
+          content: "You are a helpful assistant.",
+          role: "assistant",
+        },
+        {
+          content: [
+            {
+              text: "What do you see in the image?",
+              type: "text",
+            },
+            {
+              type: "image_url",
+              image_url: {
+                url: `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAQDAwQDAwQEAwQFBAQFBgoHBgYGBg0JCggKDw0QEA8NDw4RExgUERIXEg4PFRwVFxkZGxsbEBQdHx0aHxgaGxr/2wBDAQQFBQYFBgwHBwwaEQ8RGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhr/wAARCAAgACADASIAAhEBAxEB/8QAGAABAAMBAAAAAAAAAAAAAAAACAQHCQb/xAAvEAABAgUCBAUDBQEAAAAAAAACAQMEBQYHERIhAAgTYSIxMkJxI2KCFBVBUVKh/8QAGAEAAwEBAAAAAAAAAAAAAAAAAwQFAQL/xAAnEQABBAECAwkAAAAAAAAAAAACAQMEEQAFMiExYRITFCJBcXKBof/aAAwDAQACEQMRAD8Aufmb5mnbWREFRdvIMZ3cWcaBh2NHUGEFwtIKQp63CX0h+S7YQgRzGSq6kgqGAS8NQRc6fmkIMWwSxJEyP+m0bwggQr5iIom6KnnxXty61jK+uJUVUxzxm/M5g5EASr6G9WGwTsIIIp2FOHJfi0kyvzS9Cv0zGwEF+2whOAUY4a6mnm2lREURLPoTggNG5tS6xpmOT4GQptwNUZc6sbexzcZRVSTKTOgudMPEL0j7E2uQNOxIqcaYcqXNaxe2HKnauBiAraDZ6n0k0tTBpPNwE9pptqDP3DtlBC1Q8qNw5K4AwLEunYkWMwcYg6fnqoH/ADPHA2/qeZWquhJJ3pODmEhmg/qGl2XAloebL5HWK/K8dOMOM7xVPfJrMhmQiq0SFXOlyPc+jIq3lwakpeYNq27K491kfvbzls07ECiSdlThhWKvj1LLx0VVLWGqSBuFJ1jc3WBEUb8K4TUieHz3xni7ea3lSZvZDhUVImxAVtBso39VdLUe0nk2a+0030n+K7YUc95/J66tRIp3SVXUpGyUI7wvPxDBoJ/UaLIuIqtuInRwiiqp4z3XbBYr3cGp9P30zJXiSjk1HLsqdIvxvzV1q8ZtB3ppa5bkwZkDz7LsF09Qxgi0Roa6UUU1LnxYH5JP74D1LUjNrkXigabc6kZM5vPFZi3NPi3dVXnFT+EQUM17IvEi1tL1xUkcEHb+lo6duvRUO644wwSDpaPWgG7sAApIKqqqm4jvxo1yvcrjdoTiqtrQ2I+u5nr19ItbUA2a5IAX3GvuP8U2ypMS5pSwFC5peTtM0lnSkMWVVUJb48a+8//Z`,
+              },
+            },
+          ],
+          role: "user",
+        },
+      ]),
+    ).toEqual([
+      {
+        content: "You are a helpful assistant.",
+        role: "assistant",
+      },
+      {
+        content: [
+          {
+            text: "What do you see in the image?",
+            type: "text",
+          },
+          {
+            source: {
+              data: "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAQDAwQDAwQEAwQFBAQFBgoHBgYGBg0JCggKDw0QEA8NDw4RExgUERIXEg4PFRwVFxkZGxsbEBQdHx0aHxgaGxr/2wBDAQQFBQYFBgwHBwwaEQ8RGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhr/wAARCAAgACADASIAAhEBAxEB/8QAGAABAAMBAAAAAAAAAAAAAAAACAQHCQb/xAAvEAABAgUCBAUDBQEAAAAAAAACAQMEBQYHERIhAAgTYSIxMkJxI2KCFBVBUVKh/8QAGAEAAwEBAAAAAAAAAAAAAAAAAwQFAQL/xAAnEQABBAECAwkAAAAAAAAAAAACAQMEEQAFMiExYRITFCJBcXKBof/aAAwDAQACEQMRAD8Aufmb5mnbWREFRdvIMZ3cWcaBh2NHUGEFwtIKQp63CX0h+S7YQgRzGSq6kgqGAS8NQRc6fmkIMWwSxJEyP+m0bwggQr5iIom6KnnxXty61jK+uJUVUxzxm/M5g5EASr6G9WGwTsIIIp2FOHJfi0kyvzS9Cv0zGwEF+2whOAUY4a6mnm2lREURLPoTggNG5tS6xpmOT4GQptwNUZc6sbexzcZRVSTKTOgudMPEL0j7E2uQNOxIqcaYcqXNaxe2HKnauBiAraDZ6n0k0tTBpPNwE9pptqDP3DtlBC1Q8qNw5K4AwLEunYkWMwcYg6fnqoH/ADPHA2/qeZWquhJJ3pODmEhmg/qGl2XAloebL5HWK/K8dOMOM7xVPfJrMhmQiq0SFXOlyPc+jIq3lwakpeYNq27K491kfvbzls07ECiSdlThhWKvj1LLx0VVLWGqSBuFJ1jc3WBEUb8K4TUieHz3xni7ea3lSZvZDhUVImxAVtBso39VdLUe0nk2a+0030n+K7YUc95/J66tRIp3SVXUpGyUI7wvPxDBoJ/UaLIuIqtuInRwiiqp4z3XbBYr3cGp9P30zJXiSjk1HLsqdIvxvzV1q8ZtB3ppa5bkwZkDz7LsF09Qxgi0Roa6UUU1LnxYH5JP74D1LUjNrkXigabc6kZM5vPFZi3NPi3dVXnFT+EQUM17IvEi1tL1xUkcEHb+lo6duvRUO644wwSDpaPWgG7sAApIKqqqm4jvxo1yvcrjdoTiqtrQ2I+u5nr19ItbUA2a5IAX3GvuP8U2ypMS5pSwFC5peTtM0lnSkMWVVUJb48a+8//Z",
+              media_type: "image/jpeg",
+              type: "base64",
+            },
+            type: "image",
+          },
+        ],
+        role: "user",
+      },
+    ]);
   });
 });