Release 0.10.3 (#1898)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>

.prettierignore

@@ -7,3 +7,4 @@ dist/
 .source/
 # prttier doesn't support mdx3 we are using
 *.mdx
+packages/server/server/

apps/next/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/doc
 
+## 0.2.15
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - llamaindex@0.10.3
+  - @llamaindex/openai@0.3.5
+  - @llamaindex/cloud@4.0.4
+  - @llamaindex/node-parser@2.0.3
+  - @llamaindex/readers@3.1.1
+  - @llamaindex/workflow@1.0.4
+
 ## 0.2.14
 
 ### Patch Changes

apps/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/doc",
-  "version": "0.2.14",
+  "version": "0.2.15",
   "private": true,
   "scripts": {
     "postinstall": "fumadocs-mdx",

apps/next/src/components/demo/lazy.ts

@@ -11,8 +11,3 @@ export const CodeNodeParserDemo = dynamic(() =>
     (mod) => mod.CodeNodeParserDemo,
   ),
 );
-export const WorkflowStreamingDemo = dynamic(() =>
-  import("@/components/demo/workflow-streaming-ui").then(
-    (mod) => mod.WorkflowStreamingDemo,
-  ),
-);

apps/next/src/components/demo/workflow-streaming-ui.tsx

@@ -1,152 +0,0 @@
-"use client";
-import FlowInput from "@/components/flow-input";
-import { Button } from "@/components/ui/button";
-import {
-  StartEvent,
-  StopEvent,
-  Workflow,
-  WorkflowEvent,
-} from "@llamaindex/workflow";
-import { ReactNode, startTransition, useState } from "react";
-import { StickToBottom, useStickToBottomContext } from "use-stick-to-bottom";
-
-class ComputeEvent extends WorkflowEvent<number> {
-  constructor(data: number) {
-    super(data);
-  }
-}
-
-class ComputeResultEvent extends WorkflowEvent<number> {
-  constructor(data: number) {
-    super(data);
-  }
-}
-
-type ContextData = {
-  sum: number;
-};
-
-const workflow = new Workflow<ContextData, number, number>();
-
-const max = 1000;
-const min = 100;
-
-workflow.addStep(
-  {
-    inputs: [StartEvent<number>],
-    outputs: [StopEvent<number>],
-  },
-  async (context, event) => {
-    const total = event.data;
-    for (let i = 0; i < total; i++) {
-      context.sendEvent(new ComputeEvent(i));
-    }
-    console.log("waiting");
-    const computeResults = await Promise.all(
-      Array.from({ length: total }).map(() =>
-        context.requireEvent(ComputeResultEvent),
-      ),
-    );
-    context.data.sum = computeResults.reduce(
-      (acc, result) => acc + result.data,
-      0,
-    );
-    console.log("stop");
-    return new StopEvent(context.data.sum);
-  },
-);
-
-workflow.addStep(
-  {
-    inputs: [ComputeEvent],
-    outputs: [ComputeResultEvent],
-  },
-  async (context, event) => {
-    await new Promise((resolve) =>
-      setTimeout(resolve, Math.floor(Math.random() * (max - min + 1) + min)),
-    );
-    return new ComputeResultEvent(event.data);
-  },
-);
-
-function ScrollToBottom() {
-  const { isAtBottom, scrollToBottom } = useStickToBottomContext();
-
-  return (
-    !isAtBottom && (
-      <button
-        className="i-ph-arrow-circle-down-fill absolute bottom-0 left-[50%] translate-x-[-50%] rounded-lg text-4xl"
-        onClick={() => scrollToBottom()}
-      />
-    )
-  );
-}
-
-export function WorkflowStreamingDemo() {
-  const [ui, setUI] = useState<ReactNode[]>([
-    <div key={0} className="bg-gray-100 dark:bg-gray-800">
-      Waiting for workflow to start
-    </div>,
-  ]);
-  const [total, setTotal] = useState<number>(10);
-
-  return (
-    <div className="flex w-full flex-col items-start gap-2">
-      <div className="flex flex-row items-center justify-center">
-        <div className="mr-2 text-lg">Compute total</div>{" "}
-        <FlowInput value={total} onChange={(value) => setTotal(value)} />
-      </div>
-      <Button
-        onClick={async () => {
-          startTransition(() => {
-            setUI([]);
-          });
-          const context = workflow.run(total, {
-            sum: 0,
-          });
-          let i = 0;
-          for await (const event of context) {
-            console.log(event);
-            if (event instanceof ComputeEvent) {
-              setUI((ui) => [
-                ...ui,
-                <div key={i++} className="bg-yellow-100 dark:bg-yellow-800">
-                  Computing task id: {event.data}
-                </div>,
-              ]);
-            } else if (event instanceof ComputeResultEvent) {
-              setUI((ui) => [
-                ...ui,
-                <div key={i++} className="bg-green-100 dark:bg-green-800">
-                  Computed task id: {event.data}
-                </div>,
-              ]);
-            } else if (event instanceof StartEvent) {
-              setUI((ui) => [
-                ...ui,
-                <div key={i++} className="bg-blue-100 dark:bg-blue-800">
-                  Started workflow with total {event.data}
-                </div>,
-              ]);
-            } else if (event instanceof StopEvent) {
-              setUI((ui) => [
-                ...ui,
-                <div key={i++} className="bg-red-100 dark:bg-red-800">
-                  Workflow stopped
-                </div>,
-              ]);
-            }
-          }
-        }}
-      >
-        Start Workflow
-      </Button>
-      <StickToBottom className="flex max-h-96 w-full flex-col gap-2 overflow-y-auto rounded-lg border border-gray-200 p-2">
-        <StickToBottom.Content className="flex flex-col gap-2">
-          {ui}
-        </StickToBottom.Content>
-        <ScrollToBottom />
-      </StickToBottom>
-    </div>
-  );
-}

apps/next/src/content/docs/llamaindex/modules/agents/workflows.mdx

@@ -2,149 +2,260 @@
 title: Workflows
 ---
 
-A `Workflow` in LlamaIndexTS is an event-driven abstraction used to chain together several events. Workflows are made up of `steps`, with each step responsible for handling certain event types and emitting new events.
-
-Workflows in LlamaIndexTS work by defining step functions that handle specific event types and emit new events.
-
-When a step function is added to a workflow, you need to specify the input and optionally the output event types (used for validation). The specification of the input events ensures each step only runs when an accepted event is ready.
-
-You can create a `Workflow` to do anything! Build an agent, a RAG flow, an extraction flow, or anything else you want.
+A `Workflow` in LlamaIndex is a lightweight, event-driven abstraction used to chain together several events. Workflows are made up of `handlers`, with each one responsible for processing specific event types and emitting new events.
 
+Workflows are designed to be flexible and can be used to build agents, RAG flows, extraction flows, or anything else you want to implement.
 
 ```package-install
-npm i @llamaindex/workflow
+npm i @llama-flow/core @llamaindex/openai
 ```
 
 ## Getting Started
 
-As an illustrative example, let's consider a naive workflow where a joke is generated and then critiqued.
+Let's explore a simple workflow example where a joke is generated and then critiqued and iterated on:
 
-<include cwd>../../examples/workflow/joke.ts</include>
+```typescript
+import { OpenAI } from "@llamaindex/openai";
+import { createWorkflow, workflowEvent } from "@llama-flow/core";
+import { withStore } from "@llama-flow/core/middleware/store";
 
-There's a few moving pieces here, so let's go through this piece by piece.
+// Create LLM instance
+const llm = new OpenAI({ model: "gpt-4.1-mini", apiKey: "..."});
+
+// Define our workflow events
+const startEvent = workflowEvent<string>(); // Input topic for joke
+const jokeEvent = workflowEvent<{ joke: string }>(); // Intermediate joke
+const critiqueEvent = workflowEvent<{ joke: string, critique: string }>(); // Intermediate critique
+const resultEvent = workflowEvent<{ joke: string, critique: string }>(); // Final joke + critique
+
+// Create our workflow
+const jokeFlow = withStore(
+    () => ({
+        numIterations: 0,
+        maxIterations: 3,
+    }),
+    createWorkflow()
+);
+
+// Define handlers for each step
+jokeFlow.handle([startEvent], async (event) => {
+  // Prompt the LLM to write a joke
+  const prompt = `Write your best joke about ${event.data}. Write the joke between <joke> and </joke> tags.`;
+  const response = await llm.complete({ prompt });
+
+  // Parse the joke from the response
+  const joke = response.text.match(/<joke>([\s\S]*?)<\/joke>/)?.[1]?.trim() ?? response.text;
+  return jokeEvent.with({ joke: joke });
+});
+
+jokeFlow.handle([jokeEvent], async (event) => {
+  // Prompt the LLM to critique the joke
+  const prompt = `Give a thorough critique of the following joke. If the joke needs improvement, put "IMPROVE" somewhere in the critique: ${event.data.joke}`;
+  const response = await llm.complete({ prompt });
+
+  // If the critique includes "IMPROVE", keep iterating, else, return the result
+  if (response.text.includes("IMPROVE")) {
+    return critiqueEvent.with({ joke: event.data.joke, critique: response.text });
+  }
+
+  return resultEvent.with({ joke: event.data.joke, critique: response.text });
+});
+
+jokeFlow.handle([critiqueEvent], async (event) => {
+  // Keep track of the number of iterations
+  const store = jokeFlow.getStore();
+  store.numIterations++;
+
+  // Write a new joke based on the previous joke and critique
+  const prompt = `Write a new joke based on the following critique and the original joke. Write the joke between <joke> and </joke> tags.\n\nJoke: ${event.data.joke}\n\nCritique: ${event.data.critique}`;
+  const response = await llm.complete({ prompt });
+  
+  // Parse the joke from the response
+  const joke = response.text.match(/<joke>([\s\S]*?)<\/joke>/)?.[1]?.trim() ?? response.text;
+  
+  // If we've done less than the max number of iterations, keep iterating
+  // else, return the result
+  if (store.numIterations < store.maxIterations) {
+    return jokeEvent.with({ joke: joke });
+  }
+
+  return resultEvent.with({ joke: joke, critique: event.data.critique });
+});
+
+// Usage
+async function main() {
+  const { stream, sendEvent } = jokeFlow.createContext();
+  sendEvent(startEvent.with("pirates"));
+
+  let result: { joke: string, critique: string } | undefined;
+
+  for await (const event of stream) {
+    // console.log(event.data);  optionally log the event data
+    if (resultEvent.include(event)) {
+      result = event.data;
+      break; // Stop when we get the final result
+    }
+  }
+  
+  console.log(result);
+}
+
+main().catch(console.error);
+```
+
+There are a few moving pieces here, so let's go through this step by step.
 
 ### Defining Workflow Events
 
 ```typescript
-export class JokeEvent extends WorkflowEvent<{ joke: string }> {}
+const startEvent = workflowEvent<string>(); // Input topic for joke
+const jokeEvent = workflowEvent<{ joke: string }>(); // Intermediate joke
+const critiqueEvent = workflowEvent<{ joke: string, critique: string }>(); // Intermediate critique
+const resultEvent = workflowEvent<{ joke: string, critique: string }>(); // Final joke + critique
 ```
 
-Events are user-defined classes that extend `WorkflowEvent` and contain arbitrary data provided as template argument. In this case, our workflow relies on a single user-defined event, the `JokeEvent` with a `joke` attribute of type `string`.
+Events are defined using the `workflowEvent` function and contain arbitrary data provided as a generic type. In this example, we have four events:
+- `startEvent`: Takes a string input (the joke topic)
+- `jokeEvent`: Contains an object with a joke property
+- `critiqueEvent`: Contains both the joke and its critique, used for the feedback loop
+- `resultEvent`: Contains the final joke and critique after any iterations
 
-### Setting up the Workflow Class
+### Setting up the Workflow with Store Middleware
 
 ```typescript
-const llm = new OpenAI();
-...
-const jokeFlow = new Workflow<unknown, string, string>();
-```
-
-Our workflow is implemented by initiating the `Workflow` class with three generic types: the context type (unknown), input type (string), and output type (string). The context type is `unknown`, as we're not using a shared context in this example.
-
-For simplicity, we created an `OpenAI` llm instance that we're using for inference in our workflow.
-
-### Workflow Entry Points
-
-```typescript
-const generateJoke = async (_: unknown, ev: StartEvent<string>) => {
-  const prompt = `Write your best joke about ${ev.data}.`;
-  const response = await llm.complete({ prompt });
-  return new JokeEvent({ joke: response.text });
-};
-```
-
-Here, we come to the entry-point of our workflow. While events are user-defined, there are two special-case events, the `StartEvent` and the `StopEvent`. These events are predefined, but we can specify the payload type using generic types. We're using `StartEvent<string>` to indicate that we're going to send an input of type string.
-
-To add this step to the workflow, we use the `addStep` method with an object specifying the input and output event types:
-
-```typescript
-jokeFlow.addStep(
-  {
-    inputs: [StartEvent<string>],
-    outputs: [JokeEvent],
-  },
-  generateJoke
+const jokeFlow = withStore(
+    () => ({
+        numIterations: 0,
+        maxIterations: 3,
+    }),
+    createWorkflow()
 );
 ```
 
-### Workflow Exit Points
+Our workflow is implemented using the `createWorkflow()` function, enhanced with the `withStore` middleware. The store provides shared state across all handlers, which in this case tracks:
+- `numIterations`: Counts how many iterations of joke improvement we've done
+- `maxIterations`: Sets a limit to prevent infinite loops
+
+This store will be accesible within workflows by using the `jokeFlow.getStore()` function.
+
+### Adding Handlers with Loops
+
+We have three key handlers in our workflow:
+
+1. The first handler processes the `startEvent`, generates an initial joke, and emits a `jokeEvent`:
 
 ```typescript
-const critiqueJoke = async (_: unknown, ev: JokeEvent) => {
-  const prompt = `Give a thorough critique of the following joke: ${ev.data.joke}`;
+jokeFlow.handle([startEvent], async (event) => {
+  // Prompt the LLM to write a joke
+  const prompt = `Write your best joke about ${event.data}. Write the joke between <joke> and </joke> tags.`;
   const response = await llm.complete({ prompt });
-  return new StopEvent(response.text);
-};
+
+  // Parse the joke from the response
+  const joke = response.text.match(/<joke>([\s\S]*?)<\/joke>/)?.[1]?.trim() ?? response.text;
+  return jokeEvent.with({ joke: joke });
+});
 ```
 
-Here, we have our second and last step in the workflow. We know it's the last step because the special `StopEvent` is returned. When the workflow encounters a returned `StopEvent`, it immediately stops the workflow and returns the result. Note that we're using the generic type `StopEvent<string>` to indicate that we're returning a string.
-
-Add this step to the workflow:
+2. The second handler handles the `jokeEvent`, critiques the joke, and either:
+   - Emits a `critiqueEvent` if the joke needs improvement
+   - Emits a `resultEvent` if the joke is good enough
 
 ```typescript
-jokeFlow.addStep(
-  {
-    inputs: [JokeEvent],
-    outputs: [StopEvent<string>],
-  },
-  critiqueJoke
-);
+jokeFlow.handle([jokeEvent], async (event) => {
+  // Prompt the LLM to critique the joke
+  const prompt = `Give a thorough critique of the following joke. If the joke needs improvement, put "IMPROVE" somewhere in the critique: ${event.data.joke}`;
+  const response = await llm.complete({ prompt });
+
+  // If the critique includes "IMPROVE", keep iterating, else, return the result
+  if (response.text.includes("IMPROVE")) {
+    return critiqueEvent.with({ joke: event.data.joke, critique: response.text });
+  }
+
+  return resultEvent.with({ joke: event.data.joke, critique: response.text });
+});
+```
+
+3. The third handler processes the `critiqueEvent`, generates an improved joke based on the critique, and either:
+   - Loops back to the joke evaluation (if under the iteration limit)
+   - Emits the final `resultEvent` (if iteration limit reached)
+
+```typescript
+jokeFlow.handle([critiqueEvent], async (event) => {
+  // Keep track of the number of iterations
+  const store = jokeFlow.getStore();
+  store.numIterations++;
+
+  // Write a new joke based on the previous joke and critique
+  const prompt = `Write a new joke based on the following critique and the original joke. Write the joke between <joke> and </joke> tags.\n\nJoke: ${event.data.joke}\n\nCritique: ${event.data.critique}`;
+  const response = await llm.complete({ prompt });
+  
+  // Parse the joke from the response
+  const joke = response.text.match(/<joke>([\s\S]*?)<\/joke>/)?.[1]?.trim() ?? response.text;
+  
+  // If we've done less than the max number of iterations, keep iterating
+  // else, return the result
+  if (store.numIterations < store.maxIterations) {
+    return jokeEvent.with({ joke: joke });
+  }
+
+  return resultEvent.with({ joke: joke, critique: event.data.critique });
+});
 ```
 
 ### Running the Workflow
 
 ```typescript
-const result = await jokeFlow.run("pirates");
-console.log(result.data.result);
-```
+async function main() {
+  const { stream, sendEvent } = jokeFlow.createContext();
+  sendEvent(startEvent.with("pirates"));
 
-Lastly, we run the workflow. The `.run()` method is async, so we use await here to wait for the result.
+  let result: { joke: string, critique: string } | undefined;
 
-## Working with Shared Context/State
-
-Optionally, you can choose to use a shared context between steps by specifying a context type when creating the workflow. Here's an example where multiple steps access a shared state:
-
-```typescript
-import { HandlerContext } from "llamaindex";
-
-type MyContextData = {
-  query: string;
-  intermediateResults: any[];
+  for await (const event of stream) {
+    // console.log(event.data);  optionally log the event data
+    if (resultEvent.include(event)) {
+      result = event.data;
+      break; // Stop when we get the final result
+    }
+  }
+  
+  console.log(result);
 }
-
-const query = async (context: HandlerContext<MyContextData>, ev: MyEvent) => {
-  // get the query from the context
-  const query = context.data.query;
-  // do something with context and event
-  const val = ...
-  // store in context
-  context.data.intermediateResults.push(val);
-
-  return new StopEvent({ result });
-};
 ```
 
-## Waiting for Multiple Events
+To run the workflow, we:
+1. Create a workflow context with `createContext()`
+2. Trigger the initial event with `sendEvent()`
+3. Listen to the event stream and process events as they arrive
+4. Use `include()` to check if an event is of a specific type
+5. Break the loop when we receive our final result
 
-The context does more than just hold data, it also provides utilities to buffer and wait for multiple events.
+### Using Stream Utilities
 
-For example, you might have a step that waits for a query and retrieved nodes before synthesizing a response:
+Workflows provide utility functions to make working with event streams easier:
 
 ```typescript
-const synthesize = async (context: Context, ev1: QueryEvent, ev2: RetrieveEvent) => {
-  const subPrompts = [`Answer this query using the context provided: ${ev1.data.query}`, `Context: ${ev2.data.context}`];
-  const prompt = subPrompts.join("\n");
-  const response = await llm.complete({ prompt });
-  return new StopEvent({ result: response.text });
-};
+import { collect } from "@llama-flow/core/stream/consumer";
+import { until } from "@llama-flow/core/stream/until";
+
+// Create a workflow context and send the initial event
+const { stream, sendEvent } = jokeFlow.createContext();
+sendEvent(startEvent.with("pirates"));
+
+// Collect all events until we get a resultEvent
+const allEvents = await collect(until(stream, resultEvent));
+
+// The last event will be the resultEvent
+const finalEvent = allEvents[allEvents.length - 1];
+console.log(finalEvent.data); // Output the joke and critique
 ```
 
-Passing multiple events, we can buffer and wait for ALL expected events to arrive. The receiving step function will only be called once all events have arrived.
+The stream utilities make it easier to work with the asynchronous event flow. In this example, we use:
+- `collect`: Aggregates all events into an array
+- `until`: Creates a stream that emits events until a condition is met (in this case, until a resultEvent is received)
 
-## Manually Triggering Events
+You can combine these utilities with other stream operators like `filter` and `map` to create powerful processing pipelines.
 
-Normally, events are triggered by returning another event during a step. However, events can also be manually dispatched using the `ctx.sendEvent(event)` method within a workflow.
+## Next Steps
 
-## Examples
-
-You can find many useful examples of using workflows in the [examples folder](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/workflow).
+To learn more about workflows, check out [the documentation in the tutorial section](../../../llamaflow).

apps/next/src/deps/type.ts

@@ -1,5 +0,0 @@
-export type {
-  HandlerContext,
-  Workflow,
-  WorkflowContext,
-} from "@llamaindex/workflow";

e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.157
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.0.156
 
 ### Patch Changes

e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.156",
+  "version": "0.0.157",
   "type": "module",
   "private": true,
   "scripts": {

e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.59
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.4
+
 ## 0.0.58
 
 ### Patch Changes

e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.58",
+  "version": "0.0.59",
   "type": "module",
   "scripts": {
     "dev": "vite",
@@ -10,8 +10,8 @@
   },
   "devDependencies": {
     "typescript": "^5.7.3",
-    "vite": "^5.4.16",
-    "vite-plugin-wasm": "^3.3.0"
+    "vite": "^6.3.3",
+    "vite-plugin-wasm": "^3.4.1"
   },
   "dependencies": {
     "@llamaindex/cloud": "workspace:*"

e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/next-agent-test
 
+## 0.1.157
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.1.156
 
 ### Patch Changes

e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.156",
+  "version": "0.1.157",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,12 @@
 # test-edge-runtime
 
+## 0.1.156
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.1.155
 
 ### Patch Changes

e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.155",
+  "version": "0.1.156",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @llamaindex/next-node-runtime
 
+## 0.1.24
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+  - @llamaindex/huggingface@0.1.7
+  - @llamaindex/readers@3.1.1
+
 ## 0.1.23
 
 ### Patch Changes

e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.23",
+  "version": "0.1.24",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/vite-import-llamaindex/CHANGELOG.md

@@ -1,5 +1,12 @@
 # vite-import-llamaindex
 
+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.0.22
 
 ### Patch Changes

e2e/examples/vite-import-llamaindex/package.json

@@ -1,7 +1,7 @@
 {
   "name": "vite-import-llamaindex",
   "private": true,
-  "version": "0.0.22",
+  "version": "0.0.23",
   "type": "module",
   "scripts": {
     "build": "vite build",
@@ -16,7 +16,7 @@
     "@size-limit/preset-big-lib": "^11.1.6",
     "size-limit": "^11.1.6",
     "typescript": "^5.7.3",
-    "vite": "^5.4.16"
+    "vite": "^6.3.3"
   },
   "dependencies": {
     "llamaindex": "workspace:*"

e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.157
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.0.156
 
 ### Patch Changes

e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.156",
+  "version": "0.0.157",
   "type": "module",
   "private": true,
   "scripts": {

examples/CHANGELOG.md

@@ -1,5 +1,63 @@
 # examples
 
+## 0.3.13
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - llamaindex@0.10.3
+  - @llamaindex/anthropic@0.3.4
+  - @llamaindex/google@0.2.5
+  - @llamaindex/openai@0.3.5
+  - @llamaindex/vercel@0.1.3
+  - @llamaindex/cloud@4.0.4
+  - @llamaindex/node-parser@2.0.3
+  - @llamaindex/assemblyai@0.1.2
+  - @llamaindex/clip@0.0.53
+  - @llamaindex/cohere@0.0.17
+  - @llamaindex/deepinfra@0.0.53
+  - @llamaindex/discord@0.1.2
+  - @llamaindex/huggingface@0.1.7
+  - @llamaindex/jinaai@0.0.13
+  - @llamaindex/mistral@0.1.3
+  - @llamaindex/mixedbread@0.0.17
+  - @llamaindex/notion@0.1.2
+  - @llamaindex/ollama@0.1.3
+  - @llamaindex/perplexity@0.0.10
+  - @llamaindex/portkey-ai@0.0.45
+  - @llamaindex/replicate@0.0.45
+  - @llamaindex/astra@0.0.17
+  - @llamaindex/azure@0.1.13
+  - @llamaindex/chroma@0.0.17
+  - @llamaindex/elastic-search@0.1.3
+  - @llamaindex/firestore@1.0.10
+  - @llamaindex/milvus@0.1.12
+  - @llamaindex/mongodb@0.0.18
+  - @llamaindex/pinecone@0.1.3
+  - @llamaindex/postgres@0.0.46
+  - @llamaindex/qdrant@0.1.12
+  - @llamaindex/supabase@0.1.2
+  - @llamaindex/upstash@0.0.17
+  - @llamaindex/weaviate@0.0.17
+  - @llamaindex/voyage-ai@1.0.9
+  - @llamaindex/readers@3.1.1
+  - @llamaindex/tools@0.0.8
+  - @llamaindex/workflow@1.0.4
+  - @llamaindex/deepseek@0.0.13
+  - @llamaindex/fireworks@0.0.13
+  - @llamaindex/groq@0.0.68
+  - @llamaindex/together@0.0.13
+  - @llamaindex/vllm@0.0.39
+
+## 0.3.12
+
+### Patch Changes
+
+- Updated dependencies [82d4b46]
+  - @llamaindex/server@0.1.6
+  - @llamaindex/tools@0.0.7
+
 ## 0.3.11
 
 ### Patch Changes

examples/anthropic/chat-with-file.ts

@@ -0,0 +1,39 @@
+import { Anthropic } from "@llamaindex/anthropic";
+import fs from "fs";
+
+// Note that: Anthropic only supports PDF files for now with limited models
+// See: https://docs.anthropic.com/en/docs/build-with-claude/pdf-support?q=pdf#supported-platforms-and-models
+
+async function main() {
+  if (!process.env.ANTHROPIC_API_KEY) {
+    throw new Error("Please set the ANTHROPIC_API_KEY environment variable.");
+  }
+
+  const llm = new Anthropic({
+    apiKey: process.env.ANTHROPIC_API_KEY,
+    model: "claude-3-7-sonnet",
+  });
+
+  const result = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result.message);
+}
+
+void main().catch(console.error);

examples/gemini/chat.ts

@@ -1,11 +1,12 @@
 import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import fs from "fs";
 
 (async () => {
   if (!process.env.GOOGLE_API_KEY) {
     throw new Error("Please set the GOOGLE_API_KEY environment variable.");
   }
   const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO,
+    model: GEMINI_MODEL.GEMINI_PRO_1_5,
   });
   const result = await gemini.chat({
     messages: [
@@ -18,4 +19,26 @@ import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
     ],
   });
   console.log(result);
+
+  // chat with file
+  const resultWithFile = await gemini.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(resultWithFile);
 })();

examples/llamaindex-server/README.md

@@ -0,0 +1,17 @@
+# Run LlamaIndex Server with simple steps
+
+1. Setup environment variables
+
+```bash
+export OPENAI_API_KEY=<your-openai-api-key>
+```
+
+2. Run the server
+
+```bash
+npx tsx llamaindex-server/simple-workflow/index.ts
+```
+
+3. Open the app at `http://localhost:4000` and start chatting with the agent
+
+![Screenshot](./screenshot.png)

examples/llamaindex-server/simple-workflow/index.ts

@@ -0,0 +1,19 @@
+import { OpenAI } from "@llamaindex/openai";
+import { LlamaIndexServer } from "@llamaindex/server";
+import { weather } from "@llamaindex/tools";
+import "dotenv/config";
+import { agent } from "llamaindex";
+
+const weatherAgent = agent({
+  tools: [weather()],
+  llm: new OpenAI({ model: "gpt-4o-mini" }),
+});
+
+new LlamaIndexServer({
+  workflow: () => weatherAgent,
+  uiConfig: {
+    appTitle: "Weather Agent",
+    starterQuestions: ["Ho Chi Minh city weather", "New York weather"],
+  },
+  port: 4000,
+}).start();

examples/node/workflow/basic.ts

@@ -11,7 +11,6 @@ const workflow = new Workflow<ContextData, string, string>();
 workflow.addStep(
   {
     inputs: [StartEvent<string>],
-    outputs: [StopEvent<string>],
   },
   async (context, startEvent) => {
     const input = startEvent.data;

examples/openai/openai.ts

@@ -1,7 +1,8 @@
 import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
+import fs from "fs";
 
 (async () => {
-  const llm = new OpenAI({ model: "gpt-4.5-preview", temperature: 0.1 });
+  const llm = new OpenAI({ model: "gpt-4o" });
 
   // complete api
   const response1 = await llm.complete({ prompt: "How are you?" });
@@ -13,7 +14,51 @@ import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
   });
   console.log(response2.message.content);
 
-  // embeddings
+  // chat with file
+  const response3 = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+  console.log(response3.message.content);
+
+  // chat with image
+  const response4 = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this image? Describe it in detail.",
+          },
+          {
+            type: "image_url",
+            image_url: {
+              url: "https://storage.googleapis.com/cloud-samples-data/vision/face/faces.jpeg",
+            },
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log("Single Image Analysis:", response4.message.content);
+
+  // // embeddings
   const embedModel = new OpenAIEmbedding();
   const texts = ["hello", "world"];
   const embeddings = await embedModel.getTextEmbeddingsBatch(texts);

examples/openai/responses/chat-with-file.ts

@@ -0,0 +1,35 @@
+import { openaiResponses } from "@llamaindex/openai";
+import fs from "fs";
+
+async function main() {
+  if (!process.env.OPENAI_API_KEY) {
+    throw new Error("Please set the OPENAI_API_KEY environment variable.");
+  }
+
+  const llm = openaiResponses({
+    apiKey: process.env.OPENAI_API_KEY,
+  });
+
+  const result = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result);
+}
+
+void main().catch(console.error);

examples/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/examples",
-  "version": "0.3.11",
+  "version": "0.3.13",
   "private": true,
   "scripts": {
     "lint": "eslint .",
@@ -11,59 +11,60 @@
     "@azure/cosmos": "^4.1.1",
     "@azure/identity": "^4.4.1",
     "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.3",
-    "@llamaindex/astra": "^0.0.16",
-    "@llamaindex/azure": "^0.1.12",
-    "@llamaindex/chroma": "^0.0.16",
-    "@llamaindex/clip": "^0.0.52",
-    "@llamaindex/cloud": "^4.0.3",
-    "@llamaindex/cohere": "^0.0.16",
-    "@llamaindex/core": "^0.6.2",
-    "@llamaindex/deepinfra": "^0.0.52",
+    "@llamaindex/anthropic": "^0.3.4",
+    "@llamaindex/astra": "^0.0.17",
+    "@llamaindex/azure": "^0.1.13",
+    "@llamaindex/chroma": "^0.0.17",
+    "@llamaindex/clip": "^0.0.53",
+    "@llamaindex/cloud": "^4.0.4",
+    "@llamaindex/cohere": "^0.0.17",
+    "@llamaindex/core": "^0.6.3",
+    "@llamaindex/deepinfra": "^0.0.53",
     "@llamaindex/env": "^0.1.29",
-    "@llamaindex/firestore": "^1.0.9",
-    "@llamaindex/google": "^0.2.4",
-    "@llamaindex/groq": "^0.0.67",
-    "@llamaindex/huggingface": "^0.1.6",
-    "@llamaindex/milvus": "^0.1.11",
-    "@llamaindex/mistral": "^0.1.2",
-    "@llamaindex/mixedbread": "^0.0.16",
-    "@llamaindex/mongodb": "^0.0.17",
-    "@llamaindex/elastic-search": "^0.1.2",
-    "@llamaindex/node-parser": "^2.0.2",
-    "@llamaindex/ollama": "^0.1.2",
-    "@llamaindex/openai": "^0.3.4",
-    "@llamaindex/pinecone": "^0.1.2",
-    "@llamaindex/portkey-ai": "^0.0.44",
-    "@llamaindex/postgres": "^0.0.45",
-    "@llamaindex/qdrant": "^0.1.11",
-    "@llamaindex/readers": "^3.1.0",
-    "@llamaindex/replicate": "^0.0.44",
-    "@llamaindex/upstash": "^0.0.16",
-    "@llamaindex/vercel": "^0.1.2",
-    "@llamaindex/vllm": "^0.0.38",
-    "@llamaindex/voyage-ai": "^1.0.8",
-    "@llamaindex/weaviate": "^0.0.16",
-    "@llamaindex/workflow": "^1.0.3",
-    "@llamaindex/deepseek": "^0.0.12",
-    "@llamaindex/fireworks": "^0.0.12",
-    "@llamaindex/together": "^0.0.12",
-    "@llamaindex/jinaai": "^0.0.12",
-    "@llamaindex/perplexity": "^0.0.9",
-    "@llamaindex/supabase": "^0.1.1",
-    "@llamaindex/tools": "^0.0.6",
+    "@llamaindex/firestore": "^1.0.10",
+    "@llamaindex/google": "^0.2.5",
+    "@llamaindex/groq": "^0.0.68",
+    "@llamaindex/huggingface": "^0.1.7",
+    "@llamaindex/milvus": "^0.1.12",
+    "@llamaindex/mistral": "^0.1.3",
+    "@llamaindex/mixedbread": "^0.0.17",
+    "@llamaindex/mongodb": "^0.0.18",
+    "@llamaindex/elastic-search": "^0.1.3",
+    "@llamaindex/node-parser": "^2.0.3",
+    "@llamaindex/ollama": "^0.1.3",
+    "@llamaindex/openai": "^0.3.5",
+    "@llamaindex/pinecone": "^0.1.3",
+    "@llamaindex/portkey-ai": "^0.0.45",
+    "@llamaindex/postgres": "^0.0.46",
+    "@llamaindex/qdrant": "^0.1.12",
+    "@llamaindex/readers": "^3.1.1",
+    "@llamaindex/replicate": "^0.0.45",
+    "@llamaindex/upstash": "^0.0.17",
+    "@llamaindex/vercel": "^0.1.3",
+    "@llamaindex/vllm": "^0.0.39",
+    "@llamaindex/voyage-ai": "^1.0.9",
+    "@llamaindex/weaviate": "^0.0.17",
+    "@llamaindex/workflow": "^1.0.4",
+    "@llamaindex/deepseek": "^0.0.13",
+    "@llamaindex/fireworks": "^0.0.13",
+    "@llamaindex/together": "^0.0.13",
+    "@llamaindex/jinaai": "^0.0.13",
+    "@llamaindex/perplexity": "^0.0.10",
+    "@llamaindex/server": "^0.1.6",
+    "@llamaindex/supabase": "^0.1.2",
+    "@llamaindex/tools": "^0.0.8",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^4.0.0",
-    "@llamaindex/assemblyai": "^0.1.1",
-    "@llamaindex/discord": "^0.1.1",
-    "@llamaindex/notion": "^0.1.1",
+    "@llamaindex/assemblyai": "^0.1.2",
+    "@llamaindex/discord": "^0.1.2",
+    "@llamaindex/notion": "^0.1.2",
     "@vercel/postgres": "^0.10.0",
     "ai": "^4.0.0",
     "ajv": "^8.17.1",
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.10.2",
+    "llamaindex": "^0.10.3",
     "mongodb": "6.7.0",
     "postgres": "^3.4.4",
     "wikipedia": "^2.1.2",

examples/workflow/README.md

@@ -1,7 +0,0 @@
-# Workflow Examples
-
-These examples demonstrate LlamaIndexTS's workflow system. Check out [its documentation](https://ts.llamaindex.ai/modules/workflows) for more information.
-
-## Running the Examples
-
-To run the examples, make sure to run them from the parent folder called `examples`). For example, to run the joke workflow, run `npx tsx workflow/joke.ts`.

examples/workflow/app-creator.ts

@@ -1,157 +0,0 @@
-import { OpenAI } from "@llamaindex/openai";
-import {
-  HandlerContext,
-  StartEvent,
-  StopEvent,
-  Workflow,
-  WorkflowEvent,
-} from "llamaindex";
-
-const MAX_REVIEWS = 3;
-
-type Context = {
-  specification: string;
-  numberReviews: number;
-};
-
-// Using the o1-preview model (see https://platform.openai.com/docs/guides/reasoning?reasoning-prompt-examples=coding-planning)
-const llm = new OpenAI({ model: "o1-preview", temperature: 1 });
-
-// example specification from https://platform.openai.com/docs/guides/reasoning?reasoning-prompt-examples=coding-planning
-const specification = `Python app that takes user questions and looks them up in a 
-database where they are mapped to answers. If there is a close match, it retrieves 
-the matched answer. If there isn't, it asks the user to provide an answer and 
-stores the question/answer pair in the database.`;
-
-// Create custom event types
-export class MessageEvent extends WorkflowEvent<{ msg: string }> {}
-
-export class CodeEvent extends WorkflowEvent<{ code: string }> {}
-
-export class ReviewEvent extends WorkflowEvent<{
-  review: string;
-  code: string;
-}> {}
-
-// Helper function to truncate long strings
-const truncate = (str: string) => {
-  const MAX_LENGTH = 60;
-  if (str.length <= MAX_LENGTH) return str;
-  return str.slice(0, MAX_LENGTH) + "...";
-};
-
-// the architect is responsible for writing the structure and the initial code based on the specification
-const architect = async (
-  context: HandlerContext<Context>,
-  _: StartEvent<string>,
-) => {
-  const spec = context.data.specification;
-  // write a message to send an update to the user
-  context.sendEvent(
-    new MessageEvent({
-      msg: `Writing app using this specification: ${truncate(spec)}`,
-    }),
-  );
-  const prompt = `Build an app for this specification: <spec>${spec}</spec>. Make a plan for the directory structure you'll need, then return each file in full. Don't supply any reasoning, just code.`;
-  const code = await llm.complete({ prompt });
-  return new CodeEvent({ code: code.text });
-};
-
-// the coder is responsible for updating the code based on the review
-const coder = async (context: HandlerContext<Context>, ev: ReviewEvent) => {
-  // get the specification from the context
-  const spec = context.data.specification;
-  // get the latest review and code
-  const { review, code } = ev.data;
-  // write a message to send an update to the user
-  context.sendEvent(
-    new MessageEvent({
-      msg: `Update code based on review: ${truncate(review)}`,
-    }),
-  );
-  const prompt = `We need to improve code that should implement this specification: <spec>${spec}</spec>. Here is the current code: <code>${code}</code>. And here is a review of the code: <review>${review}</review>. Improve the code based on the review, keep the specification in mind, and return the full updated code. Don't supply any reasoning, just code.`;
-  const updatedCode = await llm.complete({ prompt });
-  return new CodeEvent({ code: updatedCode.text });
-};
-
-// the reviewer is responsible for reviewing the code and providing feedback
-const reviewer = async (context: HandlerContext<Context>, ev: CodeEvent) => {
-  // get the specification from the context
-  const spec = context.data.specification;
-  // get latest code from the event
-  const { code } = ev.data;
-  // update and check the number of reviews
-  context.data.numberReviews++;
-  if (context.data.numberReviews > MAX_REVIEWS) {
-    // the we've done this too many times - return the code
-    context.sendEvent(
-      new MessageEvent({
-        msg: `Already reviewed ${
-          context.data.numberReviews - 1
-        } times, stopping!`,
-      }),
-    );
-    return new StopEvent({ result: code });
-  }
-  // write a message to send an update to the user
-  context.sendEvent(
-    new MessageEvent({
-      msg: `Review #${context.data.numberReviews}: ${truncate(code)}`,
-    }),
-  );
-  const prompt = `Review this code: <code>${code}</code>. Check if the code quality and whether it correctly implements this specification: <spec>${spec}</spec>. If you're satisfied, just return 'Looks great', nothing else. If not, return a review with a list of changes you'd like to see.`;
-  const review = (await llm.complete({ prompt })).text;
-  if (review.includes("Looks great")) {
-    // the reviewer is satisfied with the code, let's return the review
-    context.sendEvent(
-      new MessageEvent({
-        msg: `Reviewer says: ${review}`,
-      }),
-    );
-    return new StopEvent({ result: code });
-  }
-
-  return new ReviewEvent({ review, code });
-};
-
-const codeAgent = new Workflow<Context, string, string>();
-codeAgent.addStep(
-  {
-    inputs: [StartEvent<string>],
-    outputs: [CodeEvent],
-  },
-  architect,
-);
-codeAgent.addStep(
-  {
-    inputs: [ReviewEvent],
-    outputs: [CodeEvent],
-  },
-  coder,
-);
-codeAgent.addStep(
-  {
-    inputs: [CodeEvent],
-    outputs: [ReviewEvent, StopEvent],
-  },
-  reviewer,
-);
-
-// Usage
-async function main() {
-  const run = codeAgent.run(specification).with({
-    specification,
-    numberReviews: 0,
-  });
-  for await (const event of run) {
-    if (event instanceof MessageEvent) {
-      const msg = (event as MessageEvent).data.msg;
-      console.log(`${msg}\n`);
-    } else if (event instanceof StopEvent) {
-      const result = (event as StopEvent<string>).data;
-      console.log("Final code:\n", result);
-    }
-  }
-}
-
-main().catch(console.error);

examples/workflow/conditional.ts

@@ -1,88 +0,0 @@
-import { OpenAI } from "@llamaindex/openai";
-import {
-  HandlerContext,
-  StartEvent,
-  StopEvent,
-  Workflow,
-  WorkflowEvent,
-} from "llamaindex";
-
-// Create LLM instance
-const llm = new OpenAI();
-
-// Create custom event types
-export class JokeEvent extends WorkflowEvent<{ joke: string }> {}
-
-export class CritiqueEvent extends WorkflowEvent<{ critique: string }> {}
-
-export class AnalysisEvent extends WorkflowEvent<{ analysis: string }> {}
-
-const generateJoke = async (_: unknown, ev: StartEvent<string>) => {
-  const prompt = `Write your best joke about ${ev.data}.`;
-  const response = await llm.complete({ prompt });
-  return new JokeEvent({ joke: response.text });
-};
-
-const critiqueJoke = async (_: unknown, ev: JokeEvent) => {
-  const prompt = `Give a thorough critique of the following joke: ${ev.data.joke}`;
-  const response = await llm.complete({ prompt });
-  return new CritiqueEvent({ critique: response.text });
-};
-
-const analyzeJoke = async (_: unknown, ev: JokeEvent) => {
-  const prompt = `Give a thorough analysis of the following joke: ${ev.data.joke}`;
-  const response = await llm.complete({ prompt });
-  return new AnalysisEvent({ analysis: response.text });
-};
-
-const reportJoke = async (
-  context: HandlerContext,
-  ev1: AnalysisEvent,
-  ev2: CritiqueEvent,
-) => {
-  const subPrompts = [ev1.data.analysis, ev2.data.critique];
-
-  const prompt = `Based on the following information about a joke:\n${subPrompts.join(
-    "\n",
-  )}\nProvide a comprehensive report on the joke's quality and impact.`;
-  const response = await llm.complete({ prompt });
-  return new StopEvent(response.text);
-};
-
-const jokeFlow = new Workflow<unknown, string, string>();
-jokeFlow.addStep(
-  {
-    inputs: [StartEvent<string>],
-    outputs: [JokeEvent],
-  },
-  generateJoke,
-);
-jokeFlow.addStep(
-  {
-    inputs: [JokeEvent],
-    outputs: [CritiqueEvent],
-  },
-  critiqueJoke,
-);
-jokeFlow.addStep(
-  {
-    inputs: [JokeEvent],
-    outputs: [AnalysisEvent],
-  },
-  analyzeJoke,
-);
-jokeFlow.addStep(
-  {
-    inputs: [AnalysisEvent, CritiqueEvent],
-    outputs: [StopEvent<string>],
-  },
-  reportJoke,
-);
-
-// Usage
-async function main() {
-  const result = await jokeFlow.run("pirates");
-  console.log(result.data);
-}
-
-main().catch(console.error);

examples/workflow/joke.ts

@@ -1,44 +0,0 @@
-import { OpenAI } from "@llamaindex/openai";
-import { StartEvent, StopEvent, Workflow, WorkflowEvent } from "llamaindex";
-
-// Create LLM instance
-const llm = new OpenAI();
-
-// Create a custom event type
-export class JokeEvent extends WorkflowEvent<{ joke: string }> {}
-
-const generateJoke = async (_: unknown, ev: StartEvent<string>) => {
-  const prompt = `Write your best joke about ${ev.data}.`;
-  const response = await llm.complete({ prompt });
-  return new JokeEvent({ joke: response.text });
-};
-
-const critiqueJoke = async (_: unknown, ev: JokeEvent) => {
-  const prompt = `Give a thorough critique of the following joke: ${ev.data.joke}`;
-  const response = await llm.complete({ prompt });
-  return new StopEvent(response.text);
-};
-
-const jokeFlow = new Workflow<unknown, string, string>();
-jokeFlow.addStep(
-  {
-    inputs: [StartEvent<string>],
-    outputs: [JokeEvent],
-  },
-  generateJoke,
-);
-jokeFlow.addStep(
-  {
-    inputs: [JokeEvent],
-    outputs: [StopEvent<string>],
-  },
-  critiqueJoke,
-);
-
-// Usage
-async function main() {
-  const result = await jokeFlow.run("pirates");
-  console.log(result.data);
-}
-
-main().catch(console.error);

examples/workflow/stream-events.ts

@@ -1,66 +0,0 @@
-import { OpenAI } from "@llamaindex/openai";
-import {
-  HandlerContext,
-  StartEvent,
-  StopEvent,
-  Workflow,
-  WorkflowEvent,
-} from "llamaindex";
-
-// Create LLM instance
-const llm = new OpenAI();
-
-// Create custom event types
-export class JokeEvent extends WorkflowEvent<{ joke: string }> {}
-
-export class MessageEvent extends WorkflowEvent<{ msg: string }> {}
-
-const generateJoke = async (context: HandlerContext, ev: StartEvent) => {
-  context.sendEvent(
-    new MessageEvent({ msg: `Generating a joke about: ${ev.data}` }),
-  );
-  const prompt = `Write your best joke about ${ev.data}.`;
-  const response = await llm.complete({ prompt });
-  return new JokeEvent({ joke: response.text });
-};
-
-const critiqueJoke = async (context: HandlerContext, ev: JokeEvent) => {
-  context.sendEvent(
-    new MessageEvent({ msg: `Write a critique of this joke: ${ev.data.joke}` }),
-  );
-  const prompt = `Give a thorough critique of the following joke: ${ev.data.joke}`;
-  const response = await llm.complete({ prompt });
-  return new StopEvent(response.text);
-};
-
-const jokeFlow = new Workflow();
-jokeFlow.addStep(
-  {
-    inputs: [StartEvent<string>],
-    outputs: [JokeEvent],
-  },
-  generateJoke,
-);
-jokeFlow.addStep(
-  {
-    inputs: [JokeEvent],
-    outputs: [StopEvent<string>],
-  },
-  critiqueJoke,
-);
-
-// Usage
-async function main() {
-  const run = jokeFlow.run("pirates");
-  for await (const event of run) {
-    if (event instanceof MessageEvent) {
-      console.log("Message:");
-      console.log((event as MessageEvent).data.msg);
-    } else if (event instanceof StopEvent) {
-      console.log("Result:");
-      console.log((event as StopEvent<string>).data);
-    }
-  }
-}
-
-main().catch(console.error);

examples/workflow/timeout.ts

@@ -1,48 +0,0 @@
-import { StartEvent, StopEvent, Workflow } from "llamaindex";
-
-const longRunning = async (_: unknown, ev: StartEvent<string>) => {
-  await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait for 2 seconds
-  return new StopEvent("We waited 2 seconds");
-};
-
-async function timeout() {
-  const workflow = new Workflow<unknown, string, string>({
-    timeout: 1,
-  });
-  workflow.addStep(
-    {
-      inputs: [StartEvent<string>],
-      outputs: [StopEvent<string>],
-    },
-    longRunning,
-  );
-  try {
-    await workflow.run("Let's start");
-  } catch (error) {
-    console.error(error);
-  }
-}
-
-async function notimeout() {
-  // Increase timeout to 3 seconds - no timeout
-  const workflow = new Workflow<unknown, string, string>({
-    timeout: 3,
-  });
-  workflow.addStep(
-    {
-      inputs: [StartEvent<string>],
-      outputs: [StopEvent<string>],
-    },
-    longRunning,
-  );
-  const result = await workflow.run("Let's start");
-  console.log(result.data);
-}
-
-async function main() {
-  await timeout();
-  console.log("---");
-  await notimeout();
-}
-
-main().catch(console.error);

examples/workflow/validation.ts

@@ -1,73 +0,0 @@
-import { OpenAI } from "@llamaindex/openai";
-import { StartEvent, StopEvent, Workflow, WorkflowEvent } from "llamaindex";
-
-// Create LLM instance
-const llm = new OpenAI();
-
-// Create a custom event type
-export class JokeEvent extends WorkflowEvent<{ joke: string }> {}
-
-const generateJoke = async (_: unknown, ev: StartEvent<string>) => {
-  const prompt = `Write your best joke about ${ev.data}.`;
-  const response = await llm.complete({ prompt });
-  return new JokeEvent({ joke: response.text });
-};
-
-const critiqueJoke = async (_: unknown, ev: JokeEvent) => {
-  const prompt = `Give a thorough critique of the following joke: ${ev.data.joke}`;
-  const response = await llm.complete({ prompt });
-  return new StopEvent(response.text);
-};
-
-async function validateFails() {
-  try {
-    const jokeFlow = new Workflow();
-    jokeFlow.addStep(
-      {
-        inputs: [StartEvent<string>],
-        outputs: [StopEvent<string>],
-      },
-      // @ts-expect-error outputs should be JokeEvent
-      generateJoke,
-    );
-    jokeFlow.addStep(
-      {
-        inputs: [JokeEvent],
-        outputs: [StopEvent],
-      },
-      critiqueJoke,
-    );
-    await jokeFlow.run("pirates").strict();
-  } catch (e) {
-    console.error("Validation failed:", e);
-  }
-}
-
-async function validate() {
-  const jokeFlow = new Workflow();
-  jokeFlow.addStep(
-    {
-      inputs: [StartEvent<string>],
-      outputs: [JokeEvent],
-    },
-    generateJoke,
-  );
-  jokeFlow.addStep(
-    {
-      inputs: [JokeEvent],
-      outputs: [StopEvent<string>],
-    },
-    critiqueJoke,
-  );
-  const result = await jokeFlow.run("pirates").strict();
-  console.log(result.data);
-}
-
-// Usage
-async function main() {
-  await validateFails();
-  console.log("---");
-  await validate();
-}
-
-main().catch(console.error);

packages/autotool/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/autotool
 
+## 7.0.3
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 7.0.2
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.104
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+  - @llamaindex/autotool@7.0.3
+
 ## 0.0.103
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.103"
+  "version": "0.0.104"
 }

packages/autotool/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
     "directory": "packages/autotool"
   },
-  "version": "7.0.2",
+  "version": "7.0.3",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/cloud
 
+## 4.0.4
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 4.0.3
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "4.0.3",
+  "version": "4.0.4",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/community
 
+## 0.0.97
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.0.96
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.96",
+  "version": "0.0.97",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/core
 
+## 0.6.3
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+
 ## 0.6.2
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./agent": {

packages/core/src/llms/index.ts

@@ -17,6 +17,7 @@ export type {
   LLMMetadata,
   MessageContent,
   MessageContentDetail,
+  MessageContentFileDetail,
   MessageContentImageDetail,
   MessageContentTextDetail,
   MessageType,

packages/core/src/llms/type.ts

@@ -163,9 +163,16 @@ export type MessageContentImageDetail = {
   detail?: "high" | "low" | "auto";
 };
 
+export type MessageContentFileDetail = {
+  type: "file";
+  data: Buffer;
+  mimeType: string;
+};
+
 export type MessageContentDetail =
   | MessageContentTextDetail
-  | MessageContentImageDetail;
+  | MessageContentImageDetail
+  | MessageContentFileDetail;
 
 /**
  * Extended type for the content of a message that allows for multi-modal messages.

packages/experimental/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/experimental
 
+## 0.0.173
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.0.172
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.172",
+  "version": "0.0.173",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,17 @@
 # llamaindex
 
+## 0.10.3
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+  - @llamaindex/cloud@4.0.4
+  - @llamaindex/node-parser@2.0.3
+  - @llamaindex/workflow@1.0.4
+
 ## 0.10.2
 
 ### Patch Changes

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.10.2",
+  "version": "0.10.3",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -25,7 +25,7 @@
     "@llamaindex/env": "workspace:*",
     "@llamaindex/node-parser": "workspace:*",
     "@llamaindex/openai": "workspace:*",
-    "@llamaindex/workflow": "workspace:*",
+    "@llamaindex/workflow": "1.0.4",
     "@types/lodash": "^4.17.7",
     "@types/node": "^22.9.0",
     "ajv": "^8.17.1",

packages/llamaindex/src/tools/QueryEngineTool.ts

@@ -51,7 +51,7 @@ export class QueryEngineTool implements BaseTool<QueryEngineParam> {
     const response = await this.queryEngine.query({ query });
 
     if (!this.includeSourceNodes) {
-      return { content: response.message.content };
+      return { content: response.message.content } as unknown as JSONValue;
     }
 
     return {

packages/node-parser/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/node-parser
 
+## 2.0.3
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 2.0.2
 
 ### Patch Changes

packages/node-parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/node-parser",
-  "version": "2.0.2",
+  "version": "2.0.3",
   "description": "Node parser for LlamaIndex",
   "type": "module",
   "exports": {

packages/providers/anthropic/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/anthropic
 
+## 0.3.4
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.3.3
 
 ### Patch Changes

packages/providers/anthropic/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/anthropic",
   "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/anthropic/src/llm.ts

@@ -319,6 +319,24 @@ export class Anthropic extends ToolCallLLM<
               text: content.text,
             };
           }
+
+          if (content.type === "file") {
+            if (content.mimeType !== "application/pdf") {
+              throw new Error(
+                "Only supports mimeType `application/pdf` for file content.",
+              );
+            }
+
+            return {
+              type: "document" as const,
+              source: {
+                type: "base64" as const,
+                media_type: content.mimeType,
+                data: content.data.toString("base64"),
+              },
+            };
+          }
+
           return {
             type: "image" as const,
             source: {

packages/providers/anthropic/tests/index.test.ts

@@ -164,6 +164,67 @@ describe("Message Formatting", () => {
 
       expect(anthropic.formatMessages(inputMessages)).toEqual(expectedOutput);
     });
+
+    test("Anthropic handles PDF file content", () => {
+      const anthropic = new Anthropic();
+      const pdfBuffer = Buffer.from("test PDF content");
+      const inputMessages: ChatMessage[] = [
+        {
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF document:",
+            },
+            {
+              type: "file",
+              mimeType: "application/pdf",
+              data: pdfBuffer,
+            },
+          ],
+          role: "user",
+        },
+      ];
+      const expectedOutput: MessageParam[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF document:",
+            },
+            {
+              type: "document",
+              source: {
+                type: "base64",
+                media_type: "application/pdf",
+                data: pdfBuffer.toString("base64"),
+              },
+            },
+          ],
+        },
+      ];
+
+      expect(anthropic.formatMessages(inputMessages)).toEqual(expectedOutput);
+    });
+
+    test("Anthropic throws error for non-PDF files", () => {
+      const anthropic = new Anthropic();
+      const docxBuffer = Buffer.from("fake docx content");
+      const inputMessages: ChatMessage[] = [
+        {
+          content: [
+            {
+              type: "file",
+              mimeType: "application/docx",
+              data: docxBuffer,
+            },
+          ],
+          role: "user",
+        },
+      ];
+
+      expect(() => anthropic.formatMessages(inputMessages)).toThrowError();
+    });
   });
 
   describe("Tool Message Formatting", () => {

packages/providers/assemblyai/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/assemblyai
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.1
 
 ### Patch Changes

packages/providers/assemblyai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/assemblyai",
   "description": "AssemblyAI Reader for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/clip/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/clip
 
+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.52
 
 ### Patch Changes

packages/providers/clip/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/clip",
   "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.52",
+  "version": "0.0.53",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/cohere/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/cohere
 
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.0.16
 
 ### Patch Changes

packages/providers/cohere/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/cohere",
   "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.16",
+  "version": "0.0.17",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepinfra/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/deepinfra
 
+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.52
 
 ### Patch Changes

packages/providers/deepinfra/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepinfra",
   "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.52",
+  "version": "0.0.53",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepseek/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/deepseek
 
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.12
 
 ### Patch Changes

packages/providers/deepseek/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepseek",
   "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/discord/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/discord
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.1
 
 ### Patch Changes

packages/providers/discord/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/discord",
   "description": "Discord Reader for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/fireworks/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/fireworks
 
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.12
 
 ### Patch Changes

packages/providers/fireworks/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/fireworks",
   "description": "Fireworks Adapter for LlamaIndex",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/google/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/google
 
+## 0.2.5
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.2.4
 
 ### Patch Changes

packages/providers/google/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/google",
   "description": "Google Adapter for LlamaIndex",
-  "version": "0.2.4",
+  "version": "0.2.5",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -32,7 +32,7 @@
   "dependencies": {
     "@google-cloud/vertexai": "1.9.0",
     "@google/genai": "^0.4.0",
-    "@google/generative-ai": "0.21.0",
+    "@google/generative-ai": "0.24.0",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*"
   }

packages/providers/google/src/base.ts

@@ -5,10 +5,12 @@ import {
   type FunctionCall,
   type ModelParams as GoogleModelParams,
   type RequestOptions as GoogleRequestOptions,
+  type StartChatParams as GoogleStartChatParams,
   type GenerateContentStreamResult as GoogleStreamGenerateContentResult,
   type SafetySetting,
 } from "@google/generative-ai";
 
+import type { StartChatParams as VertexStartChatParams } from "@google-cloud/vertexai";
 import { wrapLLMEvent } from "@llamaindex/core/decorator";
 import type {
   CompletionResponse,
@@ -96,6 +98,8 @@ export type GeminiConfig = Partial<typeof DEFAULT_GEMINI_PARAMS> & {
   safetySettings?: SafetySetting[];
 };
 
+type StartChatParams = GoogleStartChatParams & VertexStartChatParams;
+
 /**
  * Gemini Session to manage the connection to the Gemini API
  */
@@ -254,13 +258,13 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
     };
   }
 
-  private createStartChatParams(
+  private async createStartChatParams(
     params: GeminiChatParamsNonStreaming | GeminiChatParamsStreaming,
   ) {
-    const context = getChatContext(params);
+    const context = await getChatContext(params);
     const common = {
       history: context.history,
-      safetySettings: this.safetySettings,
+      safetySettings: this.safetySettings as SafetySetting[],
     };
 
     return params.tools?.length
@@ -282,12 +286,14 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
   protected async nonStreamChat(
     params: GeminiChatParamsNonStreaming,
   ): Promise<GeminiChatNonStreamResponse> {
-    const context = getChatContext(params);
+    const context = await getChatContext(params);
     const client = this.session.getGenerativeModel(
       this.metadata,
       this.#requestOptions,
     );
-    const chat = client.startChat(this.createStartChatParams(params));
+    const chat = client.startChat(
+      (await this.createStartChatParams(params)) as StartChatParams,
+    );
     const { response } = await chat.sendMessage(context.message);
     const topCandidate = response.candidates![0]!;
 
@@ -311,12 +317,14 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
   protected async *streamChat(
     params: GeminiChatParamsStreaming,
   ): GeminiChatStreamResponse {
-    const context = getChatContext(params);
+    const context = await getChatContext(params);
     const client = this.session.getGenerativeModel(
       this.metadata,
       this.#requestOptions,
     );
-    const chat = client.startChat(this.createStartChatParams(params));
+    const chat = client.startChat(
+      (await this.createStartChatParams(params)) as StartChatParams,
+    );
     const result = await chat.sendMessageStream(context.message);
     yield* this.session.getChatStream(result);
   }
@@ -350,7 +358,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
     if (stream) {
       const result = await client.generateContentStream(
         getPartsText(
-          GeminiHelper.messageContentToGeminiParts({ content: prompt }),
+          await GeminiHelper.messageContentToGeminiParts({ content: prompt }),
         ),
       );
       return this.session.getCompletionStream(result);
@@ -358,7 +366,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
 
     const result = await client.generateContent(
       getPartsText(
-        GeminiHelper.messageContentToGeminiParts({ content: prompt }),
+        await GeminiHelper.messageContentToGeminiParts({ content: prompt }),
       ),
     );
     return {

packages/providers/google/src/types.ts

@@ -68,8 +68,8 @@ export enum GEMINI_MODEL {
   GEMINI_PRO_1_5_LATEST = "gemini-1.5-pro-002",
   GEMINI_PRO_1_5_FLASH_LATEST = "gemini-1.5-flash-002",
   GEMINI_2_0_FLASH_EXPERIMENTAL = "gemini-2.0-flash-exp",
-  GEMINI_2_0_FLASH = "gemini-2.0-flash-001",
-  GEMINI_2_0_FLASH_LITE = "gemini-2.0-flash-lite-001",
+  GEMINI_2_0_FLASH = "gemini-2.0-flash",
+  GEMINI_2_0_FLASH_LITE = "gemini-2.0-flash-lite",
   GEMINI_2_0_FLASH_LITE_PREVIEW = "gemini-2.0-flash-lite-preview-02-05",
   GEMINI_2_0_FLASH_THINKING_EXP = "gemini-2.0-flash-thinking-exp-01-21",
   GEMINI_2_0_PRO_EXPERIMENTAL = "gemini-2.0-pro-exp-02-05",

packages/providers/google/src/utils.ts

@@ -8,15 +8,18 @@ import {
 } from "@google/generative-ai";
 
 import { type GenerateContentResponse } from "@google-cloud/vertexai";
+import { FileState, GoogleAIFileManager } from "@google/generative-ai/server";
 import type {
   BaseTool,
   ChatMessage,
+  MessageContentFileDetail,
   MessageContentImageDetail,
   MessageContentTextDetail,
   MessageType,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
 import { extractDataUrlComponents } from "@llamaindex/core/utils";
+import { getEnv } from "@llamaindex/env";
 import type {
   ChatContext,
   FileDataPart,
@@ -126,9 +129,9 @@ export const cleanParts = (
   };
 };
 
-export const getChatContext = (
+export const getChatContext = async (
   params: GeminiChatParamsStreaming | GeminiChatParamsNonStreaming,
-): ChatContext => {
+): Promise<ChatContext> => {
   // Gemini doesn't allow:
   // 1. Consecutive messages from the same role
   // 2. Parts that have empty text
@@ -145,8 +148,10 @@ export const getChatContext = (
     {} as Record<string, string>,
   );
   const messages = GeminiHelper.mergeNeighboringSameRoleMessages(
-    params.messages.map((message) =>
-      GeminiHelper.chatMessageToGemini(message, fnMap),
+    await Promise.all(
+      params.messages.map((message) =>
+        GeminiHelper.chatMessageToGemini(message, fnMap),
+      ),
     ),
   ).map(cleanParts);
 
@@ -226,13 +231,13 @@ export class GeminiHelper {
       );
   }
 
-  public static messageContentToGeminiParts({
+  public static async messageContentToGeminiParts({
     content,
     options = undefined,
     fnMap = undefined,
   }: Pick<ChatMessage<ToolCallLLMMessageOptions>, "content" | "options"> & {
     fnMap?: Record<string, string>;
-  }): Part[] {
+  }): Promise<Part[]> {
     if (options && "toolResult" in options) {
       if (!fnMap) throw Error("fnMap must be set");
       const name = fnMap[options.toolResult.id];
@@ -276,9 +281,53 @@ export class GeminiHelper {
       (i) => i.type === "text",
     ) as MessageContentTextDetail[];
     parts.push(...textContents.map((t) => ({ text: t.text })));
+
+    const fileContents = content.filter(
+      (i) => i.type === "file",
+    ) as MessageContentFileDetail[];
+
+    if (fileContents.length > 0) {
+      for (const file of fileContents) {
+        const uploadResponse = await GeminiHelper.uploadFile(
+          file.data,
+          file.mimeType,
+        );
+        parts.push({
+          fileData: {
+            mimeType: uploadResponse.file.mimeType,
+            fileUri: uploadResponse.file.uri,
+          },
+        });
+      }
+    }
+
     return parts;
   }
 
+  // Upload a file for AI processing
+  public static async uploadFile(
+    data: string | Buffer, // file name or buffer
+    mimeType: string, // eg. application/pdf
+    interval = 5_000, // time to refetch upload status
+  ) {
+    const fileManager = new GoogleAIFileManager(getEnv("GOOGLE_API_KEY")!);
+
+    const uploadResponse = await fileManager.uploadFile(data, { mimeType });
+
+    let file = await fileManager.getFile(uploadResponse.file.name);
+
+    while (file.state === FileState.PROCESSING) {
+      await new Promise((resolve) => setTimeout(resolve, interval));
+      file = await fileManager.getFile(uploadResponse.file.name);
+    }
+
+    if (file.state === FileState.FAILED) {
+      throw new Error("Failed to upload file");
+    }
+
+    return uploadResponse;
+  }
+
   public static getGeminiMessageRole(
     message: ChatMessage<ToolCallLLMMessageOptions>,
   ): GeminiMessageRole {
@@ -290,13 +339,16 @@ export class GeminiHelper {
     ];
   }
 
-  public static chatMessageToGemini(
+  public static async chatMessageToGemini(
     message: ChatMessage<ToolCallLLMMessageOptions>,
     fnMap: Record<string, string>, // mapping of fn call id to fn call name
-  ): GeminiMessageContent {
+  ): Promise<GeminiMessageContent> {
     return {
       role: GeminiHelper.getGeminiMessageRole(message),
-      parts: GeminiHelper.messageContentToGeminiParts({ ...message, fnMap }),
+      parts: await GeminiHelper.messageContentToGeminiParts({
+        ...message,
+        fnMap,
+      }),
     };
   }
 }

packages/providers/google/src/vertex.ts

@@ -1,5 +1,6 @@
 import {
   type GenerateContentResponse,
+  type SafetySetting,
   VertexAI,
   GenerativeModel as VertexGenerativeModel,
   GenerativeModelPreview as VertexGenerativeModelPreview,
@@ -62,12 +63,12 @@ export class GeminiVertexSession implements IGeminiSession {
     const safetySettings = metadata.safetySettings ?? DEFAULT_SAFETY_SETTINGS;
     if (this.preview) {
       return this.vertex.preview.getGenerativeModel({
-        safetySettings,
+        safetySettings: safetySettings as SafetySetting[],
         ...metadata,
       });
     }
     return this.vertex.getGenerativeModel({
-      safetySettings,
+      safetySettings: safetySettings as SafetySetting[],
       ...metadata,
     });
   }

packages/providers/groq/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/groq
 
+## 0.0.68
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.67
 
 ### Patch Changes

packages/providers/groq/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/groq",
   "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.67",
+  "version": "0.0.68",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/huggingface/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/huggingface
 
+## 0.1.7
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+
 ## 0.1.6
 
 ### Patch Changes

packages/providers/huggingface/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/huggingface",
   "description": "Huggingface Adapter for LlamaIndex",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/jinaai/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/jinaai
 
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.12
 
 ### Patch Changes

packages/providers/jinaai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/jinaai",
   "description": "JinaAI Adapter for LlamaIndex",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/mistral/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/mistral
 
+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.2
 
 ### Patch Changes

packages/providers/mistral/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/mistral",
   "description": "Mistral Adapter for LlamaIndex",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/mixedbread/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/mixedbread
 
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.0.16
 
 ### Patch Changes

packages/providers/mixedbread/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/mixedbread",
   "description": "Mixedbread Adapter for LlamaIndex",
-  "version": "0.0.16",
+  "version": "0.0.17",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/notion/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/notion
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.1
 
 ### Patch Changes

packages/providers/notion/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/notion",
   "description": "Notion Reader for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/ollama/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/ollama
 
+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.2
 
 ### Patch Changes

packages/providers/ollama/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/ollama",
   "description": "Ollama Adapter for LlamaIndex",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/openai/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/openai
 
+## 0.3.5
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.3.4
 
 ### Patch Changes

packages/providers/openai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/openai",
   "description": "OpenAI Adapter for LlamaIndex",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/openai/src/llm.ts

@@ -1,5 +1,6 @@
 import { wrapEventCaller, wrapLLMEvent } from "@llamaindex/core/decorator";
 import {
+  ToolCallLLM,
   type BaseTool,
   type ChatMessage,
   type ChatResponse,
@@ -9,7 +10,6 @@ import {
   type LLMMetadata,
   type MessageType,
   type PartialToolCall,
-  ToolCallLLM,
   type ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
 import { extractText } from "@llamaindex/core/utils";
@@ -24,6 +24,7 @@ import { zodResponseFormat } from "openai/helpers/zod";
 import type { ChatModel } from "openai/resources/chat/chat";
 import type {
   ChatCompletionAssistantMessageParam,
+  ChatCompletionContentPart,
   ChatCompletionMessageToolCall,
   ChatCompletionRole,
   ChatCompletionSystemMessageParam,
@@ -205,9 +206,29 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
           }),
         } satisfies ChatCompletionAssistantMessageParam;
       } else if (message.role === "user") {
+        if (typeof message.content === "string") {
+          return { role: "user", content: message.content };
+        }
+
         return {
           role: "user",
-          content: message.content,
+          content: message.content.map((item, index) => {
+            if (item.type === "file") {
+              if (item.mimeType !== "application/pdf") {
+                throw new Error("Only PDF files are supported");
+              }
+              return {
+                type: "file",
+                file: {
+                  file_data: `data:${item.mimeType};base64,${item.data.toString("base64")}`,
+                  filename: `part-${index}.pdf`,
+                },
+              } satisfies ChatCompletionContentPart.File;
+            }
+
+            // keep it as is for other types
+            return item;
+          }),
         } satisfies ChatCompletionUserMessageParam;
       }
 

packages/providers/openai/src/responses.ts

@@ -682,7 +682,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
       return content;
     }
 
-    return content.map((item) => {
+    return content.map((item, index) => {
       if (item.type === "text") {
         return {
           type: "input_text",
@@ -696,6 +696,19 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
           detail: item.detail || "auto",
         };
       }
+      if (item.type === "file") {
+        if (item.mimeType !== "application/pdf") {
+          throw new Error(
+            "Only supports mimeType `application/pdf` for file content.",
+          );
+        }
+
+        return {
+          type: "input_file",
+          filename: `part-${index}.pdf`,
+          file_data: `data:${item.mimeType};base64,${item.data.toString("base64")}`,
+        };
+      }
       throw new Error("Unsupported content type");
     });
   }

packages/providers/openai/src/utils.ts

@@ -227,9 +227,17 @@ export type ResponsesMessageContentImageDetail = {
   image_url: string;
   detail: "high" | "low" | "auto";
 };
+
+export type ResponsesMessageContentFileDetail = {
+  type: "input_file";
+  filename: string;
+  file_data: string;
+};
+
 export type ResponsesMessageContentDetail =
   | ResponsesMessageContentTextDetail
-  | ResponsesMessageContentImageDetail;
+  | ResponsesMessageContentImageDetail
+  | ResponsesMessageContentFileDetail;
 
 export type ResponseMessageContent = string | ResponsesMessageContentDetail[];