Release 0.8.17 (#1504)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

README.md

@@ -1,12 +1,16 @@
-# LlamaIndex.TS
+<p align="center">
+  <img height="100" width="100" alt="LlamaIndex logo" src="https://ts.llamaindex.ai/square.svg" />
+</p>
+<h1 align="center">LlamaIndex.TS</h1>
+<h3 align="center">
+  Data framework for your LLM application.
+</h3>
 
 [![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
 [![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
 [![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
 [![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
 
-LlamaIndex is a data framework for your LLM application.
-
 Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in JS runtime environments with TypeScript support.
 
 Documentation: https://ts.llamaindex.ai/

apps/docs/CHANGELOG.md

@@ -1,5 +1,73 @@
 # docs
 
+## 0.0.122
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.121
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.120
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.119
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.118
+
+### Patch Changes
+
+- llamaindex@0.8.13
+- @llamaindex/examples@0.0.16
+
+## 0.0.117
+
+### Patch Changes
+
+- @llamaindex/examples@0.0.15
+
+## 0.0.116
+
+### Patch Changes
+
+- llamaindex@0.8.12
+
+## 0.0.115
+
+### Patch Changes
+
+- llamaindex@0.8.11
+
+## 0.0.114
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+  - @llamaindex/examples@0.0.14
+
+## 0.0.113
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+
 ## 0.0.112
 
 ### Patch Changes

apps/docs/docusaurus.config.js

@@ -62,6 +62,12 @@ const config = {
     ({
       // Replace with your project's social card
       image: "img/favicon.png", // TODO change this
+      announcementBar: {
+        id: "migrate_to_next",
+        content:
+          'We are migrating to Next.js based documentation. Check it out <a href="https://ts.llamaindex.ai/docs/llamaindex">here</a>!',
+        isCloseable: false,
+      },
       navbar: {
         title: "LlamaIndex.TS",
         logo: {

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.112",
+  "version": "0.0.122",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

apps/next/CHANGELOG.md

@@ -1,5 +1,116 @@
 # @llamaindex/doc
 
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+  - @llamaindex/cloud@2.0.11
+  - llamaindex@0.8.17
+  - @llamaindex/node-parser@0.0.12
+  - @llamaindex/openai@0.1.36
+  - @llamaindex/readers@1.0.13
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [e60328b]
+  - @llamaindex/readers@1.0.12
+  - llamaindex@0.8.16
+
+## 0.0.18
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [ee20c44]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+  - @llamaindex/workflow@0.0.6
+  - llamaindex@0.8.13
+  - @llamaindex/cloud@2.0.10
+  - @llamaindex/node-parser@0.0.11
+  - @llamaindex/openai@0.1.35
+  - @llamaindex/readers@1.0.11
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [ea92b69]
+- Updated dependencies [fadc8b8]
+  - @llamaindex/workflow@0.0.5
+
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [7ae6eaa]
+  - @llamaindex/core@0.4.9
+  - @llamaindex/openai@0.1.34
+  - @llamaindex/cloud@2.0.9
+  - llamaindex@0.8.12
+  - @llamaindex/node-parser@0.0.10
+  - @llamaindex/readers@1.0.10
+
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [f865c98]
+  - @llamaindex/core@0.4.8
+  - @llamaindex/cloud@2.0.8
+  - llamaindex@0.8.11
+  - @llamaindex/node-parser@0.0.9
+  - @llamaindex/openai@0.1.33
+  - @llamaindex/readers@1.0.9
+
+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+- Updated dependencies [d89ebe0]
+- Updated dependencies [fd8c882]
+- Updated dependencies [fd8c882]
+  - llamaindex@0.8.10
+  - @llamaindex/core@0.4.7
+  - @llamaindex/workflow@0.0.4
+  - @llamaindex/cloud@2.0.7
+  - @llamaindex/node-parser@0.0.8
+  - @llamaindex/openai@0.1.32
+  - @llamaindex/readers@1.0.8
+
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+  - @llamaindex/cloud@2.0.6
+  - @llamaindex/core@0.4.6
+  - @llamaindex/node-parser@0.0.7
+  - @llamaindex/openai@0.1.31
+  - @llamaindex/readers@1.0.7
+
 ## 0.0.10
 
 ### Patch Changes

apps/next/next.config.mjs

@@ -6,6 +6,7 @@ const withMDX = createMDX();
 const config = {
   reactStrictMode: true,
   transpilePackages: ["monaco-editor"],
+  serverExternalPackages: ["@huggingface/transformers"],
   webpack: (config, { isServer }) => {
     if (Array.isArray(config.target) && config.target.includes("web")) {
       config.target = ["web", "es2020"];
@@ -26,6 +27,7 @@ const config = {
         }),
       );
     }
+    config.resolve.alias["replicate"] = false;
     return config;
   },
 };

apps/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/doc",
-  "version": "0.0.10",
+  "version": "0.0.20",
   "private": true,
   "scripts": {
     "build": "pnpm run build:docs && next build",
@@ -12,7 +12,7 @@
   },
   "dependencies": {
     "@icons-pack/react-simple-icons": "^10.1.0",
-    "@llamaindex/chat-ui": "0.0.5",
+    "@llamaindex/chat-ui": "0.0.8",
     "@llamaindex/cloud": "workspace:*",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/node-parser": "workspace:*",

apps/next/src/app/api/chat/route.ts

@@ -1,7 +1,10 @@
+import { llm } from "@/lib/utils";
 import { LlamaIndexAdapter, type Message } from "ai";
-import { SimpleChatEngine, type ChatMessage } from "llamaindex";
+import { Settings, SimpleChatEngine, type ChatMessage } from "llamaindex";
 import { NextResponse, type NextRequest } from "next/server";
 
+Settings.llm = llm;
+
 export async function POST(request: NextRequest) {
   try {
     const { messages } = (await request.json()) as { messages: Message[] };

apps/next/src/components/demo/chat.tsx

@@ -1,8 +0,0 @@
-"use client";
-import { ChatSection } from "@llamaindex/chat-ui";
-import { useChat } from "ai/react";
-
-export const ChatDemo = () => {
-  const handler = useChat();
-  return <ChatSection handler={handler} />;
-};

apps/next/src/components/demo/chat/api/demo.tsx

@@ -0,0 +1,16 @@
+"use client";
+import { ChatInput, ChatMessages, ChatSection } from "@llamaindex/chat-ui";
+import { useChat } from "ai/react";
+
+export const ChatDemo = () => {
+  const handler = useChat();
+  return (
+    <ChatSection handler={handler}>
+      <ChatMessages>
+        <ChatMessages.List className="h-auto max-h-[400px]" />
+        <ChatMessages.Actions />
+      </ChatMessages>
+      <ChatInput />
+    </ChatSection>
+  );
+};

apps/next/src/components/demo/chat/rsc/ai-action.tsx

@@ -0,0 +1,57 @@
+import { llm } from "@/lib/utils";
+import { Markdown } from "@llamaindex/chat-ui/widgets";
+import { generateId, Message } from "ai";
+import { createAI, createStreamableUI, getMutableAIState } from "ai/rsc";
+import { type ChatMessage, Settings, SimpleChatEngine } from "llamaindex";
+import { ReactNode } from "react";
+
+type ServerState = Message[];
+type FrontendState = Array<Message & { display: ReactNode }>;
+type Actions = {
+  chat: (message: Message) => Promise<Message & { display: ReactNode }>;
+};
+
+Settings.llm = llm;
+
+export const AI = createAI<ServerState, FrontendState, Actions>({
+  initialAIState: [],
+  initialUIState: [],
+  actions: {
+    chat: async (message: Message) => {
+      "use server";
+
+      const aiState = getMutableAIState<typeof AI>();
+      aiState.update((prev) => [...prev, message]);
+
+      const uiStream = createStreamableUI();
+      const chatEngine = new SimpleChatEngine();
+      const assistantMessage: Message = {
+        id: generateId(),
+        role: "assistant",
+        content: "",
+      };
+
+      // run the async function without blocking
+      (async () => {
+        const chatResponse = await chatEngine.chat({
+          stream: true,
+          message: message.content,
+          chatHistory: aiState.get() as ChatMessage[],
+        });
+
+        for await (const chunk of chatResponse) {
+          assistantMessage.content += chunk.delta;
+          uiStream.update(<Markdown content={assistantMessage.content} />);
+        }
+
+        aiState.done([...aiState.get(), assistantMessage]);
+        uiStream.done();
+      })();
+
+      return {
+        ...assistantMessage,
+        display: uiStream.value,
+      };
+    },
+  },
+});

apps/next/src/components/demo/chat/rsc/chat-section.tsx

@@ -0,0 +1,33 @@
+"use client";
+
+import {
+  ChatInput,
+  ChatMessage,
+  ChatMessages,
+  ChatSection as ChatSectionUI,
+} from "@llamaindex/chat-ui";
+import { useChatRSC } from "./use-chat-rsc";
+
+export const ChatSectionRSC = () => {
+  const handler = useChatRSC();
+  return (
+    <ChatSectionUI handler={handler}>
+      <ChatMessages>
+        <ChatMessages.List className="h-auto max-h-[400px]">
+          {handler.messages.map((message, index) => (
+            <ChatMessage
+              key={index}
+              message={message}
+              isLast={index === handler.messages.length - 1}
+            >
+              <ChatMessage.Avatar />
+              <ChatMessage.Content>{message.display}</ChatMessage.Content>
+            </ChatMessage>
+          ))}
+          <ChatMessages.Loading />
+        </ChatMessages.List>
+      </ChatMessages>
+      <ChatInput />
+    </ChatSectionUI>
+  );
+};

apps/next/src/components/demo/chat/rsc/demo.tsx

@@ -0,0 +1,8 @@
+import { AI } from "./ai-action";
+import { ChatSectionRSC } from "./chat-section";
+
+export const ChatDemoRSC = () => (
+  <AI>
+    <ChatSectionRSC />
+  </AI>
+);

apps/next/src/components/demo/chat/rsc/use-chat-rsc.tsx

@@ -0,0 +1,41 @@
+"use client";
+
+import { useActions } from "ai/rsc";
+
+import { generateId, Message } from "ai";
+import { useUIState } from "ai/rsc";
+import { useState } from "react";
+import { AI } from "./ai-action";
+
+export function useChatRSC() {
+  const [input, setInput] = useState<string>("");
+  const [isLoading, setIsLoading] = useState<boolean>(false);
+  const [messages, setMessages] = useUIState<typeof AI>();
+  const { chat } = useActions<typeof AI>();
+
+  const append = async (message: Omit<Message, "id">) => {
+    const newMsg: Message = { ...message, id: generateId() };
+
+    setIsLoading(true);
+    try {
+      setMessages((prev) => [...prev, { ...newMsg, display: message.content }]);
+      const assistantMsg = await chat(newMsg);
+      setMessages((prev) => [...prev, assistantMsg]);
+    } catch (error) {
+      console.error(error);
+    }
+    setIsLoading(false);
+    setInput("");
+
+    return message.content;
+  };
+
+  return {
+    input,
+    setInput,
+    isLoading,
+    messages,
+    setMessages,
+    append,
+  };
+}

apps/next/src/components/website/Footer.tsx

@@ -85,6 +85,33 @@ const Footer = () => {
                   <Text as="span">SharePoint</Text>
                 </a>
               </li>
+              <li>
+                <a
+                  href="https://llamaindex.ai/llamacloud-aws-s3-data-loading-for-generative-ai"
+                  data-tracking-variant="link"
+                  data-tracking-section="footer"
+                >
+                  <Text as="span">AWS S3</Text>
+                </a>
+              </li>
+              <li>
+                <a
+                  href="https://llamaindex.ai/llamacloud-azure-blob-storage-data-loading-for-generative-ai"
+                  data-tracking-variant="link"
+                  data-tracking-section="footer"
+                >
+                  <Text as="span">Azure Blob Storage</Text>
+                </a>
+              </li>
+              <li>
+                <a
+                  href="https://llamaindex.ai/llamacloud-google-drive-data-loading-for-generative-ai"
+                  data-tracking-variant="link"
+                  data-tracking-section="footer"
+                >
+                  <Text as="span">Google Drive</Text>
+                </a>
+              </li>
             </ul>
           </div>
           <div>
@@ -171,11 +198,6 @@ const Footer = () => {
                   <Text as="span">SEC Insights</Text>
                 </a>
               </li>
-              <li>
-                <a href="https://chat.llamaindex.ai/">
-                  <Text as="span">Chat LlamaIndex</Text>
-                </a>
-              </li>
               <li>
                 <a href="https://github.com/run-llama/llamabot">
                   <Text as="span">LlamaBot</Text>

apps/next/src/content/docs/llamaindex/guide/chat/chat.mdx

@@ -1,8 +1,8 @@
 ---
-title: Chat-UI 
-description: Use chat-ui to add a chat interface to your LlamaIndexTS application.
+title: Using API Route
+description: Chat interface for your LlamaIndexTS application using API Route
 ---
-import { ChatDemo } from '../../../../components/demo/chat';
+import { ChatDemo } from '../../../../../components/demo/chat/api/demo';
 import "@llamaindex/chat-ui/styles/code.css";
 import "@llamaindex/chat-ui/styles/katex.css";
 
@@ -26,7 +26,7 @@ This is the simplest way to add a chat interface to your application. Copy the f
 
 ```json doc-gen:file
 {
-  "file": "./src/components/demo/chat.tsx",
+  "file": "./src/components/demo/chat/api/demo.tsx",
 	"codeblock": true
 }
 ```
@@ -37,6 +37,7 @@ Combining both, you're getting a fully functional chat interface:
 
 <ChatDemo />
 
+
 ## Next Steps
 
 The steps above are the bare minimum to get a chat interface working. From here, you can go two ways:

apps/next/src/content/docs/llamaindex/guide/chat/meta.json

@@ -0,0 +1,6 @@
+{
+  "title": "Chat-UI",
+  "description": "Use chat-ui to add a chat interface to your LlamaIndexTS application.",
+  "defaultOpen": false,
+  "pages": ["chat", "rsc"]
+}

apps/next/src/content/docs/llamaindex/guide/chat/rsc.mdx

@@ -0,0 +1,68 @@
+---
+title: Using Next.js RSC
+description: Chat interface for your LlamaIndexTS application using Next.js RSC
+---
+import { ChatDemoRSC } from '../../../../../components/demo/chat/rsc/demo';
+import "@llamaindex/chat-ui/styles/code.css";
+import "@llamaindex/chat-ui/styles/katex.css";
+
+Using [chat-ui](https://github.com/run-llama/chat-ui), it's easy to add a chat interface to your LlamaIndexTS application using [Next.js RSC](https://nextjs.org/docs/app/building-your-application/rendering/server-components) and [Vercel AI RSC](https://sdk.vercel.ai/docs/ai-sdk-rsc/overview).
+
+With RSC, the chat messages are not returned as JSON from the server (like when using an [API route](./chat)), instead the chat message components are rendered on the server side.
+This is for example useful for rendering a whole chat history on the server before sending it to the client. [Check here](https://sdk.vercel.ai/docs/getting-started/navigating-the-library#when-to-use-ai-sdk-rsc), for a discussion of when to use use RSC.
+
+For implementing a chat interface with RSC, you need to create an AI action and then connect the chat interface to use it.
+
+## Create an AI action
+
+First, define an [AI context provider](https://sdk.vercel.ai/examples/rsc/state-management/ai-ui-states) with a chat server action:
+
+```json doc-gen:file
+{
+  "file": "./src/components/demo/chat/rsc/ai-action.tsx",
+	"codeblock": true
+}
+```
+
+The chat server action is using LlamaIndexTS to generate a response based on the chat history and the user input.
+
+## Create the chat UI
+
+The entrypoint of our application initializes the AI provider for the application and adds a `ChatSection` component:
+
+```json doc-gen:file
+{
+  "file": "./src/components/demo/chat/rsc/demo.tsx",
+	"codeblock": true
+}
+```
+
+The `ChatSection` component is created by using chat components from @llamaindex/chat-ui:
+
+```json doc-gen:file
+{
+  "file": "./src/components/demo/chat/rsc/chat-section.tsx",
+	"codeblock": true
+}
+```
+
+It is using a `useChatRSC` hook to conntect the chat interface to the `chat` AI action that we defined earlier:
+
+```json doc-gen:file
+{
+  "file": "./src/components/demo/chat/rsc/use-chat-rsc.tsx",
+	"codeblock": true
+}
+```
+
+## Try RSC Chat ⬇️
+
+<ChatDemoRSC />
+
+## Next Steps
+
+The steps above are the bare minimum to get a chat interface working with RSC. From here, you can go two ways:
+
+1. Use our [full-stack RSC example](https://github.com/run-llama/nextjs-rsc) based on [create-llama](https://github.com/run-llama/create-llama) to get started quickly with a fully working chat interface or
+2. Learn more about [AI RSC](https://sdk.vercel.ai/examples/rsc), [chat-ui](https://github.com/run-llama/chat-ui) and [LlamaIndexTS](https://github.com/run-llama/llamaindex-ts) to customize the chat interface and AI actions to your needs.
+

apps/next/src/lib/utils.ts

@@ -1,6 +1,34 @@
-import { clsx, type ClassValue } from "clsx"
-import { twMerge } from "tailwind-merge"
+import { clsx, type ClassValue } from "clsx";
+import { LLM, LLMMetadata } from "llamaindex";
+import { twMerge } from "tailwind-merge";
 
 export function cn(...inputs: ClassValue[]) {
   return twMerge(clsx(inputs))
 }
+
+class MockLLM  {
+  metadata: LLMMetadata = {
+    model: "MockLLM",
+    temperature: 0.5,
+    topP: 0.5,
+    contextWindow: 1024,
+    tokenizer: undefined,
+  };
+
+  chat() {
+    const mockResponse = "Hello! This is a mock response";
+    return Promise.resolve(
+      new ReadableStream({
+        async start(controller) {
+          for (const char of mockResponse) {
+            controller.enqueue({ delta: char });
+            await new Promise((resolve) => setTimeout(resolve, 20));
+          }
+          controller.close();
+        },
+      }),
+    );
+  }
+}
+
+export const llm = new MockLLM() as unknown as LLM;

apps/next/turbo.json

@@ -1,6 +1,14 @@
 {
   "extends": ["//"],
   "tasks": {
+    "build": {
+      "outputs": [
+        ".next",
+        ".source",
+        "next-env.d.ts",
+        "src/content/docs/cloud/api/**"
+      ]
+    },
     "dev": {
       "dependsOn": ["^build"]
     }

e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,65 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.113
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.112
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.111
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.110
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.109
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
+## 0.0.108
+
+### Patch Changes
+
+- llamaindex@0.8.12
+
+## 0.0.107
+
+### Patch Changes
+
+- llamaindex@0.8.11
+
+## 0.0.106
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+
+## 0.0.105
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+
 ## 0.0.104
 
 ### Patch Changes

e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.104",
+  "version": "0.0.113",
   "type": "module",
   "private": true,
   "scripts": {

e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,41 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.31
+
+### Patch Changes
+
+- @llamaindex/cloud@2.0.11
+
+## 0.0.30
+
+### Patch Changes
+
+- @llamaindex/cloud@2.0.10
+
+## 0.0.29
+
+### Patch Changes
+
+- @llamaindex/cloud@2.0.9
+
+## 0.0.28
+
+### Patch Changes
+
+- @llamaindex/cloud@2.0.8
+
+## 0.0.27
+
+### Patch Changes
+
+- @llamaindex/cloud@2.0.7
+
+## 0.0.26
+
+### Patch Changes
+
+- @llamaindex/cloud@2.0.6
+
 ## 0.0.25
 
 ### Patch Changes

e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.25",
+  "version": "0.0.31",
   "type": "module",
   "scripts": {
     "dev": "vite",

e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,65 @@
 # @llamaindex/next-agent-test
 
+## 0.1.113
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.1.112
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.1.111
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.1.110
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.1.109
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
+## 0.1.108
+
+### Patch Changes
+
+- llamaindex@0.8.12
+
+## 0.1.107
+
+### Patch Changes
+
+- llamaindex@0.8.11
+
+## 0.1.106
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+
+## 0.1.105
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+
 ## 0.1.104
 
 ### Patch Changes

e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.104",
+  "version": "0.1.113",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,65 @@
 # test-edge-runtime
 
+## 0.1.112
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.1.111
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.1.110
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.1.109
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.1.108
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
+## 0.1.107
+
+### Patch Changes
+
+- llamaindex@0.8.12
+
+## 0.1.106
+
+### Patch Changes
+
+- llamaindex@0.8.11
+
+## 0.1.105
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+
+## 0.1.104
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+
 ## 0.1.103
 
 ### Patch Changes

e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.103",
+  "version": "0.1.112",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,65 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.94
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.93
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.92
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.91
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.90
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
+## 0.0.89
+
+### Patch Changes
+
+- llamaindex@0.8.12
+
+## 0.0.88
+
+### Patch Changes
+
+- llamaindex@0.8.11
+
+## 0.0.87
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+
+## 0.0.86
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+
 ## 0.0.85
 
 ### Patch Changes

e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.85",
+  "version": "0.0.94",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-node-runtime/src/actions/openai.ts

@@ -15,7 +15,6 @@ Settings.llm = new OpenAI({
 });
 Settings.embedModel = new HuggingFaceEmbedding({
   modelType: "BAAI/bge-small-en-v1.5",
-  quantized: false,
 });
 Settings.callbackManager.on("llm-tool-call", (event) => {
   console.log(event.detail);

e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,65 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.113
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.112
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.111
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.110
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.109
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
+## 0.0.108
+
+### Patch Changes
+
+- llamaindex@0.8.12
+
+## 0.0.107
+
+### Patch Changes
+
+- llamaindex@0.8.11
+
+## 0.0.106
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+
+## 0.0.105
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+
 ## 0.0.104
 
 ### Patch Changes

e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.104",
+  "version": "0.0.113",
   "type": "module",
   "private": true,
   "scripts": {

e2e/fixtures/llm/ollama.ts

@@ -0,0 +1,3 @@
+import { OpenAI } from "./openai.js";
+
+export class Ollama extends OpenAI {}

e2e/mock-module.js

@@ -15,7 +15,17 @@ export async function resolve(specifier, context, nextResolve) {
   const targetUrl = fileURLToPath(result.url).replace(/\.js$/, ".ts");
   let relativePath = relative(packageDistDir, targetUrl);
   // todo: make it more generic if we have more sub modules fixtures in the future
-  if (relativePath.startsWith("../../llm/openai")) {
+  if (relativePath.startsWith("../../llm/anthropic")) {
+    relativePath = relativePath.replace(
+      "../../llm/ollama/dist/index.ts",
+      "llm/anthropic.ts",
+    );
+  } else if (relativePath.startsWith("../../llm/ollama")) {
+    relativePath = relativePath.replace(
+      "../../llm/ollama/dist/index.ts",
+      "llm/ollama.ts",
+    );
+  } else if (relativePath.startsWith("../../llm/openai")) {
     relativePath = relativePath.replace(
       "../../llm/openai/dist/index.ts",
       "llm/openai.ts",

e2e/node/embedding/clip.e2e.ts

@@ -64,7 +64,7 @@ await test("clip embedding", async (t) => {
   });
 
   await t.test("custom transformer", async () => {
-    const transformers = await import("@xenova/transformers");
+    const transformers = await import("@huggingface/transformers");
     const getter = test.mock.fn((t, k, r) => {
       return Reflect.get(t, k, r);
     });

e2e/node/ollama.e2e.ts

@@ -0,0 +1,35 @@
+import { Ollama } from "@llamaindex/ollama";
+import assert from "node:assert";
+import { test } from "node:test";
+import { getWeatherTool } from "./fixtures/tools.js";
+import { mockLLMEvent } from "./utils.js";
+
+await test("ollama", async (t) => {
+  await mockLLMEvent(t, "ollama");
+  await t.test("ollama function call", async (t) => {
+    const llm = new Ollama({
+      model: "llama3.2",
+    });
+    const chatResponse = await llm.chat({
+      messages: [
+        {
+          role: "user",
+          content: "What is the weather in Paris?",
+        },
+      ],
+      tools: [getWeatherTool],
+    });
+    if (
+      chatResponse.message.options &&
+      "toolCall" in chatResponse.message.options
+    ) {
+      assert.equal(chatResponse.message.options.toolCall.length, 1);
+      assert.equal(
+        chatResponse.message.options.toolCall[0]!.name,
+        getWeatherTool.metadata.name,
+      );
+    } else {
+      throw new Error("Expected tool calls in response");
+    }
+  });
+});

e2e/node/openai.e2e.ts

@@ -167,6 +167,7 @@ For questions about more specific sections, please use the vector_tool.`,
   const mockCall = t.mock.fn(({ query }: { query: string }) => {
     return originalCall({ query });
   });
+  // @ts-expect-error what?
   queryEngineTools[1]!.call = mockCall;
 
   const toolMapping = SimpleToolNodeMapping.fromObjects(queryEngineTools);

e2e/node/snapshot/ollama.snap

@@ -0,0 +1,37 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "role": "user",
+          "content": "What is the weather in Paris?"
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "message": {
+          "role": "assistant",
+          "content": "",
+          "options": {
+            "toolCall": [
+              {
+                "name": "getWeather",
+                "input": {
+                  "city": "Paris"
+                },
+                "id": "5d198775-5268-4552-993b-9ecb4425385b"
+              }
+            ]
+          }
+        },
+        "raw": null
+      }
+    }
+  ],
+  "llmEventStream": []
+}

e2e/package.json

@@ -12,10 +12,11 @@
     "@faker-js/faker": "^9.2.0",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*",
+    "@llamaindex/ollama": "workspace:*",
     "@llamaindex/openai": "workspace:*",
     "@types/node": "^22.9.0",
     "@types/pg": "^8.11.8",
-    "@xenova/transformers": "^2.17.2",
+    "@huggingface/transformers": "^3.0.2",
     "consola": "^3.2.3",
     "dotenv": "^16.4.5",
     "llamaindex": "workspace:*",

examples/CHANGELOG.md

@@ -1,5 +1,38 @@
 # examples
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [ee20c44]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+  - @llamaindex/workflow@0.0.6
+  - llamaindex@0.8.13
+  - @llamaindex/readers@1.0.11
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [ea92b69]
+- Updated dependencies [fadc8b8]
+  - @llamaindex/workflow@0.0.5
+
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+- Updated dependencies [d89ebe0]
+- Updated dependencies [fd8c882]
+- Updated dependencies [fd8c882]
+  - llamaindex@0.8.10
+  - @llamaindex/core@0.4.7
+  - @llamaindex/workflow@0.0.4
+  - @llamaindex/readers@1.0.8
+
 ## 0.0.13
 
 ### Patch Changes

examples/anthropic/prompt-caching.ts

@@ -0,0 +1,38 @@
+import { Anthropic } from "llamaindex";
+
+async function main() {
+  const anthropic = new Anthropic({
+    model: "claude-3-5-sonnet-20241022",
+  });
+
+  const entireBook = await fetch(
+    "https://www.gutenberg.org/files/1342/1342-0.txt",
+  ).then((response) => response.text());
+
+  const response = await anthropic.chat({
+    messages: [
+      {
+        content:
+          "You are an AI assistant tasked with analyzing literary works. Your goal is to provide insightful commentary on themes, characters, and writing style.\n",
+        role: "system",
+      },
+      {
+        content: entireBook,
+        role: "system",
+        options: {
+          cache_control: {
+            type: "ephemeral",
+          },
+        },
+      },
+      {
+        content: "analyze the major themes in Pride and Prejudice.",
+        role: "user",
+      },
+    ],
+  });
+
+  console.log(response.message.content);
+}
+
+main().catch(console.error);

examples/cosmosdb/queryVectorData.ts

@@ -83,14 +83,6 @@ async function query() {
     });
   }
 
-  // configure the Azure CosmosDB NoSQL Vector Store
-  const dbConfig: AzureCosmosDBNoSQLConfig = {
-    client: cosmosClient,
-    databaseName,
-    containerName,
-    flatMetadata: false,
-  };
-
   // use Azure CosmosDB as a vectorStore, docStore, and indexStore
   const { vectorStore, docStore, indexStore } = await initializeStores();
 

examples/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@llamaindex/examples",
   "private": true,
-  "version": "0.0.13",
+  "version": "0.0.16",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@azure/cosmos": "^4.1.1",
     "@azure/identity": "^4.4.1",
     "@datastax/astra-db-ts": "^1.4.1",
-    "@llamaindex/core": "^0.4.5",
-    "@llamaindex/readers": "^1.0.6",
-    "@llamaindex/workflow": "^0.0.3",
+    "@llamaindex/core": "^0.4.10",
+    "@llamaindex/readers": "^1.0.11",
+    "@llamaindex/workflow": "^0.0.6",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^3.0.2",
     "@vercel/postgres": "^0.10.0",
@@ -18,7 +18,7 @@
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.8.8",
+    "llamaindex": "^0.8.13",
     "mongodb": "^6.7.0",
     "pathe": "^1.1.2",
     "postgres": "^3.4.4"

examples/vectorIndexLocal.ts

@@ -14,7 +14,6 @@ Settings.llm = new Ollama({
 
 Settings.embedModel = new HuggingFaceEmbedding({
   modelType: "BAAI/bge-small-en-v1.5",
-  quantized: false,
 });
 
 async function main() {

examples/vllm.ts

@@ -0,0 +1,16 @@
+import { VLLM } from "llamaindex";
+
+const llm = new VLLM({
+  model: "NousResearch/Meta-Llama-3-8B-Instruct",
+});
+
+const response = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: "Hello?",
+    },
+  ],
+});
+
+console.log(response.message.content);

examples/workflow/app-creator.ts

@@ -1,14 +1,19 @@
 import {
-  Context,
+  HandlerContext,
   StartEvent,
   StopEvent,
   Workflow,
   WorkflowEvent,
-} from "@llamaindex/core/workflow";
+} from "@llamaindex/workflow";
 import { OpenAI } from "llamaindex";
 
 const MAX_REVIEWS = 3;
 
+type Context = {
+  specification: string;
+  numberReviews: number;
+};
+
 // Using the o1-preview model (see https://platform.openai.com/docs/guides/reasoning?reasoning-prompt-examples=coding-planning)
 const llm = new OpenAI({ model: "o1-preview", temperature: 1 });
 
@@ -20,7 +25,9 @@ stores the question/answer pair in the database.`;
 
 // Create custom event types
 export class MessageEvent extends WorkflowEvent<{ msg: string }> {}
+
 export class CodeEvent extends WorkflowEvent<{ code: string }> {}
+
 export class ReviewEvent extends WorkflowEvent<{
   review: string;
   code: string;
@@ -34,12 +41,13 @@ const truncate = (str: string) => {
 };
 
 // the architect is responsible for writing the structure and the initial code based on the specification
-const architect = async (context: Context, ev: StartEvent) => {
-  // get the specification from the start event and save it to context
-  context.set("specification", ev.data.input);
-  const spec = context.get("specification");
+const architect = async (
+  context: HandlerContext<Context>,
+  _: StartEvent<string>,
+) => {
+  const spec = context.data.specification;
   // write a message to send an update to the user
-  context.writeEventToStream(
+  context.sendEvent(
     new MessageEvent({
       msg: `Writing app using this specification: ${truncate(spec)}`,
     }),
@@ -50,13 +58,13 @@ const architect = async (context: Context, ev: StartEvent) => {
 };
 
 // the coder is responsible for updating the code based on the review
-const coder = async (context: Context, ev: ReviewEvent) => {
+const coder = async (context: HandlerContext<Context>, ev: ReviewEvent) => {
   // get the specification from the context
-  const spec = context.get("specification");
+  const spec = context.data.specification;
   // get the latest review and code
   const { review, code } = ev.data;
   // write a message to send an update to the user
-  context.writeEventToStream(
+  context.sendEvent(
     new MessageEvent({
       msg: `Update code based on review: ${truncate(review)}`,
     }),
@@ -67,32 +75,35 @@ const coder = async (context: Context, ev: ReviewEvent) => {
 };
 
 // the reviewer is responsible for reviewing the code and providing feedback
-const reviewer = async (context: Context, ev: CodeEvent) => {
+const reviewer = async (context: HandlerContext<Context>, ev: CodeEvent) => {
   // get the specification from the context
-  const spec = context.get("specification");
+  const spec = context.data.specification;
   // get latest code from the event
   const { code } = ev.data;
   // update and check the number of reviews
-  const numberReviews = context.get("numberReviews", 0) + 1;
-  context.set("numberReviews", numberReviews);
-  if (numberReviews > MAX_REVIEWS) {
+  context.data.numberReviews++;
+  if (context.data.numberReviews > MAX_REVIEWS) {
     // the we've done this too many times - return the code
-    context.writeEventToStream(
+    context.sendEvent(
       new MessageEvent({
-        msg: `Already reviewed ${numberReviews - 1} times, stopping!`,
+        msg: `Already reviewed ${
+          context.data.numberReviews - 1
+        } times, stopping!`,
       }),
     );
     return new StopEvent({ result: code });
   }
   // write a message to send an update to the user
-  context.writeEventToStream(
-    new MessageEvent({ msg: `Review #${numberReviews}: ${truncate(code)}` }),
+  context.sendEvent(
+    new MessageEvent({
+      msg: `Review #${context.data.numberReviews}: ${truncate(code)}`,
+    }),
   );
   const prompt = `Review this code: <code>${code}</code>. Check if the code quality and whether it correctly implements this specification: <spec>${spec}</spec>. If you're satisfied, just return 'Looks great', nothing else. If not, return a review with a list of changes you'd like to see.`;
   const review = (await llm.complete({ prompt })).text;
   if (review.includes("Looks great")) {
     // the reviewer is satisfied with the code, let's return the review
-    context.writeEventToStream(
+    context.sendEvent(
       new MessageEvent({
         msg: `Reviewer says: ${review}`,
       }),
@@ -103,20 +114,44 @@ const reviewer = async (context: Context, ev: CodeEvent) => {
   return new ReviewEvent({ review, code });
 };
 
-const codeAgent = new Workflow({ validate: true });
-codeAgent.addStep(StartEvent, architect, { outputs: CodeEvent });
-codeAgent.addStep(ReviewEvent, coder, { outputs: CodeEvent });
-codeAgent.addStep(CodeEvent, reviewer, { outputs: ReviewEvent });
+const codeAgent = new Workflow<Context, string, string>();
+codeAgent.addStep(
+  {
+    inputs: [StartEvent<string>],
+    outputs: [CodeEvent],
+  },
+  architect,
+);
+codeAgent.addStep(
+  {
+    inputs: [ReviewEvent],
+    outputs: [CodeEvent],
+  },
+  coder,
+);
+codeAgent.addStep(
+  {
+    inputs: [CodeEvent],
+    outputs: [ReviewEvent, StopEvent],
+  },
+  reviewer,
+);
 
 // Usage
 async function main() {
-  const run = codeAgent.run(specification);
-  for await (const event of codeAgent.streamEvents()) {
-    const msg = (event as MessageEvent).data.msg;
-    console.log(`${msg}\n`);
+  const run = codeAgent.run(specification).with({
+    specification,
+    numberReviews: 0,
+  });
+  for await (const event of run) {
+    if (event instanceof MessageEvent) {
+      const msg = (event as MessageEvent).data.msg;
+      console.log(`${msg}\n`);
+    } else if (event instanceof StopEvent) {
+      const result = (event as StopEvent<string>).data;
+      console.log("Final code:\n", result);
+    }
   }
-  const result = await run;
-  console.log("Final code:\n", result.data.result);
 }
 
 main().catch(console.error);

examples/workflow/conditional.ts

@@ -1,10 +1,10 @@
 import {
-  Context,
+  HandlerContext,
   StartEvent,
   StopEvent,
   Workflow,
   WorkflowEvent,
-} from "@llamaindex/core/workflow";
+} from "@llamaindex/workflow";
 import { OpenAI } from "llamaindex";
 
 // Create LLM instance
@@ -12,59 +12,77 @@ const llm = new OpenAI();
 
 // Create custom event types
 export class JokeEvent extends WorkflowEvent<{ joke: string }> {}
+
 export class CritiqueEvent extends WorkflowEvent<{ critique: string }> {}
+
 export class AnalysisEvent extends WorkflowEvent<{ analysis: string }> {}
 
-const generateJoke = async (_context: Context, ev: StartEvent) => {
-  const prompt = `Write your best joke about ${ev.data.input}.`;
+const generateJoke = async (_: unknown, ev: StartEvent<string>) => {
+  const prompt = `Write your best joke about ${ev.data}.`;
   const response = await llm.complete({ prompt });
   return new JokeEvent({ joke: response.text });
 };
 
-const critiqueJoke = async (_context: Context, ev: JokeEvent) => {
+const critiqueJoke = async (_: unknown, ev: JokeEvent) => {
   const prompt = `Give a thorough critique of the following joke: ${ev.data.joke}`;
   const response = await llm.complete({ prompt });
   return new CritiqueEvent({ critique: response.text });
 };
 
-const analyzeJoke = async (_context: Context, ev: JokeEvent) => {
+const analyzeJoke = async (_: unknown, ev: JokeEvent) => {
   const prompt = `Give a thorough analysis of the following joke: ${ev.data.joke}`;
   const response = await llm.complete({ prompt });
   return new AnalysisEvent({ analysis: response.text });
 };
 
 const reportJoke = async (
-  context: Context,
-  ev: AnalysisEvent | CritiqueEvent,
+  context: HandlerContext,
+  ev1: AnalysisEvent,
+  ev2: CritiqueEvent,
 ) => {
-  const events = context.collectEvents(ev, [AnalysisEvent, CritiqueEvent]);
-  if (!events) {
-    return;
-  }
-  const subPrompts = events.map((event) => {
-    if (event instanceof AnalysisEvent) {
-      return `Analysis: ${event.data.analysis}`;
-    } else if (event instanceof CritiqueEvent) {
-      return `Critique: ${event.data.critique}`;
-    }
-    return "";
-  });
+  const subPrompts = [ev1.data.analysis, ev2.data.critique];
 
-  const prompt = `Based on the following information about a joke:\n${subPrompts.join("\n")}\nProvide a comprehensive report on the joke's quality and impact.`;
+  const prompt = `Based on the following information about a joke:\n${subPrompts.join(
+    "\n",
+  )}\nProvide a comprehensive report on the joke's quality and impact.`;
   const response = await llm.complete({ prompt });
-  return new StopEvent({ result: response.text });
+  return new StopEvent(response.text);
 };
 
-const jokeFlow = new Workflow();
-jokeFlow.addStep(StartEvent, generateJoke);
-jokeFlow.addStep(JokeEvent, critiqueJoke);
-jokeFlow.addStep(JokeEvent, analyzeJoke);
-jokeFlow.addStep([AnalysisEvent, CritiqueEvent], reportJoke);
+const jokeFlow = new Workflow<unknown, string, string>();
+jokeFlow.addStep(
+  {
+    inputs: [StartEvent<string>],
+    outputs: [JokeEvent],
+  },
+  generateJoke,
+);
+jokeFlow.addStep(
+  {
+    inputs: [JokeEvent],
+    outputs: [CritiqueEvent],
+  },
+  critiqueJoke,
+);
+jokeFlow.addStep(
+  {
+    inputs: [JokeEvent],
+    outputs: [AnalysisEvent],
+  },
+  analyzeJoke,
+);
+jokeFlow.addStep(
+  {
+    inputs: [AnalysisEvent, CritiqueEvent],
+    outputs: [StopEvent<string>],
+  },
+  reportJoke,
+);
 
 // Usage
 async function main() {
   const result = await jokeFlow.run("pirates");
-  console.log(result.data.result);
+  console.log(result.data);
 }
 
 main().catch(console.error);

examples/workflow/joke.ts

@@ -1,10 +1,9 @@
 import {
-  Context,
   StartEvent,
   StopEvent,
   Workflow,
   WorkflowEvent,
-} from "@llamaindex/core/workflow";
+} from "@llamaindex/workflow";
 import { OpenAI } from "llamaindex";
 
 // Create LLM instance
@@ -13,26 +12,38 @@ const llm = new OpenAI();
 // Create a custom event type
 export class JokeEvent extends WorkflowEvent<{ joke: string }> {}
 
-const generateJoke = async (_context: Context, ev: StartEvent) => {
-  const prompt = `Write your best joke about ${ev.data.input}.`;
+const generateJoke = async (_: unknown, ev: StartEvent<string>) => {
+  const prompt = `Write your best joke about ${ev.data}.`;
   const response = await llm.complete({ prompt });
   return new JokeEvent({ joke: response.text });
 };
 
-const critiqueJoke = async (_context: Context, ev: JokeEvent) => {
+const critiqueJoke = async (_: unknown, ev: JokeEvent) => {
   const prompt = `Give a thorough critique of the following joke: ${ev.data.joke}`;
   const response = await llm.complete({ prompt });
-  return new StopEvent({ result: response.text });
+  return new StopEvent(response.text);
 };
 
-const jokeFlow = new Workflow({ verbose: true });
-jokeFlow.addStep(StartEvent, generateJoke);
-jokeFlow.addStep(JokeEvent, critiqueJoke);
+const jokeFlow = new Workflow<unknown, string, string>();
+jokeFlow.addStep(
+  {
+    inputs: [StartEvent<string>],
+    outputs: [JokeEvent],
+  },
+  generateJoke,
+);
+jokeFlow.addStep(
+  {
+    inputs: [JokeEvent],
+    outputs: [StopEvent<string>],
+  },
+  critiqueJoke,
+);
 
 // Usage
 async function main() {
   const result = await jokeFlow.run("pirates");
-  console.log(result.data.result);
+  console.log(result.data);
 }
 
 main().catch(console.error);

examples/workflow/stream-events.ts

@@ -1,10 +1,10 @@
 import {
-  Context,
+  HandlerContext,
   StartEvent,
   StopEvent,
   Workflow,
   WorkflowEvent,
-} from "@llamaindex/core/workflow";
+} from "@llamaindex/workflow";
 import { OpenAI } from "llamaindex";
 
 // Create LLM instance
@@ -12,38 +12,55 @@ const llm = new OpenAI();
 
 // Create custom event types
 export class JokeEvent extends WorkflowEvent<{ joke: string }> {}
+
 export class MessageEvent extends WorkflowEvent<{ msg: string }> {}
 
-const generateJoke = async (context: Context, ev: StartEvent) => {
-  context.writeEventToStream(
-    new MessageEvent({ msg: `Generating a joke about: ${ev.data.input}` }),
+const generateJoke = async (context: HandlerContext, ev: StartEvent) => {
+  context.sendEvent(
+    new MessageEvent({ msg: `Generating a joke about: ${ev.data}` }),
   );
-  const prompt = `Write your best joke about ${ev.data.input}.`;
+  const prompt = `Write your best joke about ${ev.data}.`;
   const response = await llm.complete({ prompt });
   return new JokeEvent({ joke: response.text });
 };
 
-const critiqueJoke = async (context: Context, ev: JokeEvent) => {
-  context.writeEventToStream(
+const critiqueJoke = async (context: HandlerContext, ev: JokeEvent) => {
+  context.sendEvent(
     new MessageEvent({ msg: `Write a critique of this joke: ${ev.data.joke}` }),
   );
   const prompt = `Give a thorough critique of the following joke: ${ev.data.joke}`;
   const response = await llm.complete({ prompt });
-  return new StopEvent({ result: response.text });
+  return new StopEvent(response.text);
 };
 
 const jokeFlow = new Workflow();
-jokeFlow.addStep(StartEvent, generateJoke);
-jokeFlow.addStep(JokeEvent, critiqueJoke);
+jokeFlow.addStep(
+  {
+    inputs: [StartEvent<string>],
+    outputs: [JokeEvent],
+  },
+  generateJoke,
+);
+jokeFlow.addStep(
+  {
+    inputs: [JokeEvent],
+    outputs: [StopEvent<string>],
+  },
+  critiqueJoke,
+);
 
 // Usage
 async function main() {
   const run = jokeFlow.run("pirates");
-  for await (const event of jokeFlow.streamEvents()) {
-    console.log((event as MessageEvent).data.msg);
+  for await (const event of run) {
+    if (event instanceof MessageEvent) {
+      console.log("Message:");
+      console.log((event as MessageEvent).data.msg);
+    } else if (event instanceof StopEvent) {
+      console.log("Result:");
+      console.log((event as StopEvent<string>).data);
+    }
   }
-  const result = await run;
-  console.log(result.data.result);
 }
 
 main().catch(console.error);

examples/workflow/timeout.ts

@@ -1,19 +1,21 @@
-import {
-  Context,
-  StartEvent,
-  StopEvent,
-  Workflow,
-} from "@llamaindex/core/workflow";
+import { StartEvent, StopEvent, Workflow } from "@llamaindex/workflow";
 
-const longRunning = async (_context: Context, ev: StartEvent) => {
+const longRunning = async (_: unknown, ev: StartEvent<string>) => {
   await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait for 2 seconds
-  return new StopEvent({ result: "We waited 2 seconds" });
+  return new StopEvent("We waited 2 seconds");
 };
 
 async function timeout() {
-  const workflow = new Workflow({ verbose: true, timeout: 1 });
-  workflow.addStep(StartEvent, longRunning);
-  // This will timeout
+  const workflow = new Workflow<unknown, string, string>({
+    timeout: 1,
+  });
+  workflow.addStep(
+    {
+      inputs: [StartEvent<string>],
+      outputs: [StopEvent<string>],
+    },
+    longRunning,
+  );
   try {
     await workflow.run("Let's start");
   } catch (error) {
@@ -23,14 +25,23 @@ async function timeout() {
 
 async function notimeout() {
   // Increase timeout to 3 seconds - no timeout
-  const workflow = new Workflow({ verbose: true, timeout: 3 });
-  workflow.addStep(StartEvent, longRunning);
+  const workflow = new Workflow<unknown, string, string>({
+    timeout: 3,
+  });
+  workflow.addStep(
+    {
+      inputs: [StartEvent<string>],
+      outputs: [StopEvent<string>],
+    },
+    longRunning,
+  );
   const result = await workflow.run("Let's start");
-  console.log(result.data.result);
+  console.log(result.data);
 }
 
 async function main() {
   await timeout();
+  console.log("---");
   await notimeout();
 }
 

examples/workflow/validation.ts

@@ -1,10 +1,9 @@
 import {
-  Context,
   StartEvent,
   StopEvent,
   Workflow,
   WorkflowEvent,
-} from "@llamaindex/core/workflow";
+} from "@llamaindex/workflow";
 import { OpenAI } from "llamaindex";
 
 // Create LLM instance
@@ -13,40 +12,66 @@ const llm = new OpenAI();
 // Create a custom event type
 export class JokeEvent extends WorkflowEvent<{ joke: string }> {}
 
-const generateJoke = async (_context: Context, ev: StartEvent) => {
-  const prompt = `Write your best joke about ${ev.data.input}.`;
+const generateJoke = async (_: unknown, ev: StartEvent<string>) => {
+  const prompt = `Write your best joke about ${ev.data}.`;
   const response = await llm.complete({ prompt });
   return new JokeEvent({ joke: response.text });
 };
 
-const critiqueJoke = async (_context: Context, ev: JokeEvent) => {
+const critiqueJoke = async (_: unknown, ev: JokeEvent) => {
   const prompt = `Give a thorough critique of the following joke: ${ev.data.joke}`;
   const response = await llm.complete({ prompt });
-  return new StopEvent({ result: response.text });
+  return new StopEvent(response.text);
 };
 
 async function validateFails() {
   try {
-    const jokeFlow = new Workflow({ verbose: true, validate: true });
-    jokeFlow.addStep(StartEvent, generateJoke, { outputs: StopEvent });
-    jokeFlow.addStep(JokeEvent, critiqueJoke, { outputs: StopEvent });
-    await jokeFlow.run("pirates");
+    const jokeFlow = new Workflow();
+    jokeFlow.addStep(
+      {
+        inputs: [StartEvent<string>],
+        outputs: [StopEvent<string>],
+      },
+      // @ts-expect-error outputs should be JokeEvent
+      generateJoke,
+    );
+    jokeFlow.addStep(
+      {
+        inputs: [JokeEvent],
+        outputs: [StopEvent],
+      },
+      critiqueJoke,
+    );
+    await jokeFlow.run("pirates").strict();
   } catch (e) {
     console.error("Validation failed:", e);
   }
 }
 
 async function validate() {
-  const jokeFlow = new Workflow({ verbose: true, validate: true });
-  jokeFlow.addStep(StartEvent, generateJoke, { outputs: JokeEvent });
-  jokeFlow.addStep(JokeEvent, critiqueJoke, { outputs: StopEvent });
-  const result = await jokeFlow.run("pirates");
-  console.log(result.data.result);
+  const jokeFlow = new Workflow();
+  jokeFlow.addStep(
+    {
+      inputs: [StartEvent<string>],
+      outputs: [JokeEvent],
+    },
+    generateJoke,
+  );
+  jokeFlow.addStep(
+    {
+      inputs: [JokeEvent],
+      outputs: [StopEvent<string>],
+    },
+    critiqueJoke,
+  );
+  const result = await jokeFlow.run("pirates").strict();
+  console.log(result.data);
 }
 
 // Usage
 async function main() {
   await validateFails();
+  console.log("---");
   await validate();
 }
 

package.json

@@ -35,12 +35,6 @@
     "typescript-eslint": "^8.13.0"
   },
   "packageManager": "pnpm@9.12.3",
-  "pnpm": {
-    "overrides": {
-      "trim": "1.0.1",
-      "protobufjs": "7.2.6"
-    }
-  },
   "lint-staged": {
     "(!apps/docs/i18n/**/docusaurus-plugin-content-docs/current/api/*).{js,jsx,ts,tsx,md}": "prettier --write"
   }

packages/autotool/CHANGELOG.md

@@ -1,5 +1,65 @@
 # @llamaindex/autotool
 
+## 5.0.17
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 5.0.16
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 5.0.15
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 5.0.14
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 5.0.13
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
+## 5.0.12
+
+### Patch Changes
+
+- llamaindex@0.8.12
+
+## 5.0.11
+
+### Patch Changes
+
+- llamaindex@0.8.11
+
+## 5.0.10
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+
+## 5.0.9
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+
 ## 5.0.8
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,74 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.60
+
+### Patch Changes
+
+- llamaindex@0.8.17
+- @llamaindex/autotool@5.0.17
+
+## 0.0.59
+
+### Patch Changes
+
+- llamaindex@0.8.16
+- @llamaindex/autotool@5.0.16
+
+## 0.0.58
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+  - @llamaindex/autotool@5.0.15
+
+## 0.0.57
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+  - @llamaindex/autotool@5.0.14
+
+## 0.0.56
+
+### Patch Changes
+
+- llamaindex@0.8.13
+- @llamaindex/autotool@5.0.13
+
+## 0.0.55
+
+### Patch Changes
+
+- llamaindex@0.8.12
+- @llamaindex/autotool@5.0.12
+
+## 0.0.54
+
+### Patch Changes
+
+- llamaindex@0.8.11
+- @llamaindex/autotool@5.0.11
+
+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+  - @llamaindex/autotool@5.0.10
+
+## 0.0.52
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+  - @llamaindex/autotool@5.0.9
+
 ## 0.0.51
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.51"
+  "version": "0.0.60"
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,74 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.104
+
+### Patch Changes
+
+- llamaindex@0.8.17
+- @llamaindex/autotool@5.0.17
+
+## 0.1.103
+
+### Patch Changes
+
+- llamaindex@0.8.16
+- @llamaindex/autotool@5.0.16
+
+## 0.1.102
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+  - @llamaindex/autotool@5.0.15
+
+## 0.1.101
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+  - @llamaindex/autotool@5.0.14
+
+## 0.1.100
+
+### Patch Changes
+
+- llamaindex@0.8.13
+- @llamaindex/autotool@5.0.13
+
+## 0.1.99
+
+### Patch Changes
+
+- llamaindex@0.8.12
+- @llamaindex/autotool@5.0.12
+
+## 0.1.98
+
+### Patch Changes
+
+- llamaindex@0.8.11
+- @llamaindex/autotool@5.0.11
+
+## 0.1.97
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+  - @llamaindex/autotool@5.0.10
+
+## 0.1.96
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+  - @llamaindex/autotool@5.0.9
+
 ## 0.1.95
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.95",
+  "version": "0.1.104",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "5.0.8",
+  "version": "5.0.17",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,50 @@
 # @llamaindex/cloud
 
+## 2.0.11
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 2.0.10
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
+## 2.0.9
+
+### Patch Changes
+
+- Updated dependencies [7ae6eaa]
+  - @llamaindex/core@0.4.9
+
+## 2.0.8
+
+### Patch Changes
+
+- Updated dependencies [f865c98]
+  - @llamaindex/core@0.4.8
+
+## 2.0.7
+
+### Patch Changes
+
+- Updated dependencies [d89ebe0]
+- Updated dependencies [fd8c882]
+  - @llamaindex/core@0.4.7
+
+## 2.0.6
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+  - @llamaindex/env@0.1.20
+  - @llamaindex/core@0.4.6
+
 ## 2.0.5
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "2.0.5",
+  "version": "2.0.11",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,5 +1,50 @@
 # @llamaindex/community
 
+## 0.0.69
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 0.0.68
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
+## 0.0.67
+
+### Patch Changes
+
+- Updated dependencies [7ae6eaa]
+  - @llamaindex/core@0.4.9
+
+## 0.0.66
+
+### Patch Changes
+
+- Updated dependencies [f865c98]
+  - @llamaindex/core@0.4.8
+
+## 0.0.65
+
+### Patch Changes
+
+- Updated dependencies [d89ebe0]
+- Updated dependencies [fd8c882]
+  - @llamaindex/core@0.4.7
+
+## 0.0.64
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+  - @llamaindex/env@0.1.20
+  - @llamaindex/core@0.4.6
+
 ## 0.0.63
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.63",
+  "version": "0.0.69",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -1,5 +1,44 @@
 # @llamaindex/core
 
+## 0.4.11
+
+### Patch Changes
+
+- 6d22fa2: Get PromptTemplate template variables at run-time
+
+## 0.4.10
+
+### Patch Changes
+
+- a7b0ac3: fix: update tool call llm type
+- c69605f: feat: add async support to BaseChatStore and BaseChatStoreMemory
+
+## 0.4.9
+
+### Patch Changes
+
+- 7ae6eaa: feat: allow pass `additionalChatOptions` to agent
+
+## 0.4.8
+
+### Patch Changes
+
+- f865c98: feat: async get message on chat store
+
+## 0.4.7
+
+### Patch Changes
+
+- d89ebe0: feat: better support for zod schema
+- fd8c882: chore: add warning on legacy workflow API
+
+## 0.4.6
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+  - @llamaindex/env@0.1.20
+
 ## 0.4.5
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.4.5",
+  "version": "0.4.11",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./agent": {

packages/core/src/agent/base.ts

@@ -106,11 +106,17 @@ export type AgentRunnerParams<
   >
     ? AdditionalMessageOptions
     : never,
+  AdditionalChatOptions extends object = object,
 > = {
   llm: AI;
   chatHistory: ChatMessage<AdditionalMessageOptions>[];
   systemPrompt: MessageContent | null;
-  runner: AgentWorker<AI, Store, AdditionalMessageOptions>;
+  runner: AgentWorker<
+    AI,
+    Store,
+    AdditionalMessageOptions,
+    AdditionalChatOptions
+  >;
   tools:
     | BaseToolWithCall[]
     | ((query: MessageContent) => Promise<BaseToolWithCall[]>);
@@ -125,6 +131,7 @@ export type AgentParamsBase<
   >
     ? AdditionalMessageOptions
     : never,
+  AdditionalChatOptions extends object = object,
 > =
   | {
       llm?: AI;
@@ -132,6 +139,7 @@ export type AgentParamsBase<
       systemPrompt?: MessageContent;
       verbose?: boolean;
       tools: BaseToolWithCall[];
+      additionalChatOptions?: AdditionalChatOptions;
     }
   | {
       llm?: AI;
@@ -139,6 +147,7 @@ export type AgentParamsBase<
       systemPrompt?: MessageContent;
       verbose?: boolean;
       toolRetriever: ObjectRetriever<BaseToolWithCall>;
+      additionalChatOptions?: AdditionalChatOptions;
     };
 
 /**
@@ -153,21 +162,36 @@ export abstract class AgentWorker<
   >
     ? AdditionalMessageOptions
     : never,
+  AdditionalChatOptions extends object = object,
 > {
-  #taskSet = new Set<TaskStep<AI, Store, AdditionalMessageOptions>>();
-  abstract taskHandler: TaskHandler<AI, Store, AdditionalMessageOptions>;
+  #taskSet = new Set<
+    TaskStep<AI, Store, AdditionalMessageOptions, AdditionalChatOptions>
+  >();
+  abstract taskHandler: TaskHandler<
+    AI,
+    Store,
+    AdditionalMessageOptions,
+    AdditionalChatOptions
+  >;
 
   public createTask(
     query: MessageContent,
-    context: AgentTaskContext<AI, Store, AdditionalMessageOptions>,
-  ): ReadableStream<TaskStepOutput<AI, Store, AdditionalMessageOptions>> {
+    context: AgentTaskContext<
+      AI,
+      Store,
+      AdditionalMessageOptions,
+      AdditionalChatOptions
+    >,
+  ): ReadableStream<
+    TaskStepOutput<AI, Store, AdditionalMessageOptions, AdditionalChatOptions>
+  > {
     context.store.messages.push({
       role: "user",
       content: query,
     });
     const taskOutputStream = createTaskOutputStream(this.taskHandler, context);
     return new ReadableStream<
-      TaskStepOutput<AI, Store, AdditionalMessageOptions>
+      TaskStepOutput<AI, Store, AdditionalMessageOptions, AdditionalChatOptions>
     >({
       start: async (controller) => {
         for await (const stepOutput of taskOutputStream) {
@@ -176,7 +200,8 @@ export abstract class AgentWorker<
             let currentStep: TaskStep<
               AI,
               Store,
-              AdditionalMessageOptions
+              AdditionalMessageOptions,
+              AdditionalChatOptions
             > | null = stepOutput.taskStep;
             while (currentStep) {
               this.#taskSet.delete(currentStep);
@@ -227,6 +252,7 @@ export abstract class AgentRunner<
   >
     ? AdditionalMessageOptions
     : never,
+  AdditionalChatOptions extends object = object,
 > extends BaseChatEngine {
   readonly #llm: AI;
   readonly #tools:
@@ -234,7 +260,12 @@ export abstract class AgentRunner<
     | ((query: MessageContent) => Promise<BaseToolWithCall[]>);
   readonly #systemPrompt: MessageContent | null = null;
   #chatHistory: ChatMessage<AdditionalMessageOptions>[];
-  readonly #runner: AgentWorker<AI, Store, AdditionalMessageOptions>;
+  readonly #runner: AgentWorker<
+    AI,
+    Store,
+    AdditionalMessageOptions,
+    AdditionalChatOptions
+  >;
   readonly #verbose: boolean;
 
   // create extra store
@@ -245,7 +276,7 @@ export abstract class AgentRunner<
   }
 
   static defaultTaskHandler: TaskHandler<LLM> = async (step, enqueueOutput) => {
-    const { llm, getTools, stream } = step.context;
+    const { llm, getTools, stream, additionalChatOptions } = step.context;
     const lastMessage = step.context.store.messages.at(-1)!.content;
     const tools = await getTools(lastMessage);
     if (!stream) {
@@ -253,8 +284,9 @@ export abstract class AgentRunner<
         stream,
         tools,
         messages: [...step.context.store.messages],
+        additionalChatOptions,
       });
-      await stepTools<LLM>({
+      await stepTools({
         response,
         tools,
         step,
@@ -265,6 +297,7 @@ export abstract class AgentRunner<
         stream,
         tools,
         messages: [...step.context.store.messages],
+        additionalChatOptions,
       });
       await stepToolsStreaming<LLM>({
         response,
@@ -276,7 +309,12 @@ export abstract class AgentRunner<
   };
 
   protected constructor(
-    params: AgentRunnerParams<AI, Store, AdditionalMessageOptions>,
+    params: AgentRunnerParams<
+      AI,
+      Store,
+      AdditionalMessageOptions,
+      AdditionalChatOptions
+    >,
   ) {
     super();
     const { llm, chatHistory, systemPrompt, runner, tools, verbose } = params;
@@ -330,6 +368,7 @@ export abstract class AgentRunner<
     stream: boolean = false,
     verbose: boolean | undefined = undefined,
     chatHistory?: ChatMessage<AdditionalMessageOptions>[],
+    additionalChatOptions?: AdditionalChatOptions,
   ) {
     const initialMessages = [...(chatHistory ?? this.#chatHistory)];
     if (this.#systemPrompt !== null) {
@@ -348,6 +387,7 @@ export abstract class AgentRunner<
       stream,
       toolCallCount: 0,
       llm: this.#llm,
+      additionalChatOptions: additionalChatOptions ?? {},
       getTools: (message) => this.getTools(message),
       store: {
         ...this.createStore(),
@@ -365,13 +405,29 @@ export abstract class AgentRunner<
     });
   }
 
-  async chat(params: NonStreamingChatEngineParams): Promise<EngineResponse>;
   async chat(
-    params: StreamingChatEngineParams,
+    params: NonStreamingChatEngineParams<
+      AdditionalMessageOptions,
+      AdditionalChatOptions
+    >,
+  ): Promise<EngineResponse>;
+  async chat(
+    params: StreamingChatEngineParams<
+      AdditionalMessageOptions,
+      AdditionalChatOptions
+    >,
   ): Promise<ReadableStream<EngineResponse>>;
   @wrapEventCaller
   async chat(
-    params: NonStreamingChatEngineParams | StreamingChatEngineParams,
+    params:
+      | NonStreamingChatEngineParams<
+          AdditionalMessageOptions,
+          AdditionalChatOptions
+        >
+      | StreamingChatEngineParams<
+          AdditionalMessageOptions,
+          AdditionalChatOptions
+        >,
   ): Promise<EngineResponse | ReadableStream<EngineResponse>> {
     let chatHistory: ChatMessage<AdditionalMessageOptions>[] = [];
 
@@ -388,6 +444,7 @@ export abstract class AgentRunner<
       !!params.stream,
       false,
       chatHistory,
+      params.chatOptions,
     );
     for await (const stepOutput of task) {
       // update chat history for each round
@@ -398,7 +455,12 @@ export abstract class AgentRunner<
           return output.pipeThrough(
             new TransformStream<EngineResponse>({
               transform(chunk, controller) {
-                controller.enqueue(EngineResponse.fromChatResponseChunk(chunk));
+                controller.enqueue(
+                  EngineResponse.fromChatResponseChunk(
+                    chunk,
+                    chunk.sourceNodes,
+                  ),
+                );
               },
             }),
           );

packages/core/src/agent/llm.ts

@@ -4,24 +4,66 @@ import { ObjectRetriever } from "../objects";
 import { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
 import { validateAgentParams } from "./utils.js";
 
-type LLMParamsBase = AgentParamsBase<LLM>;
+type LLMParamsBase<
+  AI extends LLM,
+  AdditionalMessageOptions extends object = AI extends LLM<
+    object,
+    infer AdditionalMessageOptions
+  >
+    ? AdditionalMessageOptions
+    : never,
+  AdditionalChatOptions extends object = object,
+> = AgentParamsBase<AI, AdditionalMessageOptions, AdditionalChatOptions>;
 
-type LLMParamsWithTools = LLMParamsBase & {
+type LLMParamsWithTools<
+  AI extends LLM,
+  AdditionalMessageOptions extends object = AI extends LLM<
+    object,
+    infer AdditionalMessageOptions
+  >
+    ? AdditionalMessageOptions
+    : never,
+  AdditionalChatOptions extends object = object,
+> = LLMParamsBase<AI, AdditionalMessageOptions, AdditionalChatOptions> & {
   tools: BaseToolWithCall[];
 };
 
-type LLMParamsWithToolRetriever = LLMParamsBase & {
+type LLMParamsWithToolRetriever<
+  AI extends LLM,
+  AdditionalMessageOptions extends object = AI extends LLM<
+    object,
+    infer AdditionalMessageOptions
+  >
+    ? AdditionalMessageOptions
+    : never,
+  AdditionalChatOptions extends object = object,
+> = LLMParamsBase<AI, AdditionalMessageOptions, AdditionalChatOptions> & {
   toolRetriever: ObjectRetriever<BaseToolWithCall>;
 };
 
-export type LLMAgentParams = LLMParamsWithTools | LLMParamsWithToolRetriever;
+export type LLMAgentParams<
+  AI extends LLM,
+  AdditionalMessageOptions extends object = AI extends LLM<
+    object,
+    infer AdditionalMessageOptions
+  >
+    ? AdditionalMessageOptions
+    : never,
+  AdditionalChatOptions extends object = object,
+> =
+  | LLMParamsWithTools<AI, AdditionalMessageOptions, AdditionalChatOptions>
+  | LLMParamsWithToolRetriever<
+      AI,
+      AdditionalMessageOptions,
+      AdditionalChatOptions
+    >;
 
 export class LLMAgentWorker extends AgentWorker<LLM> {
   taskHandler = AgentRunner.defaultTaskHandler;
 }
 
 export class LLMAgent extends AgentRunner<LLM> {
-  constructor(params: LLMAgentParams) {
+  constructor(params: LLMAgentParams<LLM>) {
     validateAgentParams(params);
     const llm = params.llm ?? (Settings.llm ? (Settings.llm as LLM) : null);
     if (!llm)

packages/core/src/agent/types.ts

@@ -19,6 +19,7 @@ export type AgentTaskContext<
   >
     ? AdditionalMessageOptions
     : never,
+  AdditionalChatOptions extends object = object,
 > = {
   readonly stream: boolean;
   readonly toolCallCount: number;
@@ -26,6 +27,7 @@ export type AgentTaskContext<
   readonly getTools: (
     input: MessageContent,
   ) => BaseToolWithCall[] | Promise<BaseToolWithCall[]>;
+  readonly additionalChatOptions: Partial<AdditionalChatOptions>;
   shouldContinue: (
     taskStep: Readonly<TaskStep<Model, Store, AdditionalMessageOptions>>,
   ) => boolean;
@@ -45,13 +47,26 @@ export type TaskStep<
   >
     ? AdditionalMessageOptions
     : never,
+  AdditionalChatOptions extends object = object,
 > = {
   id: UUID;
-  context: AgentTaskContext<Model, Store, AdditionalMessageOptions>;
+  context: AgentTaskContext<
+    Model,
+    Store,
+    AdditionalMessageOptions,
+    AdditionalChatOptions
+  >;
 
   // linked list
-  prevStep: TaskStep<Model, Store, AdditionalMessageOptions> | null;
-  nextSteps: Set<TaskStep<Model, Store, AdditionalMessageOptions>>;
+  prevStep: TaskStep<
+    Model,
+    Store,
+    AdditionalMessageOptions,
+    AdditionalChatOptions
+  > | null;
+  nextSteps: Set<
+    TaskStep<Model, Store, AdditionalMessageOptions, AdditionalChatOptions>
+  >;
 };
 
 export type TaskStepOutput<
@@ -63,8 +78,14 @@ export type TaskStepOutput<
   >
     ? AdditionalMessageOptions
     : never,
+  AdditionalChatOptions extends object = object,
 > = {
-  taskStep: TaskStep<Model, Store, AdditionalMessageOptions>;
+  taskStep: TaskStep<
+    Model,
+    Store,
+    AdditionalMessageOptions,
+    AdditionalChatOptions
+  >;
   // output shows the response to the user
   output:
     | ChatResponse<AdditionalMessageOptions>
@@ -81,10 +102,16 @@ export type TaskHandler<
   >
     ? AdditionalMessageOptions
     : never,
+  AdditionalChatOptions extends object = object,
 > = (
-  step: TaskStep<Model, Store, AdditionalMessageOptions>,
+  step: TaskStep<Model, Store, AdditionalMessageOptions, AdditionalChatOptions>,
   enqueueOutput: (
-    taskOutput: TaskStepOutput<Model, Store, AdditionalMessageOptions>,
+    taskOutput: TaskStepOutput<
+      Model,
+      Store,
+      AdditionalMessageOptions,
+      AdditionalChatOptions
+    >,
   ) => void,
 ) => Promise<void>;
 

packages/core/src/chat-engine/base.ts

@@ -16,14 +16,18 @@ export interface BaseChatEngineParams<
 
 export interface StreamingChatEngineParams<
   AdditionalMessageOptions extends object = object,
+  AdditionalChatOptions extends object = object,
 > extends BaseChatEngineParams<AdditionalMessageOptions> {
   stream: true;
+  chatOptions?: AdditionalChatOptions;
 }
 
 export interface NonStreamingChatEngineParams<
   AdditionalMessageOptions extends object = object,
+  AdditionalChatOptions extends object = object,
 > extends BaseChatEngineParams<AdditionalMessageOptions> {
   stream?: false;
+  chatOptions?: AdditionalChatOptions;
 }
 
 export abstract class BaseChatEngine {

packages/core/src/llms/base.ts

@@ -1,5 +1,4 @@
-import { streamConverter } from "../utils";
-import { extractText } from "../utils/llms";
+import { extractText, streamConverter } from "../utils";
 import type {
   ChatResponse,
   ChatResponseChunk,
@@ -67,6 +66,8 @@ export abstract class BaseLLM<
 
 export abstract class ToolCallLLM<
   AdditionalChatOptions extends object = object,
-> extends BaseLLM<AdditionalChatOptions, ToolCallLLMMessageOptions> {
+  AdditionalMessageOptions extends
+    ToolCallLLMMessageOptions = ToolCallLLMMessageOptions,
+> extends BaseLLM<AdditionalChatOptions, AdditionalMessageOptions> {
   abstract supportToolCall: boolean;
 }

packages/core/src/memory/base.ts

@@ -65,19 +65,21 @@ export abstract class BaseChatStoreMemory<
     super();
   }
 
-  getAllMessages(): ChatMessage<AdditionalMessageOptions>[] {
+  getAllMessages():
+    | ChatMessage<AdditionalMessageOptions>[]
+    | Promise<ChatMessage<AdditionalMessageOptions>[]> {
     return this.chatStore.getMessages(this.chatStoreKey);
   }
 
-  put(messages: ChatMessage<AdditionalMessageOptions>) {
+  put(messages: ChatMessage<AdditionalMessageOptions>): void | Promise<void> {
     this.chatStore.addMessage(this.chatStoreKey, messages);
   }
 
-  set(messages: ChatMessage<AdditionalMessageOptions>[]) {
+  set(messages: ChatMessage<AdditionalMessageOptions>[]): void | Promise<void> {
     this.chatStore.setMessages(this.chatStoreKey, messages);
   }
 
-  reset() {
+  reset(): void | Promise<void> {
     this.chatStore.deleteMessages(this.chatStoreKey);
   }
 }

packages/core/src/memory/chat-memory-buffer.ts

@@ -33,11 +33,11 @@ export class ChatMemoryBuffer<
     }
   }
 
-  getMessages(
+  async getMessages(
     transientMessages?: ChatMessage<AdditionalMessageOptions>[] | undefined,
     initialTokenCount: number = 0,
   ) {
-    const messages = this.getAllMessages();
+    const messages = await this.getAllMessages();
 
     if (initialTokenCount > this.tokenLimit) {
       throw new Error("Initial token count exceeds token limit");

packages/core/src/prompts/base.ts

@@ -32,6 +32,10 @@ export abstract class BasePromptTemplate<
   const Vars extends readonly string[] = string[],
 > {
   metadata: Metadata = {};
+  /**
+   * Set of template variables used in the prompt template. Used for type hints only.
+   * To get the list of template variables used in the prompt at run-time, use the `vars` method.
+   */
   templateVars: Set<string> = new Set();
   options: Partial<Record<TemplatesVar[number] | (string & {}), string>> = {};
   outputParser: BaseOutputParser | undefined;
@@ -223,4 +227,13 @@ export class PromptTemplate<
   get template(): Template {
     return this.#template;
   }
+
+  /**
+   * Returns all the template variables used in the prompt template.
+   */
+  vars(): string[] {
+    const template = this.template;
+    const matches = template.match(/\{([^}]+)\}/g) || [];
+    return [...new Set(matches.map((match) => match.slice(1, -1)))];
+  }
 }

packages/core/src/storage/chat-store/base-chat-store.ts

@@ -7,7 +7,11 @@ export abstract class BaseChatStore<
     key: string,
     messages: ChatMessage<AdditionalMessageOptions>[],
   ): void;
-  abstract getMessages(key: string): ChatMessage<AdditionalMessageOptions>[];
+  abstract getMessages(
+    key: string,
+  ):
+    | ChatMessage<AdditionalMessageOptions>[]
+    | Promise<ChatMessage<AdditionalMessageOptions>[]>;
   abstract addMessage(
     key: string,
     message: ChatMessage<AdditionalMessageOptions>,
@@ -15,5 +19,7 @@ export abstract class BaseChatStore<
   ): void;
   abstract deleteMessages(key: string): void;
   abstract deleteMessage(key: string, idx: number): void;
-  abstract getKeys(): IterableIterator<string>;
+  abstract getKeys():
+    | IterableIterator<string>
+    | Promise<IterableIterator<string>>;
 }

packages/core/src/tools/function-tool.ts

@@ -4,18 +4,12 @@ import { zodToJsonSchema } from "zod-to-json-schema";
 import type { JSONValue } from "../global";
 import type { BaseTool, ToolMetadata } from "../llms";
 
-const kOriginalFn = Symbol("originalFn");
-
 export class FunctionTool<T, R extends JSONValue | Promise<JSONValue>>
   implements BaseTool<T>
 {
-  [kOriginalFn]?: (input: T) => R;
-
   #fn: (input: T) => R;
-  #metadata: ToolMetadata<JSONSchemaType<T>>;
-  // todo: for the future, we can use zod to validate the input parameters
-  // eslint-disable-next-line no-unused-private-class-members
-  #zodType: z.ZodType<T> | null = null;
+  readonly #metadata: ToolMetadata<JSONSchemaType<T>>;
+  readonly #zodType: z.ZodType<T> | null = null;
   constructor(
     fn: (input: T) => R,
     metadata: ToolMetadata<JSONSchemaType<T>>,
@@ -32,6 +26,12 @@ export class FunctionTool<T, R extends JSONValue | Promise<JSONValue>>
     fn: (input: T) => JSONValue | Promise<JSONValue>,
     schema: ToolMetadata<JSONSchemaType<T>>,
   ): FunctionTool<T, JSONValue | Promise<JSONValue>>;
+  static from<R extends z.ZodType>(
+    fn: (input: z.infer<R>) => JSONValue | Promise<JSONValue>,
+    schema: Omit<ToolMetadata, "parameters"> & {
+      parameters: R;
+    },
+  ): FunctionTool<z.infer<R>, JSONValue | Promise<JSONValue>>;
   static from<T, R extends z.ZodType<T>>(
     fn: (input: T) => JSONValue | Promise<JSONValue>,
     schema: Omit<ToolMetadata, "parameters"> & {
@@ -40,15 +40,15 @@ export class FunctionTool<T, R extends JSONValue | Promise<JSONValue>>
   ): FunctionTool<T, JSONValue>;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   static from(fn: any, schema: any): any {
-    if (schema.parameter instanceof z.ZodSchema) {
-      const jsonSchema = zodToJsonSchema(schema.parameter);
+    if (schema.parameters instanceof z.ZodSchema) {
+      const jsonSchema = zodToJsonSchema(schema.parameters);
       return new FunctionTool(
         fn,
         {
           ...schema,
           parameters: jsonSchema,
         },
-        schema.parameter,
+        schema.parameters,
       );
     }
     return new FunctionTool(fn, schema);
@@ -58,7 +58,15 @@ export class FunctionTool<T, R extends JSONValue | Promise<JSONValue>>
     return this.#metadata as BaseTool<T>["metadata"];
   }
 
-  call(input: T) {
+  call = (input: T) => {
+    if (this.#zodType) {
+      const result = this.#zodType.safeParse(input);
+      if (result.success) {
+        return this.#fn.call(null, result.data);
+      } else {
+        console.warn(result.error.errors);
+      }
+    }
     return this.#fn.call(null, input);
-  }
+  };
 }

packages/core/src/workflow/workflow.ts

@@ -13,6 +13,8 @@ export type StepFunction<T extends WorkflowEvent = WorkflowEvent> = (
 
 type EventTypeParam = EventTypes | EventTypes[];
 
+let once = false;
+
 export class Workflow {
   #steps: Map<
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -29,8 +31,20 @@ export class Workflow {
       verbose?: boolean;
       timeout?: number;
       validate?: boolean;
+      ignoreDeprecatedWarning?: boolean;
     } = {},
   ) {
+    if (!once && !params.ignoreDeprecatedWarning) {
+      console.warn(
+        "@llamaindex/core/workflow is going to use the new workflow API in the next major version.",
+        "Please update your imports to @llamaindex/workflow",
+      );
+      console.warn(
+        "See https://ts.llamaindex.ai/docs/llamaindex/guide/workflow for more information",
+      );
+      once = true;
+    }
+
     this.#verbose = params.verbose ?? false;
     this.#timeout = params.timeout ?? null;
     this.#validate = params.validate ?? false;

packages/core/tests/memory/chat-memory-buffer.test.ts

@@ -19,7 +19,7 @@ describe("ChatMemoryBuffer", () => {
     expect(buffer.tokenLimit).toBe(500);
   });
 
-  test("getMessages returns all messages when under token limit", () => {
+  test("getMessages returns all messages when under token limit", async () => {
     const messages: ChatMessage[] = [
       { role: "user", content: "Hello" },
       { role: "assistant", content: "Hi there!" },
@@ -30,11 +30,11 @@ describe("ChatMemoryBuffer", () => {
       chatHistory: messages,
     });
 
-    const result = buffer.getMessages();
+    const result = await buffer.getMessages();
     expect(result).toEqual(messages);
   });
 
-  test("getMessages truncates messages when over token limit", () => {
+  test("getMessages truncates messages when over token limit", async () => {
     const messages: ChatMessage[] = [
       { role: "user", content: "This is a long message" },
       { role: "assistant", content: "This is also a long reply" },
@@ -45,11 +45,11 @@ describe("ChatMemoryBuffer", () => {
       chatHistory: messages,
     });
 
-    const result = buffer.getMessages();
+    const result = await buffer.getMessages();
     expect(result).toEqual([{ role: "user", content: "Short" }]);
   });
 
-  test("getMessages handles input messages", () => {
+  test("getMessages handles input messages", async () => {
     const storedMessages: ChatMessage[] = [
       { role: "user", content: "Hello" },
       { role: "assistant", content: "Hi there!" },
@@ -62,13 +62,13 @@ describe("ChatMemoryBuffer", () => {
     const inputMessages: ChatMessage[] = [
       { role: "user", content: "New message" },
     ];
-    const result = buffer.getMessages(inputMessages);
+    const result = await buffer.getMessages(inputMessages);
     expect(result).toEqual([...inputMessages, ...storedMessages]);
   });
 
   test("getMessages throws error when initial token count exceeds limit", () => {
     const buffer = new ChatMemoryBuffer({ tokenLimit: 10 });
-    expect(() => buffer.getMessages(undefined, 20)).toThrow(
+    expect(async () => buffer.getMessages(undefined, 20)).rejects.toThrow(
       "Initial token count exceeds token limit",
     );
   });

packages/core/tests/prompts.test.ts

@@ -158,4 +158,15 @@ describe("PromptTemplate", () => {
     const formatted = prompt.format({ text: "world", foo: "bar" });
     expect(formatted).toBe("hello world bar\noutput_instruction");
   });
+
+  test("should return all unique template vars of a prompt", () => {
+    const prompt = new PromptTemplate({
+      template: "Hello {name}! Your age is {age}. Nice to meet you {name}!",
+    });
+
+    const vars = prompt.vars();
+    expect(vars).toHaveLength(2);
+    expect(vars).toContain("name");
+    expect(vars).toContain("age");
+  });
 });

packages/env/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/env
 
+## 0.1.20
+
+### Patch Changes
+
+- 4fc001c: chore: bump `@huggingface/transformers`
+
+  Upgrade to v3, please read https://github.com/huggingface/transformers.js/releases/tag/3.0.0 for more information.
+
 ## 0.1.19
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.19",
+  "version": "0.1.20",
   "type": "module",
   "types": "dist/index.d.ts",
   "module": "dist/index.js",
@@ -124,7 +124,7 @@
   "devDependencies": {
     "@types/node": "^22.9.0",
     "@types/readable-stream": "^4.0.15",
-    "@xenova/transformers": "^2.17.2",
+    "@huggingface/transformers": "^3.0.2",
     "bunchee": "5.6.1",
     "gpt-tokenizer": "^2.6.0",
     "pathe": "^1.1.2",
@@ -132,7 +132,7 @@
   },
   "peerDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@xenova/transformers": "^2.17.2",
+    "@huggingface/transformers": "^3.0.2",
     "gpt-tokenizer": "^2.5.0",
     "js-tiktoken": "^1.0.12",
     "pathe": "^1.1.2"
@@ -141,7 +141,7 @@
     "@aws-crypto/sha256-js": {
       "optional": true
     },
-    "@xenova/transformers": {
+    "@huggingface/transformers": {
       "optional": true
     },
     "pathe": {

packages/env/src/internal/multi-model/browser.ts

@@ -8,8 +8,10 @@ export {
 export async function loadTransformers(onLoad: OnLoad) {
   if (getTransformers() === null) {
     setTransformers(
-      // @ts-expect-error no type
-      await import("https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2"),
+      await import(
+        // @ts-expect-error no type
+        "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.2"
+      ),
     );
   } else {
     return getTransformers()!;

packages/env/src/internal/multi-model/node.ts

@@ -8,7 +8,7 @@ export {
 
 export async function loadTransformers(onLoad: OnLoad) {
   if (getTransformers() === null) {
-    setTransformers(await import("@xenova/transformers"));
+    setTransformers(await import("@huggingface/transformers"));
   } else {
     return getTransformers()!;
   }

packages/env/src/internal/multi-model/non-node.ts

@@ -9,7 +9,7 @@ export async function loadTransformers(onLoad: OnLoad) {
   if (getTransformers() === null) {
     /**
      * If you see this warning, it means that the current environment does not support the transformer.
-     *  because "@xeonva/transformers" highly depends on Node.js APIs.
+     *  because "@huggingface/transformers" highly depends on Node.js APIs.
      *
      * One possible solution is to fix their implementation to make it work in the non-Node.js environment,
      *  but it's not worth the effort because Edge Runtime and Cloudflare Workers are not the for heavy Machine Learning task.
@@ -17,14 +17,14 @@ export async function loadTransformers(onLoad: OnLoad) {
      * Or you can provide an RPC server that runs the transformer in a Node.js environment.
      * Or you just run the code in a Node.js environment.
      *
-     * Refs: https://github.com/xenova/transformers.js/issues/309
+     * Refs: https://github.com/huggingface/transformers.js/issues/309
      */
     console.warn(
-      '"@xenova/transformers" is not officially supported in this environment, some features may not work as expected.',
+      '"@huggingface/transformers" is not officially supported in this environment, some features may not work as expected.',
     );
     setTransformers(
       // @ts-expect-error no type
-      await import("@xenova/transformers/dist/transformers"),
+      await import("@huggingface/transformers/dist/transformers.js"),
     );
   } else {
     return getTransformers()!;

packages/env/src/internal/multi-model/shared.ts

@@ -1,17 +1,17 @@
-let transformer: typeof import("@xenova/transformers") | null = null;
+let transformer: typeof import("@huggingface/transformers") | null = null;
 
 export function getTransformers() {
   return transformer;
 }
 
-export function setTransformers(t: typeof import("@xenova/transformers")) {
+export function setTransformers(t: typeof import("@huggingface/transformers")) {
   transformer = t;
 }
 
 export type OnLoad = (
-  transformer: typeof import("@xenova/transformers"),
+  transformer: typeof import("@huggingface/transformers"),
 ) => void;
 
 export type LoadTransformerEvent = {
-  transformer: typeof import("@xenova/transformers");
+  transformer: typeof import("@huggingface/transformers");
 };

packages/experimental/CHANGELOG.md

@@ -1,5 +1,65 @@
 # @llamaindex/experimental
 
+## 0.0.129
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.128
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.127
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.126
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.125
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
+## 0.0.124
+
+### Patch Changes
+
+- llamaindex@0.8.12
+
+## 0.0.123
+
+### Patch Changes
+
+- llamaindex@0.8.11
+
+## 0.0.122
+
+### Patch Changes
+
+- Updated dependencies [f066e50]
+  - llamaindex@0.8.10
+
+## 0.0.121
+
+### Patch Changes
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - llamaindex@0.8.9
+
 ## 0.0.120
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.120",
+  "version": "0.0.129",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,159 @@
 # llamaindex
 
+## 0.8.17
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+  - @llamaindex/cloud@2.0.11
+  - @llamaindex/node-parser@0.0.12
+  - @llamaindex/anthropic@0.0.20
+  - @llamaindex/clip@0.0.20
+  - @llamaindex/deepinfra@0.0.20
+  - @llamaindex/huggingface@0.0.20
+  - @llamaindex/ollama@0.0.27
+  - @llamaindex/openai@0.1.36
+  - @llamaindex/portkey-ai@0.0.20
+  - @llamaindex/replicate@0.0.20
+  - @llamaindex/readers@1.0.13
+  - @llamaindex/groq@0.0.35
+  - @llamaindex/vllm@0.0.6
+
+## 0.8.16
+
+### Patch Changes
+
+- Updated dependencies [e60328b]
+  - @llamaindex/readers@1.0.12
+
+## 0.8.15
+
+### Patch Changes
+
+- 3d503cb: Update azure cosmos db
+- 5dae534: fix: propagate queryStr to concrete vectorStore
+
+## 0.8.14
+
+### Patch Changes
+
+- 630b425: feat: add Azure CosmosDB NoSql Chat store
+
+## 0.8.13
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/anthropic@0.0.19
+  - @llamaindex/core@0.4.10
+  - @llamaindex/cloud@2.0.10
+  - @llamaindex/node-parser@0.0.11
+  - @llamaindex/clip@0.0.19
+  - @llamaindex/deepinfra@0.0.19
+  - @llamaindex/huggingface@0.0.19
+  - @llamaindex/ollama@0.0.26
+  - @llamaindex/openai@0.1.35
+  - @llamaindex/portkey-ai@0.0.19
+  - @llamaindex/replicate@0.0.19
+  - @llamaindex/readers@1.0.11
+  - @llamaindex/groq@0.0.34
+  - @llamaindex/vllm@0.0.5
+
+## 0.8.12
+
+### Patch Changes
+
+- Updated dependencies [7ae6eaa]
+  - @llamaindex/core@0.4.9
+  - @llamaindex/openai@0.1.34
+  - @llamaindex/cloud@2.0.9
+  - @llamaindex/node-parser@0.0.10
+  - @llamaindex/anthropic@0.0.18
+  - @llamaindex/clip@0.0.18
+  - @llamaindex/deepinfra@0.0.18
+  - @llamaindex/huggingface@0.0.18
+  - @llamaindex/ollama@0.0.25
+  - @llamaindex/portkey-ai@0.0.18
+  - @llamaindex/replicate@0.0.18
+  - @llamaindex/readers@1.0.10
+  - @llamaindex/groq@0.0.33
+  - @llamaindex/vllm@0.0.4
+
+## 0.8.11
+
+### Patch Changes
+
+- Updated dependencies [f865c98]
+  - @llamaindex/core@0.4.8
+  - @llamaindex/cloud@2.0.8
+  - @llamaindex/node-parser@0.0.9
+  - @llamaindex/anthropic@0.0.17
+  - @llamaindex/clip@0.0.17
+  - @llamaindex/deepinfra@0.0.17
+  - @llamaindex/huggingface@0.0.17
+  - @llamaindex/ollama@0.0.24
+  - @llamaindex/openai@0.1.33
+  - @llamaindex/portkey-ai@0.0.17
+  - @llamaindex/replicate@0.0.17
+  - @llamaindex/readers@1.0.9
+  - @llamaindex/groq@0.0.32
+  - @llamaindex/vllm@0.0.3
+
+## 0.8.10
+
+### Patch Changes
+
+- f066e50: feat: vllm support
+- Updated dependencies [f066e50]
+- Updated dependencies [d89ebe0]
+- Updated dependencies [fd8c882]
+  - @llamaindex/vllm@0.0.2
+  - @llamaindex/core@0.4.7
+  - @llamaindex/cloud@2.0.7
+  - @llamaindex/node-parser@0.0.8
+  - @llamaindex/anthropic@0.0.16
+  - @llamaindex/clip@0.0.16
+  - @llamaindex/deepinfra@0.0.16
+  - @llamaindex/huggingface@0.0.16
+  - @llamaindex/ollama@0.0.23
+  - @llamaindex/openai@0.1.32
+  - @llamaindex/portkey-ai@0.0.16
+  - @llamaindex/replicate@0.0.16
+  - @llamaindex/readers@1.0.8
+  - @llamaindex/groq@0.0.31
+
+## 0.8.9
+
+### Patch Changes
+
+- 4fc001c: chore: bump `@huggingface/transformers`
+
+  Upgrade to v3, please read https://github.com/huggingface/transformers.js/releases/tag/3.0.0 for more information.
+
+- 4d4cd8a: feat: support ollama tool call
+
+  Note that `OllamaEmbedding` now is not the subclass of `Ollama`.
+
+- Updated dependencies [4fc001c]
+- Updated dependencies [4d4cd8a]
+  - @llamaindex/env@0.1.20
+  - @llamaindex/clip@0.0.15
+  - @llamaindex/huggingface@0.0.15
+  - @llamaindex/ollama@0.0.22
+  - @llamaindex/cloud@2.0.6
+  - @llamaindex/core@0.4.6
+  - @llamaindex/node-parser@0.0.7
+  - @llamaindex/anthropic@0.0.15
+  - @llamaindex/deepinfra@0.0.15
+  - @llamaindex/groq@0.0.30
+  - @llamaindex/openai@0.1.31
+  - @llamaindex/portkey-ai@0.0.15
+  - @llamaindex/replicate@0.0.15
+  - @llamaindex/readers@1.0.7
+
 ## 0.8.8
 
 ### Patch Changes

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.8.8",
+  "version": "0.8.17",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -29,7 +29,7 @@
     "@discoveryjs/json-ext": "^0.6.1",
     "@google-cloud/vertexai": "1.2.0",
     "@google/generative-ai": "0.12.0",
-    "@grpc/grpc-js": "^1.11.1",
+    "@grpc/grpc-js": "^1.12.2",
     "@llamaindex/anthropic": "workspace:*",
     "@llamaindex/clip": "workspace:*",
     "@llamaindex/cloud": "workspace:*",
@@ -44,6 +44,7 @@
     "@llamaindex/portkey-ai": "workspace:*",
     "@llamaindex/readers": "workspace:*",
     "@llamaindex/replicate": "workspace:*",
+    "@llamaindex/vllm": "workspace:*",
     "@mistralai/mistralai": "^1.0.4",
     "@mixedbread-ai/sdk": "^2.2.11",
     "@pinecone-database/pinecone": "^3.0.2",
@@ -85,10 +86,10 @@
     }
   },
   "devDependencies": {
+    "@huggingface/transformers": "^3.0.2",
     "@swc/cli": "^0.5.0",
     "@swc/core": "^1.7.22",
     "@vercel/postgres": "^0.10.0",
-    "@xenova/transformers": "^2.17.2",
     "concurrently": "^9.1.0",
     "glob": "^11.0.0",
     "pg": "^8.12.0",

packages/llamaindex/src/agent/index.ts

@@ -1,4 +1,9 @@
 export * from "@llamaindex/core/agent";
+export {
+  OllamaAgent,
+  OllamaAgentWorker,
+  type OllamaAgentParams,
+} from "@llamaindex/ollama";
 export {
   AnthropicAgent,
   AnthropicAgentWorker,
@@ -16,7 +21,6 @@ export {
   ReActAgent,
   type ReACTAgentParams,
 } from "./react.js";
-
 // todo: ParallelAgent
 // todo: CustomAgent
 // todo: ReactMultiModal

packages/llamaindex/src/embeddings/OllamaEmbedding.ts

@@ -1,7 +1 @@
-import type { BaseEmbedding } from "@llamaindex/core/embeddings";
-import { Ollama } from "@llamaindex/ollama";
-
-/**
- * OllamaEmbedding is an alias for Ollama that implements the BaseEmbedding interface.
- */
-export class OllamaEmbedding extends Ollama implements BaseEmbedding {}
+export { OllamaEmbedding } from "@llamaindex/ollama";

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -18,6 +18,7 @@ import {
   type NodeWithScore,
 } from "@llamaindex/core/schema";
 import type { BaseIndexStore } from "@llamaindex/core/storage/index-store";
+import { extractText } from "@llamaindex/core/utils";
 import type { ServiceContext } from "../../ServiceContext.js";
 import { nodeParserFromSettingsOrContext } from "../../Settings.js";
 import { RetrieverQueryEngine } from "../../engines/query/RetrieverQueryEngine.js";
@@ -449,8 +450,13 @@ export class VectorIndexRetriever extends BaseRetriever {
     filters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
     // convert string message to multi-modal format
+
+    let queryStr = query;
     if (typeof query === "string") {
-      query = [{ type: "text", text: query }];
+      queryStr = query;
+      query = [{ type: "text", text: queryStr }];
+    } else {
+      queryStr = extractText(query);
     }
     // overwrite embed model if specified, otherwise use the one from the vector store
     const embedModel = this.index.embedModel ?? vectorStore.embedModel;
@@ -460,6 +466,7 @@ export class VectorIndexRetriever extends BaseRetriever {
       const queryEmbedding = await embedModel.getQueryEmbedding(item);
       if (queryEmbedding) {
         const result = await vectorStore.query({
+          queryStr,
           queryEmbedding,
           mode: VectorStoreQueryMode.DEFAULT,
           similarityTopK: this.topK[type]!,

packages/llamaindex/src/llm/index.ts

@@ -1,3 +1,4 @@
+export { VLLM, type VLLMParams } from "@llamaindex/vllm";
 export {
   ALL_AVAILABLE_ANTHROPIC_LEGACY_MODELS,
   ALL_AVAILABLE_ANTHROPIC_MODELS,
@@ -6,7 +7,6 @@ export {
 } from "./anthropic.js";
 export { FireworksLLM } from "./fireworks.js";
 export { Gemini, GeminiSession } from "./gemini/base.js";
-
 export {
   GEMINI_MODEL,
   type GoogleGeminiSessionOptions,

packages/llamaindex/src/next.ts

@@ -22,7 +22,7 @@ export default function withLlamaIndex(config: any) {
   config.experimental.serverComponentsExternalPackages =
     config.experimental.serverComponentsExternalPackages ?? [];
   config.experimental.serverComponentsExternalPackages.push(
-    "@xenova/transformers",
+    "@huggingface/transformers",
   );
   const userWebpack = config.webpack;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any

packages/llamaindex/src/storage/chatStore/AzureCosmosNoSqlChatStore.ts

@@ -0,0 +1,345 @@
+import { CosmosClient, type Container, type Database } from "@azure/cosmos";
+import { DefaultAzureCredential, type TokenCredential } from "@azure/identity";
+import type {
+  ChatMessage,
+  MessageContent,
+  MessageType,
+} from "@llamaindex/core/llms";
+import { BaseChatStore } from "@llamaindex/core/storage/chat-store";
+import { getEnv } from "@llamaindex/env";
+
+const USER_AGENT_SUFFIX = "llamaindex-cdbnosql-chatstore-javascript";
+const DEFAULT_CHAT_DATABASE = "ChatMessagesDB";
+const DEFAULT_CHAT_CONTAINER = "ChatMessagesContainer";
+const DEFAULT_OFFER_THROUGHPUT = 400;
+
+function parseConnectionString(connectionString: string): {
+  endpoint: string;
+  key: string;
+} {
+  const parts = connectionString.split(";");
+  let endpoint = "";
+  let accountKey = "";
+
+  parts.forEach((part) => {
+    const [key, value] = part.split("=");
+    if (key && key.trim() === "AccountEndpoint") {
+      endpoint = value?.trim() ?? "";
+    } else if ((key ?? "").trim() === "AccountKey") {
+      accountKey = value?.trim() ?? "";
+    }
+  });
+
+  if (!endpoint || !accountKey) {
+    throw new Error(
+      "Invalid connection string: missing AccountEndpoint or AccountKey.",
+    );
+  }
+  return { endpoint, key: accountKey };
+}
+
+export interface AzureCosmosChatDatabaseProperties {
+  throughput?: number;
+}
+
+export interface AzureCosmosChatContainerProperties {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [key: string]: any;
+}
+
+export interface AzureCosmosNoSqlChatStoreConfig {
+  cosmosClient?: CosmosClient;
+  dbName?: string;
+  containerName?: string;
+  userId?: string;
+  sessionId?: string;
+  cosmosContainerProperties?: AzureCosmosChatContainerProperties;
+  cosmosDatabaseProperties?: AzureCosmosChatDatabaseProperties;
+  ttlInSeconds?: number;
+}
+
+export class AzureCosmosNoSqlChatStore<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatStore<AdditionalMessageOptions> {
+  private userId: string;
+  private ttl: number;
+  private cosmosClient: CosmosClient;
+  private database!: Database;
+  private container!: Container;
+  private initPromise?: Promise<void>;
+
+  private dbName: string;
+  private containerName: string;
+  private cosmosContainerProperties: AzureCosmosChatContainerProperties;
+  private cosmosDatabaseProperties: AzureCosmosChatDatabaseProperties;
+  private initialize: () => Promise<void>;
+
+  constructor({
+    cosmosClient,
+    dbName = DEFAULT_CHAT_DATABASE,
+    containerName = DEFAULT_CHAT_CONTAINER,
+    cosmosContainerProperties = { partitionKey: "/userId" },
+    cosmosDatabaseProperties = {},
+    ttlInSeconds = -1,
+  }: AzureCosmosNoSqlChatStoreConfig) {
+    super();
+    if (!cosmosClient) {
+      throw new Error(
+        "CosmosClient is required for AzureCosmosDBNoSQLChatStore initialization",
+      );
+    }
+    this.ttl = ttlInSeconds;
+    this.userId = cosmosContainerProperties.userId || "anonymous";
+    this.cosmosClient = cosmosClient;
+    this.dbName = dbName;
+    this.containerName = containerName;
+    this.cosmosContainerProperties = cosmosContainerProperties;
+    this.cosmosDatabaseProperties = cosmosDatabaseProperties;
+
+    this.initialize = () => {
+      if (this.initPromise === undefined) {
+        this.initPromise = this.init().catch((error) => {
+          console.error(
+            "Error during AzureCosmosDBNoSQLChatStore initialization",
+            error,
+          );
+        });
+      }
+      return this.initPromise;
+    };
+  }
+
+  client(): CosmosClient {
+    return this.cosmosClient;
+  }
+
+  // Asynchronous initialization method to create database and container
+  private async init(): Promise<void> {
+    // Set default throughput if not provided
+    const throughput =
+      this.cosmosDatabaseProperties?.throughput || DEFAULT_OFFER_THROUGHPUT;
+
+    // Create the database if it doesn't exist
+    const { database } = await this.cosmosClient.databases.createIfNotExists({
+      id: this.dbName,
+      throughput,
+    });
+    this.database = database;
+
+    // Create the container if it doesn't exist
+    const { container } = await this.database.containers.createIfNotExists({
+      id: this.containerName,
+      throughput: this.cosmosContainerProperties?.throughput,
+      partitionKey: "/userId",
+      indexingPolicy: this.cosmosContainerProperties?.indexingPolicy,
+      defaultTtl: this.ttl,
+      uniqueKeyPolicy: this.cosmosContainerProperties?.uniqueKeyPolicy,
+      conflictResolutionPolicy:
+        this.cosmosContainerProperties?.conflictResolutionPolicy,
+      computedProperties: this.cosmosContainerProperties?.computedProperties,
+    });
+    this.container = container;
+  }
+  /**
+   * Static method for creating an instance using a connection string.
+   * If no connection string is provided, it will attempt to use the env variable `AZURE_COSMOSDB_NOSQL_CONNECTION_STRING` as connection string.
+   * @returns Instance of AzureCosmosNoSqlKVStore
+   */
+  static fromConnectionString(
+    config: {
+      connectionString?: string;
+    } & AzureCosmosNoSqlChatStoreConfig = {},
+  ): AzureCosmosNoSqlChatStore {
+    const cosmosConnectionString =
+      config.connectionString ||
+      (getEnv("AZURE_COSMOSDB_NOSQL_CONNECTION_STRING") as string);
+    if (!cosmosConnectionString) {
+      throw new Error("Azure CosmosDB connection string must be provided");
+    }
+    const { endpoint, key } = parseConnectionString(cosmosConnectionString);
+    const cosmosClient = new CosmosClient({
+      endpoint,
+      key,
+      userAgentSuffix: USER_AGENT_SUFFIX,
+    });
+    return new AzureCosmosNoSqlChatStore({
+      ...config,
+      cosmosClient,
+    });
+  }
+
+  /**
+   * Static method for creating an instance using a account endpoint and key.
+   * If no endpoint and key  is provided, it will attempt to use the env variable `AZURE_COSMOSDB_NOSQL_ACCOUNT_ENDPOINT` as enpoint and `AZURE_COSMOSDB_NOSQL_ACCOUNT_KEY` as key.
+   * @returns Instance of AzureCosmosNoSqlKVStore
+   */
+  static fromAccountAndKey(
+    config: {
+      endpoint?: string;
+      key?: string;
+    } & AzureCosmosNoSqlChatStoreConfig = {},
+  ): AzureCosmosNoSqlChatStore {
+    const cosmosEndpoint =
+      config.endpoint ||
+      (getEnv("AZURE_COSMOSDB_NOSQL_ACCOUNT_ENDPOINT") as string);
+    const cosmosKey =
+      config.key || (getEnv("AZURE_COSMOSDB_NOSQL_ACCOUNT_KEY") as string);
+
+    if (!cosmosEndpoint || !cosmosKey) {
+      throw new Error(
+        "Azure CosmosDB account endpoint and key must be provided",
+      );
+    }
+    const cosmosClient = new CosmosClient({
+      endpoint: cosmosEndpoint,
+      key: cosmosKey,
+      userAgentSuffix: USER_AGENT_SUFFIX,
+    });
+    return new AzureCosmosNoSqlChatStore({
+      ...config,
+      cosmosClient,
+    });
+  }
+
+  /**
+   * Static method for creating an instance using AAD token.
+   * If no endpoint and credentials are provided, it will attempt to use the env variable `AZURE_COSMOSDB_NOSQL_ACCOUNT_ENDPOINT` as endpoint and use DefaultAzureCredential() as credentials.
+   * @returns Instance of AzureCosmosNoSqlKVStore
+   */
+  static fromAadToken(
+    config: {
+      endpoint?: string;
+      credential?: TokenCredential;
+    } & AzureCosmosNoSqlChatStoreConfig = {},
+  ): AzureCosmosNoSqlChatStore {
+    const cosmosEndpoint =
+      config.endpoint ||
+      (getEnv("AZURE_COSMOSDB_NOSQL_CONNECTION_STRING") as string);
+
+    if (!cosmosEndpoint) {
+      throw new Error("Azure CosmosDB account endpoint must be provided");
+    }
+    const credentials = config.credential ?? new DefaultAzureCredential();
+    const cosmosClient = new CosmosClient({
+      endpoint: cosmosEndpoint,
+      aadCredentials: credentials,
+      userAgentSuffix: USER_AGENT_SUFFIX,
+    });
+    return new AzureCosmosNoSqlChatStore({
+      ...config,
+      cosmosClient,
+    });
+  }
+
+  private convertToChatMessage(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    message: any,
+  ): ChatMessage<AdditionalMessageOptions> {
+    return {
+      content: message.content as MessageContent,
+      role: message.role as MessageType,
+      options: message.options as AdditionalMessageOptions,
+    } as ChatMessage<AdditionalMessageOptions>;
+  }
+
+  private convertToCosmosMessage(
+    message: ChatMessage<AdditionalMessageOptions>,
+  ): // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  any {
+    return {
+      content: message.content,
+      role: message.role,
+      options: message.options,
+    };
+  }
+
+  /**
+   * Set messages for a given key.
+   */
+  async setMessages(
+    key: string,
+    messages: ChatMessage<AdditionalMessageOptions>[],
+  ): Promise<void> {
+    await this.initialize();
+    const inputMessages = messages.map(this.convertToCosmosMessage);
+    await this.container.items.upsert({
+      id: key,
+      messages: inputMessages,
+      userId: this.userId,
+    });
+  }
+
+  /**
+   * Get messages for a given key.
+   */
+  async getMessages(
+    key: string,
+  ): Promise<ChatMessage<AdditionalMessageOptions>[]> {
+    await this.initialize();
+    const res = await this.container.item(key, this.userId).read();
+    const messageHistory = res?.resource?.messages ?? [];
+    const result = messageHistory.map(this.convertToChatMessage);
+    return result;
+  }
+
+  /**
+   * Add a message for a given key.
+   */
+  async addMessage(
+    key: string,
+    message: ChatMessage<AdditionalMessageOptions>,
+    idx?: number,
+  ): Promise<void> {
+    await this.initialize();
+    const res = await this.container.item(key, this.userId).read();
+    const messageHistory = res?.resource?.messages ?? [];
+    if (idx === undefined) {
+      messageHistory.push(this.convertToCosmosMessage(message));
+    } else {
+      messageHistory.splice(idx, 0, this.convertToCosmosMessage(message));
+    }
+    await this.setMessages(key, messageHistory);
+  }
+
+  /**
+   * Deletes all messages for a given key.
+   */
+  async deleteMessages(key: string): Promise<void> {
+    await this.initialize();
+    try {
+      await this.container.item(key, this.userId).delete();
+      // eslint-disable-next-line no-empty
+    } catch (e) {}
+  }
+
+  /**
+   * Deletes one message at idx index for a given key.
+   */
+  async deleteMessage(key: string, idx: number): Promise<void> {
+    await this.initialize();
+    const res = await this.container.item(key, this.userId).read();
+    const messageHistory = res?.resource?.messages ?? [];
+    if (idx >= 0 && idx < messageHistory.length) {
+      messageHistory.splice(idx, 1);
+      await this.setMessages(key, messageHistory);
+    }
+  }
+
+  /**
+   * Get all keys.
+   */
+  async getKeys(): Promise<IterableIterator<string>> {
+    await this.initialize();
+    const result = await this.container.items
+      .query("Select c.id from c")
+      .fetchAll();
+    const keys = result.resources.map((res: { id: string }) => res.id);
+
+    function* keyGenerator(): IterableIterator<string> {
+      for (const key of keys) {
+        yield key;
+      }
+    }
+    return keyGenerator();
+  }
+}

packages/llamaindex/src/storage/index.ts

@@ -2,6 +2,7 @@ export * from "@llamaindex/core/storage/chat-store";
 export * from "@llamaindex/core/storage/doc-store";
 export * from "@llamaindex/core/storage/index-store";
 export * from "@llamaindex/core/storage/kv-store";
+export * from "./chatStore/AzureCosmosNoSqlChatStore.js";
 export * from "./docStore/AzureCosmosNoSqlDocumentStore.js";
 export { PostgresDocumentStore } from "./docStore/PostgresDocumentStore.js";
 export { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js";

packages/llamaindex/src/storage/kvStore/AzureCosmosNoSqlKVStore.ts

@@ -3,7 +3,7 @@ import { Container, CosmosClient, Database } from "@azure/cosmos";
 import { DefaultAzureCredential, type TokenCredential } from "@azure/identity";
 import { BaseKVStore } from "@llamaindex/core/storage/kv-store";
 import { getEnv } from "@llamaindex/env";
-const USER_AGENT_SUFFIX = "LlamaIndex-CDBNoSQL-KVStore-JavaScript";
+const USER_AGENT_SUFFIX = "llamaindex-cdbnosql-kvstore-javascript";
 const DEFAULT_CHAT_DATABASE = "KVStoreDB";
 const DEFAULT_CHAT_CONTAINER = "KVStoreContainer";
 const DEFAULT_OFFER_THROUGHPUT = 400;
@@ -82,7 +82,7 @@ export class AzureCosmosNoSqlKVStore extends BaseKVStore {
     super();
     if (!cosmosClient) {
       throw new Error(
-        "CosmosClient is required for AzureCosmosDBNoSQLVectorStore initialization",
+        "CosmosClient is required for AzureCosmosDBNoSQLKVStore initialization",
       );
     }
     this.cosmosClient = cosmosClient;

packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts

@@ -55,8 +55,18 @@ export interface AzureCosmosDBNoSQLConfig
   readonly flatMetadata?: boolean;
   readonly idKey?: string;
 }
+/**
+ * Query options for the `AzureCosmosDBNoSQLVectorStore.query` method.
+ * @property includeEmbeddings - Whether to include the embeddings in the result. Default false
+ * @property includeVectorDistance - Whether to include the vector distance in the result. Default true
+ * @property whereClause - The where clause to use in the query. While writing this clause, use `c` as the alias for the container and do not include the `WHERE` keyword.
+ */
+export interface AzureCosmosQueryOptions {
+  includeVectorDistance?: boolean;
+  whereClause?: string;
+}
 
-const USER_AGENT_SUFFIX = "LlamaIndex-CDBNoSQL-VectorStore-JavaScript";
+const USER_AGENT_SUFFIX = "llamaindex-cdbnosql-vectorstore-javascript";
 
 const DEFAULT_VECTOR_EMBEDDING_POLICY = {
   vectorEmbeddings: [
@@ -98,6 +108,22 @@ function parseConnectionString(connectionString: string): {
 
   return { endpoint, key: accountKey };
 }
+/**
+ * utility function to build the query string for the CosmosDB query
+ */
+function queryBuilder(options: AzureCosmosQueryOptions): string {
+  let initialQuery =
+    "SELECT TOP @k c[@id] as id, c[@text] as text, c[@metadata] as metadata";
+  if (options.includeVectorDistance !== false) {
+    initialQuery += `, VectorDistance(c[@embeddingKey],@embedding) AS SimilarityScore`;
+  }
+  initialQuery += ` FROM c`;
+  if (options.whereClause) {
+    initialQuery += ` WHERE ${options.whereClause}`;
+  }
+  initialQuery += ` ORDER BY VectorDistance(c[@embeddingKey],@embedding)`;
+  return initialQuery;
+}
 
 export class AzureCosmosDBNoSqlVectorStore extends BaseVectorStore {
   storesText: boolean = true;
@@ -334,21 +360,25 @@ export class AzureCosmosDBNoSqlVectorStore extends BaseVectorStore {
    */
   async query(
     query: VectorStoreQuery,
-    options?: object,
+    options: AzureCosmosQueryOptions = {},
   ): Promise<VectorStoreQueryResult> {
     await this.initialize();
+    if (!query.queryEmbedding || query.queryEmbedding.length === 0) {
+      throw new Error(
+        "queryEmbedding is required for AzureCosmosDBNoSqlVectorStore query",
+      );
+    }
     const params = {
       vector: query.queryEmbedding!,
       k: query.similarityTopK,
     };
-
+    const builtQuery = queryBuilder(options);
     const nodes: BaseNode[] = [];
     const ids: string[] = [];
     const similarities: number[] = [];
     const queryResults = await this.container.items
       .query({
-        query:
-          "SELECT TOP @k c[@id] as id, c[@text] as text, c[@metadata] as metadata, VectorDistance(c[@embeddingKey],@embedding) AS SimilarityScore FROM c ORDER BY VectorDistance(c[@embeddingKey],@embedding)",
+        query: builtQuery,
         parameters: [
           { name: "@k", value: params.k },
           { name: "@id", value: this.idKey },

packages/llamaindex/tests/vector-stores/AzureCosmosDBNoSqlVectorStore.int.test.ts

@@ -14,9 +14,10 @@ import {
   Settings,
   VectorStoreQueryMode,
   type AzureCosmosDBNoSQLConfig,
+  type AzureCosmosQueryOptions,
   type VectorStoreQueryResult,
 } from "llamaindex";
-import { beforeEach, describe, expect, it } from "vitest";
+import { beforeAll, describe, expect, it } from "vitest";
 dotenv.config();
 /*
  * To run this test, you need have an Azure Cosmos DB for NoSQL instance
@@ -64,7 +65,10 @@ Settings.llm = new OpenAI(llmInit);
 Settings.embedModel = new OpenAIEmbedding(embedModelInit);
 // This test is skipped because it requires an Azure Cosmos DB instance and OpenAI API keys
 describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
-  beforeEach(async () => {
+  let vectorStore: AzureCosmosDBNoSqlVectorStore;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let embeddings: any = [];
+  beforeAll(async () => {
     if (process.env.AZURE_COSMOSDB_NOSQL_CONNECTION_STRING) {
       client = new CosmosClient(
         process.env.AZURE_COSMOSDB_NOSQL_CONNECTION_STRING,
@@ -79,15 +83,12 @@ describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
         "Please set the environment variable AZURE_COSMOSDB_NOSQL_CONNECTION_STRING or AZURE_COSMOSDB_NOSQL_ENDPOINT",
       );
     }
-
     // Make sure the database does not exists
     try {
       await client.database(DATABASE_NAME).delete();
     } catch {
       // Ignore error if the database does not exist
     }
-  });
-  it("perform query", async () => {
     const config: AzureCosmosDBNoSQLConfig = {
       idKey: "name",
       textKey: "customText",
@@ -134,9 +135,9 @@ describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
       },
     };
 
-    const vectorStore = new AzureCosmosDBNoSqlVectorStore(config);
+    vectorStore = new AzureCosmosDBNoSqlVectorStore(config);
 
-    const embeddings = await Settings.embedModel.getTextEmbeddings([
+    embeddings = await Settings.embedModel.getTextEmbeddings([
       "This book is about politics",
       "Cats sleeps a lot.",
       "Sandwiches taste good.",
@@ -150,28 +151,29 @@ describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
         id_: "1",
         text: "This book is about politics",
         embedding: embeddings[0],
-        metadata: { key: "politics" },
+        metadata: { key: "politics", number: 1 },
       }),
       new Document({
         id_: "2",
         text: "Cats sleeps a lot.",
         embedding: embeddings[1],
-        metadata: { key: "cats" },
+        metadata: { key: "cats", number: 2 },
       }),
       new Document({
         id_: "3",
         text: "Sandwiches taste good.",
         embedding: embeddings[2],
-        metadata: { key: "sandwiches" },
+        metadata: { key: "sandwiches", number: 3 },
       }),
       new Document({
         id_: "4",
         text: "The house is open",
         embedding: embeddings[3],
-        metadata: { key: "house" },
+        metadata: { key: "house", number: 4 },
       }),
     ]);
-
+  });
+  it("perform query", async () => {
     const results: VectorStoreQueryResult = await vectorStore.query({
       queryEmbedding: embeddings[4] || [],
       similarityTopK: 1,
@@ -179,5 +181,62 @@ describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
     });
     expect(results.ids.length).toEqual(1);
     expect(results.ids[0]).toEqual("3");
+    expect(results.similarities).toBeDefined();
+    expect(results.similarities[0]).toBeDefined();
+  }, 1000000);
+
+  it("perform query with where clause", async () => {
+    const options: AzureCosmosQueryOptions = {
+      whereClause: "c.customMetadata.number > 3",
+    };
+    const results: VectorStoreQueryResult = await vectorStore.query(
+      {
+        queryEmbedding: embeddings[4] || [],
+        similarityTopK: 1,
+        mode: VectorStoreQueryMode.DEFAULT,
+      },
+      options,
+    );
+    expect(results.ids.length).toEqual(1);
+    expect(results.ids[0]).toEqual("4");
+    expect(results.similarities).toBeDefined();
+    expect(results.similarities[0]).toBeDefined();
+  }, 1000000);
+
+  it("perform query with includeVectorDistance false", async () => {
+    const options: AzureCosmosQueryOptions = {
+      includeVectorDistance: false,
+    };
+    const results: VectorStoreQueryResult = await vectorStore.query(
+      {
+        queryEmbedding: embeddings[4] || [],
+        similarityTopK: 1,
+        mode: VectorStoreQueryMode.DEFAULT,
+      },
+      options,
+    );
+    expect(results.ids.length).toEqual(1);
+    expect(results.ids[0]).toEqual("3");
+    expect(results.similarities).toBeDefined();
+    expect(results.similarities[0]).toBeUndefined();
+  }, 1000000);
+
+  it("perform query with includeVectorDistance false and whereClause", async () => {
+    const options: AzureCosmosQueryOptions = {
+      includeVectorDistance: false,
+      whereClause: "c.customMetadata.number > 3",
+    };
+    const results: VectorStoreQueryResult = await vectorStore.query(
+      {
+        queryEmbedding: embeddings[4] || [],
+        similarityTopK: 1,
+        mode: VectorStoreQueryMode.DEFAULT,
+      },
+      options,
+    );
+    expect(results.ids.length).toEqual(1);
+    expect(results.ids[0]).toEqual("4");
+    expect(results.similarities).toBeDefined();
+    expect(results.similarities[0]).toBeUndefined();
   }, 1000000);
 });

packages/llamaindex/tests/vector-stores/AzureCosmosDBNoSqlVectorStore.test.ts

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import type { BaseNode } from "@llamaindex/core/schema";
 import { beforeEach, describe, expect, it, vi } from "vitest";
+import { VectorStoreQueryMode } from "../../src/vector-store.js";
 import { TestableAzureCosmosDBNoSqlVectorStore } from "../mocks/TestableAzureCosmosDBNoSqlVectorStore.js";
 import { createMockClient } from "../utility/mockCosmosClient.js"; // Import the mock client
 
@@ -95,4 +96,27 @@ describe("AzureCosmosDBNoSqlVectorStore Tests", () => {
     expect(client.databases.containers.items.create).toHaveBeenCalledTimes(2);
     expect(result).toEqual(["node-0", "node-1"]);
   });
+
+  it("should throw error if no query embedding is provided", async () => {
+    const client = createMockClient();
+    const store = new TestableAzureCosmosDBNoSqlVectorStore({
+      client: client as any,
+      endpoint: "https://example.com",
+      idKey: "id",
+      textKey: "text",
+      metadataKey: "metadata",
+    });
+
+    expect(store).toBeDefined();
+
+    await expect(
+      store.query({
+        queryEmbedding: [],
+        similarityTopK: 4,
+        mode: VectorStoreQueryMode.DEFAULT,
+      }),
+    ).rejects.toThrowError(
+      "queryEmbedding is required for AzureCosmosDBNoSqlVectorStore query",
+    );
+  });
 });