Release 0.8.17 (#1504)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

README.md

@@ -1,12 +1,16 @@
-# LlamaIndex.TS
+<p align="center">
+  <img height="100" width="100" alt="LlamaIndex logo" src="https://ts.llamaindex.ai/square.svg" />
+</p>
+<h1 align="center">LlamaIndex.TS</h1>
+<h3 align="center">
+  Data framework for your LLM application.
+</h3>
 
 [![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
 [![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
 [![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
 [![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
 
-LlamaIndex is a data framework for your LLM application.
-
 Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in JS runtime environments with TypeScript support.
 
 Documentation: https://ts.llamaindex.ai/

apps/docs/CHANGELOG.md

@@ -1,5 +1,39 @@
 # docs
 
+## 0.0.122
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.121
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.120
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.119
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.118
+
+### Patch Changes
+
+- llamaindex@0.8.13
+- @llamaindex/examples@0.0.16
+
 ## 0.0.117
 
 ### Patch Changes

apps/docs/docusaurus.config.js

@@ -62,6 +62,12 @@ const config = {
     ({
       // Replace with your project's social card
       image: "img/favicon.png", // TODO change this
+      announcementBar: {
+        id: "migrate_to_next",
+        content:
+          'We are migrating to Next.js based documentation. Check it out <a href="https://ts.llamaindex.ai/docs/llamaindex">here</a>!',
+        isCloseable: false,
+      },
       navbar: {
         title: "LlamaIndex.TS",
         logo: {

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.117",
+  "version": "0.0.122",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

apps/next/CHANGELOG.md

@@ -1,5 +1,55 @@
 # @llamaindex/doc
 
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+  - @llamaindex/cloud@2.0.11
+  - llamaindex@0.8.17
+  - @llamaindex/node-parser@0.0.12
+  - @llamaindex/openai@0.1.36
+  - @llamaindex/readers@1.0.13
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [e60328b]
+  - @llamaindex/readers@1.0.12
+  - llamaindex@0.8.16
+
+## 0.0.18
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [ee20c44]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+  - @llamaindex/workflow@0.0.6
+  - llamaindex@0.8.13
+  - @llamaindex/cloud@2.0.10
+  - @llamaindex/node-parser@0.0.11
+  - @llamaindex/openai@0.1.35
+  - @llamaindex/readers@1.0.11
+
 ## 0.0.15
 
 ### Patch Changes

apps/next/next.config.mjs

@@ -6,6 +6,7 @@ const withMDX = createMDX();
 const config = {
   reactStrictMode: true,
   transpilePackages: ["monaco-editor"],
+  serverExternalPackages: ["@huggingface/transformers"],
   webpack: (config, { isServer }) => {
     if (Array.isArray(config.target) && config.target.includes("web")) {
       config.target = ["web", "es2020"];
@@ -26,6 +27,7 @@ const config = {
         }),
       );
     }
+    config.resolve.alias["replicate"] = false;
     return config;
   },
 };

apps/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/doc",
-  "version": "0.0.15",
+  "version": "0.0.20",
   "private": true,
   "scripts": {
     "build": "pnpm run build:docs && next build",
@@ -12,7 +12,7 @@
   },
   "dependencies": {
     "@icons-pack/react-simple-icons": "^10.1.0",
-    "@llamaindex/chat-ui": "0.0.5",
+    "@llamaindex/chat-ui": "0.0.8",
     "@llamaindex/cloud": "workspace:*",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/node-parser": "workspace:*",

apps/next/src/app/api/chat/route.ts

@@ -1,7 +1,10 @@
+import { llm } from "@/lib/utils";
 import { LlamaIndexAdapter, type Message } from "ai";
-import { SimpleChatEngine, type ChatMessage } from "llamaindex";
+import { Settings, SimpleChatEngine, type ChatMessage } from "llamaindex";
 import { NextResponse, type NextRequest } from "next/server";
 
+Settings.llm = llm;
+
 export async function POST(request: NextRequest) {
   try {
     const { messages } = (await request.json()) as { messages: Message[] };

apps/next/src/components/demo/chat.tsx

@@ -1,8 +0,0 @@
-"use client";
-import { ChatSection } from "@llamaindex/chat-ui";
-import { useChat } from "ai/react";
-
-export const ChatDemo = () => {
-  const handler = useChat();
-  return <ChatSection handler={handler} />;
-};

apps/next/src/components/demo/chat/api/demo.tsx

@@ -0,0 +1,16 @@
+"use client";
+import { ChatInput, ChatMessages, ChatSection } from "@llamaindex/chat-ui";
+import { useChat } from "ai/react";
+
+export const ChatDemo = () => {
+  const handler = useChat();
+  return (
+    <ChatSection handler={handler}>
+      <ChatMessages>
+        <ChatMessages.List className="h-auto max-h-[400px]" />
+        <ChatMessages.Actions />
+      </ChatMessages>
+      <ChatInput />
+    </ChatSection>
+  );
+};

apps/next/src/components/demo/chat/rsc/ai-action.tsx

@@ -0,0 +1,57 @@
+import { llm } from "@/lib/utils";
+import { Markdown } from "@llamaindex/chat-ui/widgets";
+import { generateId, Message } from "ai";
+import { createAI, createStreamableUI, getMutableAIState } from "ai/rsc";
+import { type ChatMessage, Settings, SimpleChatEngine } from "llamaindex";
+import { ReactNode } from "react";
+
+type ServerState = Message[];
+type FrontendState = Array<Message & { display: ReactNode }>;
+type Actions = {
+  chat: (message: Message) => Promise<Message & { display: ReactNode }>;
+};
+
+Settings.llm = llm;
+
+export const AI = createAI<ServerState, FrontendState, Actions>({
+  initialAIState: [],
+  initialUIState: [],
+  actions: {
+    chat: async (message: Message) => {
+      "use server";
+
+      const aiState = getMutableAIState<typeof AI>();
+      aiState.update((prev) => [...prev, message]);
+
+      const uiStream = createStreamableUI();
+      const chatEngine = new SimpleChatEngine();
+      const assistantMessage: Message = {
+        id: generateId(),
+        role: "assistant",
+        content: "",
+      };
+
+      // run the async function without blocking
+      (async () => {
+        const chatResponse = await chatEngine.chat({
+          stream: true,
+          message: message.content,
+          chatHistory: aiState.get() as ChatMessage[],
+        });
+
+        for await (const chunk of chatResponse) {
+          assistantMessage.content += chunk.delta;
+          uiStream.update(<Markdown content={assistantMessage.content} />);
+        }
+
+        aiState.done([...aiState.get(), assistantMessage]);
+        uiStream.done();
+      })();
+
+      return {
+        ...assistantMessage,
+        display: uiStream.value,
+      };
+    },
+  },
+});

apps/next/src/components/demo/chat/rsc/chat-section.tsx

@@ -0,0 +1,33 @@
+"use client";
+
+import {
+  ChatInput,
+  ChatMessage,
+  ChatMessages,
+  ChatSection as ChatSectionUI,
+} from "@llamaindex/chat-ui";
+import { useChatRSC } from "./use-chat-rsc";
+
+export const ChatSectionRSC = () => {
+  const handler = useChatRSC();
+  return (
+    <ChatSectionUI handler={handler}>
+      <ChatMessages>
+        <ChatMessages.List className="h-auto max-h-[400px]">
+          {handler.messages.map((message, index) => (
+            <ChatMessage
+              key={index}
+              message={message}
+              isLast={index === handler.messages.length - 1}
+            >
+              <ChatMessage.Avatar />
+              <ChatMessage.Content>{message.display}</ChatMessage.Content>
+            </ChatMessage>
+          ))}
+          <ChatMessages.Loading />
+        </ChatMessages.List>
+      </ChatMessages>
+      <ChatInput />
+    </ChatSectionUI>
+  );
+};

apps/next/src/components/demo/chat/rsc/demo.tsx

@@ -0,0 +1,8 @@
+import { AI } from "./ai-action";
+import { ChatSectionRSC } from "./chat-section";
+
+export const ChatDemoRSC = () => (
+  <AI>
+    <ChatSectionRSC />
+  </AI>
+);

apps/next/src/components/demo/chat/rsc/use-chat-rsc.tsx

@@ -0,0 +1,41 @@
+"use client";
+
+import { useActions } from "ai/rsc";
+
+import { generateId, Message } from "ai";
+import { useUIState } from "ai/rsc";
+import { useState } from "react";
+import { AI } from "./ai-action";
+
+export function useChatRSC() {
+  const [input, setInput] = useState<string>("");
+  const [isLoading, setIsLoading] = useState<boolean>(false);
+  const [messages, setMessages] = useUIState<typeof AI>();
+  const { chat } = useActions<typeof AI>();
+
+  const append = async (message: Omit<Message, "id">) => {
+    const newMsg: Message = { ...message, id: generateId() };
+
+    setIsLoading(true);
+    try {
+      setMessages((prev) => [...prev, { ...newMsg, display: message.content }]);
+      const assistantMsg = await chat(newMsg);
+      setMessages((prev) => [...prev, assistantMsg]);
+    } catch (error) {
+      console.error(error);
+    }
+    setIsLoading(false);
+    setInput("");
+
+    return message.content;
+  };
+
+  return {
+    input,
+    setInput,
+    isLoading,
+    messages,
+    setMessages,
+    append,
+  };
+}

apps/next/src/components/website/Footer.tsx

@@ -85,6 +85,33 @@ const Footer = () => {
                   <Text as="span">SharePoint</Text>
                 </a>
               </li>
+              <li>
+                <a
+                  href="https://llamaindex.ai/llamacloud-aws-s3-data-loading-for-generative-ai"
+                  data-tracking-variant="link"
+                  data-tracking-section="footer"
+                >
+                  <Text as="span">AWS S3</Text>
+                </a>
+              </li>
+              <li>
+                <a
+                  href="https://llamaindex.ai/llamacloud-azure-blob-storage-data-loading-for-generative-ai"
+                  data-tracking-variant="link"
+                  data-tracking-section="footer"
+                >
+                  <Text as="span">Azure Blob Storage</Text>
+                </a>
+              </li>
+              <li>
+                <a
+                  href="https://llamaindex.ai/llamacloud-google-drive-data-loading-for-generative-ai"
+                  data-tracking-variant="link"
+                  data-tracking-section="footer"
+                >
+                  <Text as="span">Google Drive</Text>
+                </a>
+              </li>
             </ul>
           </div>
           <div>
@@ -171,11 +198,6 @@ const Footer = () => {
                   <Text as="span">SEC Insights</Text>
                 </a>
               </li>
-              <li>
-                <a href="https://chat.llamaindex.ai/">
-                  <Text as="span">Chat LlamaIndex</Text>
-                </a>
-              </li>
               <li>
                 <a href="https://github.com/run-llama/llamabot">
                   <Text as="span">LlamaBot</Text>

apps/next/src/content/docs/llamaindex/guide/chat/chat.mdx

@@ -1,8 +1,8 @@
 ---
-title: Chat-UI 
-description: Use chat-ui to add a chat interface to your LlamaIndexTS application.
+title: Using API Route
+description: Chat interface for your LlamaIndexTS application using API Route
 ---
-import { ChatDemo } from '../../../../components/demo/chat';
+import { ChatDemo } from '../../../../../components/demo/chat/api/demo';
 import "@llamaindex/chat-ui/styles/code.css";
 import "@llamaindex/chat-ui/styles/katex.css";
 
@@ -26,7 +26,7 @@ This is the simplest way to add a chat interface to your application. Copy the f
 
 ```json doc-gen:file
 {
-  "file": "./src/components/demo/chat.tsx",
+  "file": "./src/components/demo/chat/api/demo.tsx",
 	"codeblock": true
 }
 ```
@@ -37,6 +37,7 @@ Combining both, you're getting a fully functional chat interface:
 
 <ChatDemo />
 
+
 ## Next Steps
 
 The steps above are the bare minimum to get a chat interface working. From here, you can go two ways:

apps/next/src/content/docs/llamaindex/guide/chat/meta.json

@@ -0,0 +1,6 @@
+{
+  "title": "Chat-UI",
+  "description": "Use chat-ui to add a chat interface to your LlamaIndexTS application.",
+  "defaultOpen": false,
+  "pages": ["chat", "rsc"]
+}

apps/next/src/content/docs/llamaindex/guide/chat/rsc.mdx

@@ -0,0 +1,68 @@
+---
+title: Using Next.js RSC
+description: Chat interface for your LlamaIndexTS application using Next.js RSC
+---
+import { ChatDemoRSC } from '../../../../../components/demo/chat/rsc/demo';
+import "@llamaindex/chat-ui/styles/code.css";
+import "@llamaindex/chat-ui/styles/katex.css";
+
+Using [chat-ui](https://github.com/run-llama/chat-ui), it's easy to add a chat interface to your LlamaIndexTS application using [Next.js RSC](https://nextjs.org/docs/app/building-your-application/rendering/server-components) and [Vercel AI RSC](https://sdk.vercel.ai/docs/ai-sdk-rsc/overview).
+
+With RSC, the chat messages are not returned as JSON from the server (like when using an [API route](./chat)), instead the chat message components are rendered on the server side.
+This is for example useful for rendering a whole chat history on the server before sending it to the client. [Check here](https://sdk.vercel.ai/docs/getting-started/navigating-the-library#when-to-use-ai-sdk-rsc), for a discussion of when to use use RSC.
+
+For implementing a chat interface with RSC, you need to create an AI action and then connect the chat interface to use it.
+
+## Create an AI action
+
+First, define an [AI context provider](https://sdk.vercel.ai/examples/rsc/state-management/ai-ui-states) with a chat server action:
+
+```json doc-gen:file
+{
+  "file": "./src/components/demo/chat/rsc/ai-action.tsx",
+	"codeblock": true
+}
+```
+
+The chat server action is using LlamaIndexTS to generate a response based on the chat history and the user input.
+
+## Create the chat UI
+
+The entrypoint of our application initializes the AI provider for the application and adds a `ChatSection` component:
+
+```json doc-gen:file
+{
+  "file": "./src/components/demo/chat/rsc/demo.tsx",
+	"codeblock": true
+}
+```
+
+The `ChatSection` component is created by using chat components from @llamaindex/chat-ui:
+
+```json doc-gen:file
+{
+  "file": "./src/components/demo/chat/rsc/chat-section.tsx",
+	"codeblock": true
+}
+```
+
+It is using a `useChatRSC` hook to conntect the chat interface to the `chat` AI action that we defined earlier:
+
+```json doc-gen:file
+{
+  "file": "./src/components/demo/chat/rsc/use-chat-rsc.tsx",
+	"codeblock": true
+}
+```
+
+## Try RSC Chat ⬇️
+
+<ChatDemoRSC />
+
+## Next Steps
+
+The steps above are the bare minimum to get a chat interface working with RSC. From here, you can go two ways:
+
+1. Use our [full-stack RSC example](https://github.com/run-llama/nextjs-rsc) based on [create-llama](https://github.com/run-llama/create-llama) to get started quickly with a fully working chat interface or
+2. Learn more about [AI RSC](https://sdk.vercel.ai/examples/rsc), [chat-ui](https://github.com/run-llama/chat-ui) and [LlamaIndexTS](https://github.com/run-llama/llamaindex-ts) to customize the chat interface and AI actions to your needs.
+

apps/next/src/lib/utils.ts

@@ -1,6 +1,34 @@
-import { clsx, type ClassValue } from "clsx"
-import { twMerge } from "tailwind-merge"
+import { clsx, type ClassValue } from "clsx";
+import { LLM, LLMMetadata } from "llamaindex";
+import { twMerge } from "tailwind-merge";
 
 export function cn(...inputs: ClassValue[]) {
   return twMerge(clsx(inputs))
 }
+
+class MockLLM  {
+  metadata: LLMMetadata = {
+    model: "MockLLM",
+    temperature: 0.5,
+    topP: 0.5,
+    contextWindow: 1024,
+    tokenizer: undefined,
+  };
+
+  chat() {
+    const mockResponse = "Hello! This is a mock response";
+    return Promise.resolve(
+      new ReadableStream({
+        async start(controller) {
+          for (const char of mockResponse) {
+            controller.enqueue({ delta: char });
+            await new Promise((resolve) => setTimeout(resolve, 20));
+          }
+          controller.close();
+        },
+      }),
+    );
+  }
+}
+
+export const llm = new MockLLM() as unknown as LLM;

e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.113
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.112
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.111
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.110
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.109
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
 ## 0.0.108
 
 ### Patch Changes

e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.108",
+  "version": "0.0.113",
   "type": "module",
   "private": true,
   "scripts": {

e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.31
+
+### Patch Changes
+
+- @llamaindex/cloud@2.0.11
+
+## 0.0.30
+
+### Patch Changes
+
+- @llamaindex/cloud@2.0.10
+
 ## 0.0.29
 
 ### Patch Changes

e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.29",
+  "version": "0.0.31",
   "type": "module",
   "scripts": {
     "dev": "vite",

e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/next-agent-test
 
+## 0.1.113
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.1.112
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.1.111
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.1.110
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.1.109
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
 ## 0.1.108
 
 ### Patch Changes

e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.108",
+  "version": "0.1.113",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,38 @@
 # test-edge-runtime
 
+## 0.1.112
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.1.111
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.1.110
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.1.109
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.1.108
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
 ## 0.1.107
 
 ### Patch Changes

e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.107",
+  "version": "0.1.112",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.94
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.93
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.92
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.91
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.90
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
 ## 0.0.89
 
 ### Patch Changes

e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.89",
+  "version": "0.0.94",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.113
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.112
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.111
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.110
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.109
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
 ## 0.0.108
 
 ### Patch Changes

e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.108",
+  "version": "0.0.113",
   "type": "module",
   "private": true,
   "scripts": {

examples/CHANGELOG.md

@@ -1,5 +1,17 @@
 # examples
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [ee20c44]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+  - @llamaindex/workflow@0.0.6
+  - llamaindex@0.8.13
+  - @llamaindex/readers@1.0.11
+
 ## 0.0.15
 
 ### Patch Changes

examples/anthropic/prompt-caching.ts

@@ -0,0 +1,38 @@
+import { Anthropic } from "llamaindex";
+
+async function main() {
+  const anthropic = new Anthropic({
+    model: "claude-3-5-sonnet-20241022",
+  });
+
+  const entireBook = await fetch(
+    "https://www.gutenberg.org/files/1342/1342-0.txt",
+  ).then((response) => response.text());
+
+  const response = await anthropic.chat({
+    messages: [
+      {
+        content:
+          "You are an AI assistant tasked with analyzing literary works. Your goal is to provide insightful commentary on themes, characters, and writing style.\n",
+        role: "system",
+      },
+      {
+        content: entireBook,
+        role: "system",
+        options: {
+          cache_control: {
+            type: "ephemeral",
+          },
+        },
+      },
+      {
+        content: "analyze the major themes in Pride and Prejudice.",
+        role: "user",
+      },
+    ],
+  });
+
+  console.log(response.message.content);
+}
+
+main().catch(console.error);

examples/cosmosdb/queryVectorData.ts

@@ -83,14 +83,6 @@ async function query() {
     });
   }
 
-  // configure the Azure CosmosDB NoSQL Vector Store
-  const dbConfig: AzureCosmosDBNoSQLConfig = {
-    client: cosmosClient,
-    databaseName,
-    containerName,
-    flatMetadata: false,
-  };
-
   // use Azure CosmosDB as a vectorStore, docStore, and indexStore
   const { vectorStore, docStore, indexStore } = await initializeStores();
 

examples/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@llamaindex/examples",
   "private": true,
-  "version": "0.0.15",
+  "version": "0.0.16",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@azure/cosmos": "^4.1.1",
     "@azure/identity": "^4.4.1",
     "@datastax/astra-db-ts": "^1.4.1",
-    "@llamaindex/core": "^0.4.7",
-    "@llamaindex/readers": "^1.0.8",
-    "@llamaindex/workflow": "^0.0.5",
+    "@llamaindex/core": "^0.4.10",
+    "@llamaindex/readers": "^1.0.11",
+    "@llamaindex/workflow": "^0.0.6",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^3.0.2",
     "@vercel/postgres": "^0.10.0",
@@ -18,7 +18,7 @@
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.8.10",
+    "llamaindex": "^0.8.13",
     "mongodb": "^6.7.0",
     "pathe": "^1.1.2",
     "postgres": "^3.4.4"

packages/autotool/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/autotool
 
+## 5.0.17
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 5.0.16
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 5.0.15
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 5.0.14
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 5.0.13
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
 ## 5.0.12
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,43 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.60
+
+### Patch Changes
+
+- llamaindex@0.8.17
+- @llamaindex/autotool@5.0.17
+
+## 0.0.59
+
+### Patch Changes
+
+- llamaindex@0.8.16
+- @llamaindex/autotool@5.0.16
+
+## 0.0.58
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+  - @llamaindex/autotool@5.0.15
+
+## 0.0.57
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+  - @llamaindex/autotool@5.0.14
+
+## 0.0.56
+
+### Patch Changes
+
+- llamaindex@0.8.13
+- @llamaindex/autotool@5.0.13
+
 ## 0.0.55
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.55"
+  "version": "0.0.60"
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,43 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.104
+
+### Patch Changes
+
+- llamaindex@0.8.17
+- @llamaindex/autotool@5.0.17
+
+## 0.1.103
+
+### Patch Changes
+
+- llamaindex@0.8.16
+- @llamaindex/autotool@5.0.16
+
+## 0.1.102
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+  - @llamaindex/autotool@5.0.15
+
+## 0.1.101
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+  - @llamaindex/autotool@5.0.14
+
+## 0.1.100
+
+### Patch Changes
+
+- llamaindex@0.8.13
+- @llamaindex/autotool@5.0.13
+
 ## 0.1.99
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.99",
+  "version": "0.1.104",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "5.0.12",
+  "version": "5.0.17",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/cloud
 
+## 2.0.11
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 2.0.10
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
 ## 2.0.9
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "2.0.9",
+  "version": "2.0.11",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/community
 
+## 0.0.69
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 0.0.68
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
 ## 0.0.67
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.67",
+  "version": "0.0.69",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/core
 
+## 0.4.11
+
+### Patch Changes
+
+- 6d22fa2: Get PromptTemplate template variables at run-time
+
+## 0.4.10
+
+### Patch Changes
+
+- a7b0ac3: fix: update tool call llm type
+- c69605f: feat: add async support to BaseChatStore and BaseChatStoreMemory
+
 ## 0.4.9
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.4.9",
+  "version": "0.4.11",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./agent": {

packages/core/src/llms/base.ts

@@ -66,6 +66,8 @@ export abstract class BaseLLM<
 
 export abstract class ToolCallLLM<
   AdditionalChatOptions extends object = object,
-> extends BaseLLM<AdditionalChatOptions, ToolCallLLMMessageOptions> {
+  AdditionalMessageOptions extends
+    ToolCallLLMMessageOptions = ToolCallLLMMessageOptions,
+> extends BaseLLM<AdditionalChatOptions, AdditionalMessageOptions> {
   abstract supportToolCall: boolean;
 }

packages/core/src/memory/base.ts

@@ -71,15 +71,15 @@ export abstract class BaseChatStoreMemory<
     return this.chatStore.getMessages(this.chatStoreKey);
   }
 
-  put(messages: ChatMessage<AdditionalMessageOptions>) {
+  put(messages: ChatMessage<AdditionalMessageOptions>): void | Promise<void> {
     this.chatStore.addMessage(this.chatStoreKey, messages);
   }
 
-  set(messages: ChatMessage<AdditionalMessageOptions>[]) {
+  set(messages: ChatMessage<AdditionalMessageOptions>[]): void | Promise<void> {
     this.chatStore.setMessages(this.chatStoreKey, messages);
   }
 
-  reset() {
+  reset(): void | Promise<void> {
     this.chatStore.deleteMessages(this.chatStoreKey);
   }
 }

packages/core/src/prompts/base.ts

@@ -32,6 +32,10 @@ export abstract class BasePromptTemplate<
   const Vars extends readonly string[] = string[],
 > {
   metadata: Metadata = {};
+  /**
+   * Set of template variables used in the prompt template. Used for type hints only.
+   * To get the list of template variables used in the prompt at run-time, use the `vars` method.
+   */
   templateVars: Set<string> = new Set();
   options: Partial<Record<TemplatesVar[number] | (string & {}), string>> = {};
   outputParser: BaseOutputParser | undefined;
@@ -223,4 +227,13 @@ export class PromptTemplate<
   get template(): Template {
     return this.#template;
   }
+
+  /**
+   * Returns all the template variables used in the prompt template.
+   */
+  vars(): string[] {
+    const template = this.template;
+    const matches = template.match(/\{([^}]+)\}/g) || [];
+    return [...new Set(matches.map((match) => match.slice(1, -1)))];
+  }
 }

packages/core/src/storage/chat-store/base-chat-store.ts

@@ -19,5 +19,7 @@ export abstract class BaseChatStore<
   ): void;
   abstract deleteMessages(key: string): void;
   abstract deleteMessage(key: string, idx: number): void;
-  abstract getKeys(): IterableIterator<string>;
+  abstract getKeys():
+    | IterableIterator<string>
+    | Promise<IterableIterator<string>>;
 }

packages/core/tests/prompts.test.ts

@@ -158,4 +158,15 @@ describe("PromptTemplate", () => {
     const formatted = prompt.format({ text: "world", foo: "bar" });
     expect(formatted).toBe("hello world bar\noutput_instruction");
   });
+
+  test("should return all unique template vars of a prompt", () => {
+    const prompt = new PromptTemplate({
+      template: "Hello {name}! Your age is {age}. Nice to meet you {name}!",
+    });
+
+    const vars = prompt.vars();
+    expect(vars).toHaveLength(2);
+    expect(vars).toContain("name");
+    expect(vars).toContain("age");
+  });
 });

packages/experimental/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/experimental
 
+## 0.0.129
+
+### Patch Changes
+
+- llamaindex@0.8.17
+
+## 0.0.128
+
+### Patch Changes
+
+- llamaindex@0.8.16
+
+## 0.0.127
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.126
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.125
+
+### Patch Changes
+
+- llamaindex@0.8.13
+
 ## 0.0.124
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.124",
+  "version": "0.0.129",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,67 @@
 # llamaindex
 
+## 0.8.17
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+  - @llamaindex/cloud@2.0.11
+  - @llamaindex/node-parser@0.0.12
+  - @llamaindex/anthropic@0.0.20
+  - @llamaindex/clip@0.0.20
+  - @llamaindex/deepinfra@0.0.20
+  - @llamaindex/huggingface@0.0.20
+  - @llamaindex/ollama@0.0.27
+  - @llamaindex/openai@0.1.36
+  - @llamaindex/portkey-ai@0.0.20
+  - @llamaindex/replicate@0.0.20
+  - @llamaindex/readers@1.0.13
+  - @llamaindex/groq@0.0.35
+  - @llamaindex/vllm@0.0.6
+
+## 0.8.16
+
+### Patch Changes
+
+- Updated dependencies [e60328b]
+  - @llamaindex/readers@1.0.12
+
+## 0.8.15
+
+### Patch Changes
+
+- 3d503cb: Update azure cosmos db
+- 5dae534: fix: propagate queryStr to concrete vectorStore
+
+## 0.8.14
+
+### Patch Changes
+
+- 630b425: feat: add Azure CosmosDB NoSql Chat store
+
+## 0.8.13
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/anthropic@0.0.19
+  - @llamaindex/core@0.4.10
+  - @llamaindex/cloud@2.0.10
+  - @llamaindex/node-parser@0.0.11
+  - @llamaindex/clip@0.0.19
+  - @llamaindex/deepinfra@0.0.19
+  - @llamaindex/huggingface@0.0.19
+  - @llamaindex/ollama@0.0.26
+  - @llamaindex/openai@0.1.35
+  - @llamaindex/portkey-ai@0.0.19
+  - @llamaindex/replicate@0.0.19
+  - @llamaindex/readers@1.0.11
+  - @llamaindex/groq@0.0.34
+  - @llamaindex/vllm@0.0.5
+
 ## 0.8.12
 
 ### Patch Changes

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.8.12",
+  "version": "0.8.17",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -18,6 +18,7 @@ import {
   type NodeWithScore,
 } from "@llamaindex/core/schema";
 import type { BaseIndexStore } from "@llamaindex/core/storage/index-store";
+import { extractText } from "@llamaindex/core/utils";
 import type { ServiceContext } from "../../ServiceContext.js";
 import { nodeParserFromSettingsOrContext } from "../../Settings.js";
 import { RetrieverQueryEngine } from "../../engines/query/RetrieverQueryEngine.js";
@@ -449,8 +450,13 @@ export class VectorIndexRetriever extends BaseRetriever {
     filters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
     // convert string message to multi-modal format
+
+    let queryStr = query;
     if (typeof query === "string") {
-      query = [{ type: "text", text: query }];
+      queryStr = query;
+      query = [{ type: "text", text: queryStr }];
+    } else {
+      queryStr = extractText(query);
     }
     // overwrite embed model if specified, otherwise use the one from the vector store
     const embedModel = this.index.embedModel ?? vectorStore.embedModel;
@@ -460,6 +466,7 @@ export class VectorIndexRetriever extends BaseRetriever {
       const queryEmbedding = await embedModel.getQueryEmbedding(item);
       if (queryEmbedding) {
         const result = await vectorStore.query({
+          queryStr,
           queryEmbedding,
           mode: VectorStoreQueryMode.DEFAULT,
           similarityTopK: this.topK[type]!,

packages/llamaindex/src/storage/chatStore/AzureCosmosNoSqlChatStore.ts

@@ -0,0 +1,345 @@
+import { CosmosClient, type Container, type Database } from "@azure/cosmos";
+import { DefaultAzureCredential, type TokenCredential } from "@azure/identity";
+import type {
+  ChatMessage,
+  MessageContent,
+  MessageType,
+} from "@llamaindex/core/llms";
+import { BaseChatStore } from "@llamaindex/core/storage/chat-store";
+import { getEnv } from "@llamaindex/env";
+
+const USER_AGENT_SUFFIX = "llamaindex-cdbnosql-chatstore-javascript";
+const DEFAULT_CHAT_DATABASE = "ChatMessagesDB";
+const DEFAULT_CHAT_CONTAINER = "ChatMessagesContainer";
+const DEFAULT_OFFER_THROUGHPUT = 400;
+
+function parseConnectionString(connectionString: string): {
+  endpoint: string;
+  key: string;
+} {
+  const parts = connectionString.split(";");
+  let endpoint = "";
+  let accountKey = "";
+
+  parts.forEach((part) => {
+    const [key, value] = part.split("=");
+    if (key && key.trim() === "AccountEndpoint") {
+      endpoint = value?.trim() ?? "";
+    } else if ((key ?? "").trim() === "AccountKey") {
+      accountKey = value?.trim() ?? "";
+    }
+  });
+
+  if (!endpoint || !accountKey) {
+    throw new Error(
+      "Invalid connection string: missing AccountEndpoint or AccountKey.",
+    );
+  }
+  return { endpoint, key: accountKey };
+}
+
+export interface AzureCosmosChatDatabaseProperties {
+  throughput?: number;
+}
+
+export interface AzureCosmosChatContainerProperties {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [key: string]: any;
+}
+
+export interface AzureCosmosNoSqlChatStoreConfig {
+  cosmosClient?: CosmosClient;
+  dbName?: string;
+  containerName?: string;
+  userId?: string;
+  sessionId?: string;
+  cosmosContainerProperties?: AzureCosmosChatContainerProperties;
+  cosmosDatabaseProperties?: AzureCosmosChatDatabaseProperties;
+  ttlInSeconds?: number;
+}
+
+export class AzureCosmosNoSqlChatStore<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatStore<AdditionalMessageOptions> {
+  private userId: string;
+  private ttl: number;
+  private cosmosClient: CosmosClient;
+  private database!: Database;
+  private container!: Container;
+  private initPromise?: Promise<void>;
+
+  private dbName: string;
+  private containerName: string;
+  private cosmosContainerProperties: AzureCosmosChatContainerProperties;
+  private cosmosDatabaseProperties: AzureCosmosChatDatabaseProperties;
+  private initialize: () => Promise<void>;
+
+  constructor({
+    cosmosClient,
+    dbName = DEFAULT_CHAT_DATABASE,
+    containerName = DEFAULT_CHAT_CONTAINER,
+    cosmosContainerProperties = { partitionKey: "/userId" },
+    cosmosDatabaseProperties = {},
+    ttlInSeconds = -1,
+  }: AzureCosmosNoSqlChatStoreConfig) {
+    super();
+    if (!cosmosClient) {
+      throw new Error(
+        "CosmosClient is required for AzureCosmosDBNoSQLChatStore initialization",
+      );
+    }
+    this.ttl = ttlInSeconds;
+    this.userId = cosmosContainerProperties.userId || "anonymous";
+    this.cosmosClient = cosmosClient;
+    this.dbName = dbName;
+    this.containerName = containerName;
+    this.cosmosContainerProperties = cosmosContainerProperties;
+    this.cosmosDatabaseProperties = cosmosDatabaseProperties;
+
+    this.initialize = () => {
+      if (this.initPromise === undefined) {
+        this.initPromise = this.init().catch((error) => {
+          console.error(
+            "Error during AzureCosmosDBNoSQLChatStore initialization",
+            error,
+          );
+        });
+      }
+      return this.initPromise;
+    };
+  }
+
+  client(): CosmosClient {
+    return this.cosmosClient;
+  }
+
+  // Asynchronous initialization method to create database and container
+  private async init(): Promise<void> {
+    // Set default throughput if not provided
+    const throughput =
+      this.cosmosDatabaseProperties?.throughput || DEFAULT_OFFER_THROUGHPUT;
+
+    // Create the database if it doesn't exist
+    const { database } = await this.cosmosClient.databases.createIfNotExists({
+      id: this.dbName,
+      throughput,
+    });
+    this.database = database;
+
+    // Create the container if it doesn't exist
+    const { container } = await this.database.containers.createIfNotExists({
+      id: this.containerName,
+      throughput: this.cosmosContainerProperties?.throughput,
+      partitionKey: "/userId",
+      indexingPolicy: this.cosmosContainerProperties?.indexingPolicy,
+      defaultTtl: this.ttl,
+      uniqueKeyPolicy: this.cosmosContainerProperties?.uniqueKeyPolicy,
+      conflictResolutionPolicy:
+        this.cosmosContainerProperties?.conflictResolutionPolicy,
+      computedProperties: this.cosmosContainerProperties?.computedProperties,
+    });
+    this.container = container;
+  }
+  /**
+   * Static method for creating an instance using a connection string.
+   * If no connection string is provided, it will attempt to use the env variable `AZURE_COSMOSDB_NOSQL_CONNECTION_STRING` as connection string.
+   * @returns Instance of AzureCosmosNoSqlKVStore
+   */
+  static fromConnectionString(
+    config: {
+      connectionString?: string;
+    } & AzureCosmosNoSqlChatStoreConfig = {},
+  ): AzureCosmosNoSqlChatStore {
+    const cosmosConnectionString =
+      config.connectionString ||
+      (getEnv("AZURE_COSMOSDB_NOSQL_CONNECTION_STRING") as string);
+    if (!cosmosConnectionString) {
+      throw new Error("Azure CosmosDB connection string must be provided");
+    }
+    const { endpoint, key } = parseConnectionString(cosmosConnectionString);
+    const cosmosClient = new CosmosClient({
+      endpoint,
+      key,
+      userAgentSuffix: USER_AGENT_SUFFIX,
+    });
+    return new AzureCosmosNoSqlChatStore({
+      ...config,
+      cosmosClient,
+    });
+  }
+
+  /**
+   * Static method for creating an instance using a account endpoint and key.
+   * If no endpoint and key  is provided, it will attempt to use the env variable `AZURE_COSMOSDB_NOSQL_ACCOUNT_ENDPOINT` as enpoint and `AZURE_COSMOSDB_NOSQL_ACCOUNT_KEY` as key.
+   * @returns Instance of AzureCosmosNoSqlKVStore
+   */
+  static fromAccountAndKey(
+    config: {
+      endpoint?: string;
+      key?: string;
+    } & AzureCosmosNoSqlChatStoreConfig = {},
+  ): AzureCosmosNoSqlChatStore {
+    const cosmosEndpoint =
+      config.endpoint ||
+      (getEnv("AZURE_COSMOSDB_NOSQL_ACCOUNT_ENDPOINT") as string);
+    const cosmosKey =
+      config.key || (getEnv("AZURE_COSMOSDB_NOSQL_ACCOUNT_KEY") as string);
+
+    if (!cosmosEndpoint || !cosmosKey) {
+      throw new Error(
+        "Azure CosmosDB account endpoint and key must be provided",
+      );
+    }
+    const cosmosClient = new CosmosClient({
+      endpoint: cosmosEndpoint,
+      key: cosmosKey,
+      userAgentSuffix: USER_AGENT_SUFFIX,
+    });
+    return new AzureCosmosNoSqlChatStore({
+      ...config,
+      cosmosClient,
+    });
+  }
+
+  /**
+   * Static method for creating an instance using AAD token.
+   * If no endpoint and credentials are provided, it will attempt to use the env variable `AZURE_COSMOSDB_NOSQL_ACCOUNT_ENDPOINT` as endpoint and use DefaultAzureCredential() as credentials.
+   * @returns Instance of AzureCosmosNoSqlKVStore
+   */
+  static fromAadToken(
+    config: {
+      endpoint?: string;
+      credential?: TokenCredential;
+    } & AzureCosmosNoSqlChatStoreConfig = {},
+  ): AzureCosmosNoSqlChatStore {
+    const cosmosEndpoint =
+      config.endpoint ||
+      (getEnv("AZURE_COSMOSDB_NOSQL_CONNECTION_STRING") as string);
+
+    if (!cosmosEndpoint) {
+      throw new Error("Azure CosmosDB account endpoint must be provided");
+    }
+    const credentials = config.credential ?? new DefaultAzureCredential();
+    const cosmosClient = new CosmosClient({
+      endpoint: cosmosEndpoint,
+      aadCredentials: credentials,
+      userAgentSuffix: USER_AGENT_SUFFIX,
+    });
+    return new AzureCosmosNoSqlChatStore({
+      ...config,
+      cosmosClient,
+    });
+  }
+
+  private convertToChatMessage(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    message: any,
+  ): ChatMessage<AdditionalMessageOptions> {
+    return {
+      content: message.content as MessageContent,
+      role: message.role as MessageType,
+      options: message.options as AdditionalMessageOptions,
+    } as ChatMessage<AdditionalMessageOptions>;
+  }
+
+  private convertToCosmosMessage(
+    message: ChatMessage<AdditionalMessageOptions>,
+  ): // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  any {
+    return {
+      content: message.content,
+      role: message.role,
+      options: message.options,
+    };
+  }
+
+  /**
+   * Set messages for a given key.
+   */
+  async setMessages(
+    key: string,
+    messages: ChatMessage<AdditionalMessageOptions>[],
+  ): Promise<void> {
+    await this.initialize();
+    const inputMessages = messages.map(this.convertToCosmosMessage);
+    await this.container.items.upsert({
+      id: key,
+      messages: inputMessages,
+      userId: this.userId,
+    });
+  }
+
+  /**
+   * Get messages for a given key.
+   */
+  async getMessages(
+    key: string,
+  ): Promise<ChatMessage<AdditionalMessageOptions>[]> {
+    await this.initialize();
+    const res = await this.container.item(key, this.userId).read();
+    const messageHistory = res?.resource?.messages ?? [];
+    const result = messageHistory.map(this.convertToChatMessage);
+    return result;
+  }
+
+  /**
+   * Add a message for a given key.
+   */
+  async addMessage(
+    key: string,
+    message: ChatMessage<AdditionalMessageOptions>,
+    idx?: number,
+  ): Promise<void> {
+    await this.initialize();
+    const res = await this.container.item(key, this.userId).read();
+    const messageHistory = res?.resource?.messages ?? [];
+    if (idx === undefined) {
+      messageHistory.push(this.convertToCosmosMessage(message));
+    } else {
+      messageHistory.splice(idx, 0, this.convertToCosmosMessage(message));
+    }
+    await this.setMessages(key, messageHistory);
+  }
+
+  /**
+   * Deletes all messages for a given key.
+   */
+  async deleteMessages(key: string): Promise<void> {
+    await this.initialize();
+    try {
+      await this.container.item(key, this.userId).delete();
+      // eslint-disable-next-line no-empty
+    } catch (e) {}
+  }
+
+  /**
+   * Deletes one message at idx index for a given key.
+   */
+  async deleteMessage(key: string, idx: number): Promise<void> {
+    await this.initialize();
+    const res = await this.container.item(key, this.userId).read();
+    const messageHistory = res?.resource?.messages ?? [];
+    if (idx >= 0 && idx < messageHistory.length) {
+      messageHistory.splice(idx, 1);
+      await this.setMessages(key, messageHistory);
+    }
+  }
+
+  /**
+   * Get all keys.
+   */
+  async getKeys(): Promise<IterableIterator<string>> {
+    await this.initialize();
+    const result = await this.container.items
+      .query("Select c.id from c")
+      .fetchAll();
+    const keys = result.resources.map((res: { id: string }) => res.id);
+
+    function* keyGenerator(): IterableIterator<string> {
+      for (const key of keys) {
+        yield key;
+      }
+    }
+    return keyGenerator();
+  }
+}

packages/llamaindex/src/storage/index.ts

@@ -2,6 +2,7 @@ export * from "@llamaindex/core/storage/chat-store";
 export * from "@llamaindex/core/storage/doc-store";
 export * from "@llamaindex/core/storage/index-store";
 export * from "@llamaindex/core/storage/kv-store";
+export * from "./chatStore/AzureCosmosNoSqlChatStore.js";
 export * from "./docStore/AzureCosmosNoSqlDocumentStore.js";
 export { PostgresDocumentStore } from "./docStore/PostgresDocumentStore.js";
 export { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js";

packages/llamaindex/src/storage/kvStore/AzureCosmosNoSqlKVStore.ts

@@ -3,7 +3,7 @@ import { Container, CosmosClient, Database } from "@azure/cosmos";
 import { DefaultAzureCredential, type TokenCredential } from "@azure/identity";
 import { BaseKVStore } from "@llamaindex/core/storage/kv-store";
 import { getEnv } from "@llamaindex/env";
-const USER_AGENT_SUFFIX = "LlamaIndex-CDBNoSQL-KVStore-JavaScript";
+const USER_AGENT_SUFFIX = "llamaindex-cdbnosql-kvstore-javascript";
 const DEFAULT_CHAT_DATABASE = "KVStoreDB";
 const DEFAULT_CHAT_CONTAINER = "KVStoreContainer";
 const DEFAULT_OFFER_THROUGHPUT = 400;
@@ -82,7 +82,7 @@ export class AzureCosmosNoSqlKVStore extends BaseKVStore {
     super();
     if (!cosmosClient) {
       throw new Error(
-        "CosmosClient is required for AzureCosmosDBNoSQLVectorStore initialization",
+        "CosmosClient is required for AzureCosmosDBNoSQLKVStore initialization",
       );
     }
     this.cosmosClient = cosmosClient;

packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts

@@ -55,8 +55,18 @@ export interface AzureCosmosDBNoSQLConfig
   readonly flatMetadata?: boolean;
   readonly idKey?: string;
 }
+/**
+ * Query options for the `AzureCosmosDBNoSQLVectorStore.query` method.
+ * @property includeEmbeddings - Whether to include the embeddings in the result. Default false
+ * @property includeVectorDistance - Whether to include the vector distance in the result. Default true
+ * @property whereClause - The where clause to use in the query. While writing this clause, use `c` as the alias for the container and do not include the `WHERE` keyword.
+ */
+export interface AzureCosmosQueryOptions {
+  includeVectorDistance?: boolean;
+  whereClause?: string;
+}
 
-const USER_AGENT_SUFFIX = "LlamaIndex-CDBNoSQL-VectorStore-JavaScript";
+const USER_AGENT_SUFFIX = "llamaindex-cdbnosql-vectorstore-javascript";
 
 const DEFAULT_VECTOR_EMBEDDING_POLICY = {
   vectorEmbeddings: [
@@ -98,6 +108,22 @@ function parseConnectionString(connectionString: string): {
 
   return { endpoint, key: accountKey };
 }
+/**
+ * utility function to build the query string for the CosmosDB query
+ */
+function queryBuilder(options: AzureCosmosQueryOptions): string {
+  let initialQuery =
+    "SELECT TOP @k c[@id] as id, c[@text] as text, c[@metadata] as metadata";
+  if (options.includeVectorDistance !== false) {
+    initialQuery += `, VectorDistance(c[@embeddingKey],@embedding) AS SimilarityScore`;
+  }
+  initialQuery += ` FROM c`;
+  if (options.whereClause) {
+    initialQuery += ` WHERE ${options.whereClause}`;
+  }
+  initialQuery += ` ORDER BY VectorDistance(c[@embeddingKey],@embedding)`;
+  return initialQuery;
+}
 
 export class AzureCosmosDBNoSqlVectorStore extends BaseVectorStore {
   storesText: boolean = true;
@@ -334,21 +360,25 @@ export class AzureCosmosDBNoSqlVectorStore extends BaseVectorStore {
    */
   async query(
     query: VectorStoreQuery,
-    options?: object,
+    options: AzureCosmosQueryOptions = {},
   ): Promise<VectorStoreQueryResult> {
     await this.initialize();
+    if (!query.queryEmbedding || query.queryEmbedding.length === 0) {
+      throw new Error(
+        "queryEmbedding is required for AzureCosmosDBNoSqlVectorStore query",
+      );
+    }
     const params = {
       vector: query.queryEmbedding!,
       k: query.similarityTopK,
     };
-
+    const builtQuery = queryBuilder(options);
     const nodes: BaseNode[] = [];
     const ids: string[] = [];
     const similarities: number[] = [];
     const queryResults = await this.container.items
       .query({
-        query:
-          "SELECT TOP @k c[@id] as id, c[@text] as text, c[@metadata] as metadata, VectorDistance(c[@embeddingKey],@embedding) AS SimilarityScore FROM c ORDER BY VectorDistance(c[@embeddingKey],@embedding)",
+        query: builtQuery,
         parameters: [
           { name: "@k", value: params.k },
           { name: "@id", value: this.idKey },

packages/llamaindex/tests/vector-stores/AzureCosmosDBNoSqlVectorStore.int.test.ts

@@ -14,9 +14,10 @@ import {
   Settings,
   VectorStoreQueryMode,
   type AzureCosmosDBNoSQLConfig,
+  type AzureCosmosQueryOptions,
   type VectorStoreQueryResult,
 } from "llamaindex";
-import { beforeEach, describe, expect, it } from "vitest";
+import { beforeAll, describe, expect, it } from "vitest";
 dotenv.config();
 /*
  * To run this test, you need have an Azure Cosmos DB for NoSQL instance
@@ -64,7 +65,10 @@ Settings.llm = new OpenAI(llmInit);
 Settings.embedModel = new OpenAIEmbedding(embedModelInit);
 // This test is skipped because it requires an Azure Cosmos DB instance and OpenAI API keys
 describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
-  beforeEach(async () => {
+  let vectorStore: AzureCosmosDBNoSqlVectorStore;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let embeddings: any = [];
+  beforeAll(async () => {
     if (process.env.AZURE_COSMOSDB_NOSQL_CONNECTION_STRING) {
       client = new CosmosClient(
         process.env.AZURE_COSMOSDB_NOSQL_CONNECTION_STRING,
@@ -79,15 +83,12 @@ describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
         "Please set the environment variable AZURE_COSMOSDB_NOSQL_CONNECTION_STRING or AZURE_COSMOSDB_NOSQL_ENDPOINT",
       );
     }
-
     // Make sure the database does not exists
     try {
       await client.database(DATABASE_NAME).delete();
     } catch {
       // Ignore error if the database does not exist
     }
-  });
-  it("perform query", async () => {
     const config: AzureCosmosDBNoSQLConfig = {
       idKey: "name",
       textKey: "customText",
@@ -134,9 +135,9 @@ describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
       },
     };
 
-    const vectorStore = new AzureCosmosDBNoSqlVectorStore(config);
+    vectorStore = new AzureCosmosDBNoSqlVectorStore(config);
 
-    const embeddings = await Settings.embedModel.getTextEmbeddings([
+    embeddings = await Settings.embedModel.getTextEmbeddings([
       "This book is about politics",
       "Cats sleeps a lot.",
       "Sandwiches taste good.",
@@ -150,28 +151,29 @@ describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
         id_: "1",
         text: "This book is about politics",
         embedding: embeddings[0],
-        metadata: { key: "politics" },
+        metadata: { key: "politics", number: 1 },
       }),
       new Document({
         id_: "2",
         text: "Cats sleeps a lot.",
         embedding: embeddings[1],
-        metadata: { key: "cats" },
+        metadata: { key: "cats", number: 2 },
       }),
       new Document({
         id_: "3",
         text: "Sandwiches taste good.",
         embedding: embeddings[2],
-        metadata: { key: "sandwiches" },
+        metadata: { key: "sandwiches", number: 3 },
       }),
       new Document({
         id_: "4",
         text: "The house is open",
         embedding: embeddings[3],
-        metadata: { key: "house" },
+        metadata: { key: "house", number: 4 },
       }),
     ]);
-
+  });
+  it("perform query", async () => {
     const results: VectorStoreQueryResult = await vectorStore.query({
       queryEmbedding: embeddings[4] || [],
       similarityTopK: 1,
@@ -179,5 +181,62 @@ describe.skip("AzureCosmosDBNoSQLVectorStore", () => {
     });
     expect(results.ids.length).toEqual(1);
     expect(results.ids[0]).toEqual("3");
+    expect(results.similarities).toBeDefined();
+    expect(results.similarities[0]).toBeDefined();
+  }, 1000000);
+
+  it("perform query with where clause", async () => {
+    const options: AzureCosmosQueryOptions = {
+      whereClause: "c.customMetadata.number > 3",
+    };
+    const results: VectorStoreQueryResult = await vectorStore.query(
+      {
+        queryEmbedding: embeddings[4] || [],
+        similarityTopK: 1,
+        mode: VectorStoreQueryMode.DEFAULT,
+      },
+      options,
+    );
+    expect(results.ids.length).toEqual(1);
+    expect(results.ids[0]).toEqual("4");
+    expect(results.similarities).toBeDefined();
+    expect(results.similarities[0]).toBeDefined();
+  }, 1000000);
+
+  it("perform query with includeVectorDistance false", async () => {
+    const options: AzureCosmosQueryOptions = {
+      includeVectorDistance: false,
+    };
+    const results: VectorStoreQueryResult = await vectorStore.query(
+      {
+        queryEmbedding: embeddings[4] || [],
+        similarityTopK: 1,
+        mode: VectorStoreQueryMode.DEFAULT,
+      },
+      options,
+    );
+    expect(results.ids.length).toEqual(1);
+    expect(results.ids[0]).toEqual("3");
+    expect(results.similarities).toBeDefined();
+    expect(results.similarities[0]).toBeUndefined();
+  }, 1000000);
+
+  it("perform query with includeVectorDistance false and whereClause", async () => {
+    const options: AzureCosmosQueryOptions = {
+      includeVectorDistance: false,
+      whereClause: "c.customMetadata.number > 3",
+    };
+    const results: VectorStoreQueryResult = await vectorStore.query(
+      {
+        queryEmbedding: embeddings[4] || [],
+        similarityTopK: 1,
+        mode: VectorStoreQueryMode.DEFAULT,
+      },
+      options,
+    );
+    expect(results.ids.length).toEqual(1);
+    expect(results.ids[0]).toEqual("4");
+    expect(results.similarities).toBeDefined();
+    expect(results.similarities[0]).toBeUndefined();
   }, 1000000);
 });

packages/llamaindex/tests/vector-stores/AzureCosmosDBNoSqlVectorStore.test.ts

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import type { BaseNode } from "@llamaindex/core/schema";
 import { beforeEach, describe, expect, it, vi } from "vitest";
+import { VectorStoreQueryMode } from "../../src/vector-store.js";
 import { TestableAzureCosmosDBNoSqlVectorStore } from "../mocks/TestableAzureCosmosDBNoSqlVectorStore.js";
 import { createMockClient } from "../utility/mockCosmosClient.js"; // Import the mock client
 
@@ -95,4 +96,27 @@ describe("AzureCosmosDBNoSqlVectorStore Tests", () => {
     expect(client.databases.containers.items.create).toHaveBeenCalledTimes(2);
     expect(result).toEqual(["node-0", "node-1"]);
   });
+
+  it("should throw error if no query embedding is provided", async () => {
+    const client = createMockClient();
+    const store = new TestableAzureCosmosDBNoSqlVectorStore({
+      client: client as any,
+      endpoint: "https://example.com",
+      idKey: "id",
+      textKey: "text",
+      metadataKey: "metadata",
+    });
+
+    expect(store).toBeDefined();
+
+    await expect(
+      store.query({
+        queryEmbedding: [],
+        similarityTopK: 4,
+        mode: VectorStoreQueryMode.DEFAULT,
+      }),
+    ).rejects.toThrowError(
+      "queryEmbedding is required for AzureCosmosDBNoSqlVectorStore query",
+    );
+  });
 });

packages/node-parser/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/node-parser
 
+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
 ## 0.0.10
 
 ### Patch Changes

packages/node-parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/node-parser",
-  "version": "0.0.10",
+  "version": "0.0.12",
   "description": "Node parser for LlamaIndex",
   "type": "module",
   "exports": {

packages/providers/anthropic/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @llamaindex/anthropic
 
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 0.0.19
+
+### Patch Changes
+
+- a7b0ac3: feat(anthropic): support prompt caching
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
 ## 0.0.18
 
 ### Patch Changes

packages/providers/anthropic/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/anthropic",
   "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.0.18",
+  "version": "0.0.20",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -33,7 +33,7 @@
     "bunchee": "5.6.1"
   },
   "dependencies": {
-    "@anthropic-ai/sdk": "0.27.1",
+    "@anthropic-ai/sdk": "0.32.1",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*",
     "remeda": "^2.12.0"

packages/providers/anthropic/src/llm.ts

@@ -1,5 +1,9 @@
 import type { ClientOptions } from "@anthropic-ai/sdk";
 import { Anthropic as SDKAnthropic } from "@anthropic-ai/sdk";
+import type {
+  BetaCacheControlEphemeral,
+  BetaTextBlockParam,
+} from "@anthropic-ai/sdk/resources/beta/index";
 import type {
   TextBlock,
   TextBlockParam,
@@ -8,6 +12,7 @@ import type {
   ImageBlockParam,
   MessageCreateParamsNonStreaming,
   MessageParam,
+  Model,
   Tool,
   ToolResultBlockParam,
   ToolUseBlock,
@@ -75,6 +80,9 @@ export const ALL_AVAILABLE_ANTHROPIC_LEGACY_MODELS = {
   "claude-2.1": {
     contextWindow: 200000,
   },
+  "claude-2.0": {
+    contextWindow: 100000,
+  },
   "claude-instant-1.2": {
     contextWindow: 100000,
   },
@@ -82,18 +90,30 @@ export const ALL_AVAILABLE_ANTHROPIC_LEGACY_MODELS = {
 
 export const ALL_AVAILABLE_V3_MODELS = {
   "claude-3-opus": { contextWindow: 200000 },
+  "claude-3-opus-latest": { contextWindow: 200000 },
+  "claude-3-opus-20240229": { contextWindow: 200000 },
   "claude-3-sonnet": { contextWindow: 200000 },
+  "claude-3-sonnet-20240229": { contextWindow: 200000 },
   "claude-3-haiku": { contextWindow: 200000 },
+  "claude-3-haiku-20240307": { contextWindow: 200000 },
 };
 
 export const ALL_AVAILABLE_V3_5_MODELS = {
   "claude-3-5-sonnet": { contextWindow: 200000 },
+  "claude-3-5-sonnet-20241022": { contextWindow: 200000 },
+  "claude-3-5-sonnet-20240620": { contextWindow: 200000 },
+  "claude-3-5-sonnet-latest": { contextWindow: 200000 },
+  "claude-3-5-haiku": { contextWindow: 200000 },
+  "claude-3-5-haiku-latest": { contextWindow: 200000 },
+  "claude-3-5-haiku-20241022": { contextWindow: 200000 },
 };
 
 export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
   ...ALL_AVAILABLE_ANTHROPIC_LEGACY_MODELS,
   ...ALL_AVAILABLE_V3_MODELS,
   ...ALL_AVAILABLE_V3_5_MODELS,
+} satisfies {
+  [key in Model]: { contextWindow: number };
 };
 
 const AVAILABLE_ANTHROPIC_MODELS_WITHOUT_DATE: { [key: string]: string } = {
@@ -104,10 +124,16 @@ const AVAILABLE_ANTHROPIC_MODELS_WITHOUT_DATE: { [key: string]: string } = {
 } as { [key in keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS]: string };
 
 export type AnthropicAdditionalChatOptions = object;
+export type AnthropicToolCallLLMMessageOptions = ToolCallLLMMessageOptions & {
+  cache_control?: BetaCacheControlEphemeral | null;
+};
 
-export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
+export class Anthropic extends ToolCallLLM<
+  AnthropicAdditionalChatOptions,
+  AnthropicToolCallLLMMessageOptions
+> {
   // Per completion Anthropic params
-  model: keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS;
+  model: keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS | ({} & string);
   temperature: number;
   topP: number;
   maxTokens?: number | undefined;
@@ -147,7 +173,12 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
       temperature: this.temperature,
       topP: this.topP,
       maxTokens: this.maxTokens,
-      contextWindow: ALL_AVAILABLE_ANTHROPIC_MODELS[this.model].contextWindow,
+      contextWindow:
+        this.model in ALL_AVAILABLE_ANTHROPIC_MODELS
+          ? ALL_AVAILABLE_ANTHROPIC_MODELS[
+              this.model as keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS
+            ].contextWindow
+          : 200000,
       tokenizer: undefined,
     };
   }
@@ -291,56 +322,74 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
   chat(
     params: LLMChatParamsStreaming<
       AnthropicAdditionalChatOptions,
-      ToolCallLLMMessageOptions
+      AnthropicToolCallLLMMessageOptions
     >,
-  ): Promise<AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>>;
+  ): Promise<
+    AsyncIterable<ChatResponseChunk<AnthropicToolCallLLMMessageOptions>>
+  >;
   chat(
     params: LLMChatParamsNonStreaming<
       AnthropicAdditionalChatOptions,
-      ToolCallLLMMessageOptions
+      AnthropicToolCallLLMMessageOptions
     >,
-  ): Promise<ChatResponse<ToolCallLLMMessageOptions>>;
+  ): Promise<ChatResponse<AnthropicToolCallLLMMessageOptions>>;
   @wrapLLMEvent
   async chat(
     params:
       | LLMChatParamsNonStreaming<
           AnthropicAdditionalChatOptions,
-          ToolCallLLMMessageOptions
+          AnthropicToolCallLLMMessageOptions
         >
       | LLMChatParamsStreaming<
           AnthropicAdditionalChatOptions,
-          ToolCallLLMMessageOptions
+          AnthropicToolCallLLMMessageOptions
         >,
   ): Promise<
-    | ChatResponse<ToolCallLLMMessageOptions>
-    | AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>
+    | ChatResponse<AnthropicToolCallLLMMessageOptions>
+    | AsyncIterable<ChatResponseChunk<AnthropicToolCallLLMMessageOptions>>
   > {
     let { messages } = params;
 
     const { stream, tools } = params;
 
-    let systemPrompt: string | null = null;
+    let systemPrompt: string | Array<BetaTextBlockParam> | null = null;
 
     const systemMessages = messages.filter(
       (message) => message.role === "system",
     );
 
     if (systemMessages.length > 0) {
-      systemPrompt = systemMessages
-        .map((message) => message.content)
-        .join("\n");
+      systemPrompt = systemMessages.map((message) =>
+        message.options && "cache_control" in message.options
+          ? {
+              type: "text",
+              text: extractText(message.content),
+              cache_control: message.options.cache_control,
+            }
+          : {
+              type: "text",
+              text: extractText(message.content),
+            },
+      );
       messages = messages.filter((message) => message.role !== "system");
     }
+    const beta =
+      systemPrompt?.find((message) => "cache_control" in message) !== undefined;
+
+    // case: Non-streaming
+    let anthropic = this.session.anthropic;
+    if (beta) {
+      // @ts-expect-error type casting
+      anthropic = anthropic.beta.promptCaching;
+    }
 
     // case: Streaming
     if (stream) {
       if (tools) {
         console.error("Tools are not supported in streaming mode");
       }
-      return this.streamChat(messages, systemPrompt);
+      return this.streamChat(messages, systemPrompt, anthropic);
     }
-    // case: Non-streaming
-    const anthropic = this.session.anthropic;
 
     if (tools) {
       const params: MessageCreateParamsNonStreaming = {
@@ -378,7 +427,10 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
                   toolCall: toolUseBlock.map((block) => ({
                     id: block.id,
                     name: block.name,
-                    input: block.input,
+                    input:
+                      typeof block.input === "object"
+                        ? JSON.stringify(block.input)
+                        : `${block.input}`,
                   })),
                 }
               : {},
@@ -411,10 +463,11 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
   }
 
   protected async *streamChat(
-    messages: ChatMessage<ToolCallLLMMessageOptions>[],
-    systemPrompt?: string | null,
-  ): AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>> {
-    const stream = await this.session.anthropic.messages.create({
+    messages: ChatMessage<AnthropicToolCallLLMMessageOptions>[],
+    systemPrompt: string | Array<BetaTextBlockParam> | null,
+    anthropic: SDKAnthropic,
+  ): AsyncIterable<ChatResponseChunk<AnthropicToolCallLLMMessageOptions>> {
+    const stream = await anthropic.messages.create({
       model: this.getModelName(this.model),
       messages: this.formatMessages(messages),
       max_tokens: this.maxTokens ?? 4096,

packages/providers/clip/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/clip
 
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+  - @llamaindex/openai@0.1.36
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+  - @llamaindex/openai@0.1.35
+
 ## 0.0.18
 
 ### Patch Changes

packages/providers/clip/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/clip",
   "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.18",
+  "version": "0.0.20",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/deepinfra/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/deepinfra
 
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+  - @llamaindex/openai@0.1.36
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+  - @llamaindex/openai@0.1.35
+
 ## 0.0.18
 
 ### Patch Changes

packages/providers/deepinfra/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepinfra",
   "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.18",
+  "version": "0.0.20",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/groq/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @llamaindex/groq
 
+## 0.0.35
+
+### Patch Changes
+
+- @llamaindex/openai@0.1.36
+
+## 0.0.34
+
+### Patch Changes
+
+- @llamaindex/openai@0.1.35
+
 ## 0.0.33
 
 ### Patch Changes

packages/providers/groq/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/groq",
   "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.33",
+  "version": "0.0.35",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/huggingface/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/huggingface
 
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+  - @llamaindex/openai@0.1.36
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+  - @llamaindex/openai@0.1.35
+
 ## 0.0.18
 
 ### Patch Changes

packages/providers/huggingface/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/huggingface",
   "description": "Huggingface Adapter for LlamaIndex",
-  "version": "0.0.18",
+  "version": "0.0.20",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/ollama/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/ollama
 
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
 ## 0.0.25
 
 ### Patch Changes

packages/providers/ollama/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/ollama",
   "description": "Ollama Adapter for LlamaIndex",
-  "version": "0.0.25",
+  "version": "0.0.27",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/openai/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/openai
 
+## 0.1.36
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 0.1.35
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
 ## 0.1.34
 
 ### Patch Changes

packages/providers/openai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/openai",
   "description": "OpenAI Adapter for LlamaIndex",
-  "version": "0.1.34",
+  "version": "0.1.36",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/portkey-ai/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/portkey-ai
 
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
 ## 0.0.18
 
 ### Patch Changes

packages/providers/portkey-ai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/portkey-ai",
   "description": "Portkey Adapter for LlamaIndex",
-  "version": "0.0.18",
+  "version": "0.0.20",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/replicate/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/replicate
 
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
 ## 0.0.18
 
 ### Patch Changes

packages/providers/replicate/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/replicate",
   "description": "Replicate Adapter for LlamaIndex",
-  "version": "0.0.18",
+  "version": "0.0.20",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/vllm/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @llamaindex/vllm
 
+## 0.0.6
+
+### Patch Changes
+
+- @llamaindex/openai@0.1.36
+
+## 0.0.5
+
+### Patch Changes
+
+- @llamaindex/openai@0.1.35
+
 ## 0.0.4
 
 ### Patch Changes

packages/providers/vllm/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/vllm",
   "description": "vLLM Adapter for LlamaIndex",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/readers/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @llamaindex/readers
 
+## 1.0.13
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+
+## 1.0.12
+
+### Patch Changes
+
+- e60328b: fix: import error in cjs
+
+## 1.0.11
+
+### Patch Changes
+
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+
 ## 1.0.10
 
 ### Patch Changes

packages/readers/package.json

@@ -1,20 +1,12 @@
 {
   "name": "@llamaindex/readers",
   "description": "LlamaIndex Readers",
-  "version": "1.0.10",
+  "version": "1.0.13",
   "type": "module",
   "exports": {
     "./node/hook": "./node/dist/hook.js",
     "./node": "./node/dist/index.js",
     "./assembly-ai": {
-      "edge-light": {
-        "types": "./assembly-ai/dist/index.edge-light.d.ts",
-        "default": "./assembly-ai/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./assembly-ai/dist/index.workerd.d.ts",
-        "default": "./assembly-ai/dist/index.workerd.js"
-      },
       "require": {
         "types": "./assembly-ai/dist/index.d.cts",
         "default": "./assembly-ai/dist/index.cjs"
@@ -25,14 +17,6 @@
       }
     },
     "./cosmosdb": {
-      "edge-light": {
-        "types": "./cosmosdb/dist/index.edge-light.d.ts",
-        "default": "./cosmosdb/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./cosmosdb/dist/index.workerd.d.ts",
-        "default": "./cosmosdb/dist/index.workerd.js"
-      },
       "require": {
         "types": "./cosmosdb/dist/index.d.cts",
         "default": "./cosmosdb/dist/index.cjs"
@@ -79,14 +63,6 @@
       }
     },
     "./discord": {
-      "edge-light": {
-        "types": "./discord/dist/index.edge-light.d.ts",
-        "default": "./discord/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./discord/dist/index.workerd.d.ts",
-        "default": "./discord/dist/index.workerd.js"
-      },
       "require": {
         "types": "./discord/dist/index.d.cts",
         "default": "./discord/dist/index.cjs"
@@ -97,14 +73,6 @@
       }
     },
     "./docx": {
-      "edge-light": {
-        "types": "./docx/dist/index.edge-light.d.ts",
-        "default": "./docx/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./docx/dist/index.workerd.d.ts",
-        "default": "./docx/dist/index.workerd.js"
-      },
       "require": {
         "types": "./docx/dist/index.d.cts",
         "default": "./docx/dist/index.cjs"
@@ -115,14 +83,6 @@
       }
     },
     "./html": {
-      "edge-light": {
-        "types": "./html/dist/index.edge-light.d.ts",
-        "default": "./html/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./html/dist/index.workerd.d.ts",
-        "default": "./html/dist/index.workerd.js"
-      },
       "require": {
         "types": "./html/dist/index.d.cts",
         "default": "./html/dist/index.cjs"
@@ -133,14 +93,6 @@
       }
     },
     "./image": {
-      "edge-light": {
-        "types": "./image/dist/index.edge-light.d.ts",
-        "default": "./image/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./image/dist/index.workerd.d.ts",
-        "default": "./image/dist/index.workerd.js"
-      },
       "require": {
         "types": "./image/dist/index.d.cts",
         "default": "./image/dist/index.cjs"
@@ -151,14 +103,6 @@
       }
     },
     "./json": {
-      "edge-light": {
-        "types": "./json/dist/index.edge-light.d.ts",
-        "default": "./json/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./json/dist/index.workerd.d.ts",
-        "default": "./json/dist/index.workerd.js"
-      },
       "require": {
         "types": "./json/dist/index.d.cts",
         "default": "./json/dist/index.cjs"
@@ -169,14 +113,6 @@
       }
     },
     "./markdown": {
-      "edge-light": {
-        "types": "./markdown/dist/index.edge-light.d.ts",
-        "default": "./markdown/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./markdown/dist/index.workerd.d.ts",
-        "default": "./markdown/dist/index.workerd.js"
-      },
       "require": {
         "types": "./markdown/dist/index.d.cts",
         "default": "./markdown/dist/index.cjs"
@@ -187,14 +123,6 @@
       }
     },
     "./mongo": {
-      "edge-light": {
-        "types": "./mongo/dist/index.edge-light.d.ts",
-        "default": "./mongo/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./mongo/dist/index.workerd.d.ts",
-        "default": "./mongo/dist/index.workerd.js"
-      },
       "require": {
         "types": "./mongo/dist/index.d.cts",
         "default": "./mongo/dist/index.cjs"
@@ -205,14 +133,6 @@
       }
     },
     "./notion": {
-      "edge-light": {
-        "types": "./notion/dist/index.edge-light.d.ts",
-        "default": "./notion/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./notion/dist/index.workerd.d.ts",
-        "default": "./notion/dist/index.workerd.js"
-      },
       "require": {
         "types": "./notion/dist/index.d.cts",
         "default": "./notion/dist/index.cjs"
@@ -223,14 +143,6 @@
       }
     },
     "./pdf": {
-      "edge-light": {
-        "types": "./pdf/dist/index.edge-light.d.ts",
-        "default": "./pdf/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./pdf/dist/index.workerd.d.ts",
-        "default": "./pdf/dist/index.workerd.js"
-      },
       "require": {
         "types": "./pdf/dist/index.d.cts",
         "default": "./pdf/dist/index.cjs"
@@ -241,14 +153,6 @@
       }
     },
     "./text": {
-      "edge-light": {
-        "types": "./text/dist/index.edge-light.d.ts",
-        "default": "./text/dist/index.edge-light.js"
-      },
-      "workerd": {
-        "types": "./text/dist/index.workerd.d.ts",
-        "default": "./text/dist/index.workerd.js"
-      },
       "require": {
         "types": "./text/dist/index.d.cts",
         "default": "./text/dist/index.cjs"

packages/workflow/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/workflow
 
+## 0.0.6
+
+### Patch Changes
+
+- ee20c44: feat(workflow): allow send event with no output
+
 ## 0.0.5
 
 ### Patch Changes

packages/workflow/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/workflow",
   "description": "Workflow API",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "type": "module",
   "types": "dist/index.d.ts",
   "module": "dist/index.js",

packages/workflow/src/workflow-context.ts

@@ -13,19 +13,18 @@ export type StepHandler<
     AnyWorkflowEventConstructor | StartEventConstructor,
     ...(AnyWorkflowEventConstructor | StopEventConstructor)[],
   ] = [AnyWorkflowEventConstructor | StartEventConstructor],
-  Out extends [
-    AnyWorkflowEventConstructor | StartEventConstructor,
-    ...(AnyWorkflowEventConstructor | StopEventConstructor)[],
-  ] = [AnyWorkflowEventConstructor | StopEventConstructor],
+  Out extends (AnyWorkflowEventConstructor | StopEventConstructor)[] = [],
 > = (
   context: HandlerContext<Data>,
   ...events: {
     [K in keyof Inputs]: InstanceType<Inputs[K]>;
   }
 ) => Promise<
-  {
-    [K in keyof Out]: InstanceType<Out[K]>;
-  }[number]
+  Out extends []
+    ? void
+    : {
+        [K in keyof Out]: InstanceType<Out[K]>;
+      }[number]
 >;
 
 export type ReadonlyStepMap<Data> = ReadonlyMap<
@@ -275,7 +274,7 @@ export class WorkflowContext<Start = string, Stop = string, Data = unknown>
    */
   #createStreamEvents(): AsyncIterableIterator<WorkflowEvent<unknown>> {
     const isPendingEvents = new WeakSet<WorkflowEvent<unknown>>();
-    const pendingTasks = new Set<Promise<WorkflowEvent<unknown>>>();
+    const pendingTasks = new Set<Promise<WorkflowEvent<unknown> | void>>();
     const enqueuedEvents = new Set<WorkflowEvent<unknown>>();
     const stream = new ReadableStream<WorkflowEvent<unknown>>({
       start: async (controller) => {
@@ -325,102 +324,104 @@ export class WorkflowContext<Start = string, Stop = string, Data = unknown>
                   }
                   const [steps, inputsMap, outputsMap] =
                     this.#getStepFunction(event);
-                  const nextEventPromises: Promise<WorkflowEvent<unknown>>[] = [
-                    ...steps,
-                  ]
-                    .map((step) => {
-                      const inputs = [...(inputsMap.get(step) ?? [])];
-                      const acceptableInputs: WorkflowEvent<unknown>[] =
-                        this.#pendingInputQueue.filter((event) =>
-                          inputs.some((input) => event instanceof input),
-                        );
-                      const events: WorkflowEvent<unknown>[] = flattenEvents(
-                        inputs,
-                        [event, ...acceptableInputs],
-                      );
-                      // remove the event from the queue, in case of infinite loop
-                      events.forEach((event) => {
-                        const protocolIdx = this.#queue.findIndex(
-                          (protocol) =>
-                            protocol.type === "event" &&
-                            protocol.event === event,
-                        );
-                        if (protocolIdx !== -1) {
-                          this.#queue.splice(protocolIdx, 1);
-                        }
-                      });
-                      if (events.length !== inputs.length) {
-                        if (this.#verbose) {
-                          console.log(
-                            `Not enough inputs for step ${step.name}, waiting for more events`,
+                  const nextEventPromises: Promise<WorkflowEvent<unknown> | void>[] =
+                    [...steps]
+                      .map((step) => {
+                        const inputs = [...(inputsMap.get(step) ?? [])];
+                        const acceptableInputs: WorkflowEvent<unknown>[] =
+                          this.#pendingInputQueue.filter((event) =>
+                            inputs.some((input) => event instanceof input),
                           );
-                        }
-                        // not enough to run the step, push back to the queue
-                        this.#sendEvent(event);
-                        isPendingEvents.add(event);
-                        return null;
-                      }
-                      if (isPendingEvents.has(event)) {
-                        isPendingEvents.delete(event);
-                      }
-                      if (this.#verbose) {
-                        console.log(
-                          `Running step ${step.name} with inputs ${events}`,
+                        const events: WorkflowEvent<unknown>[] = flattenEvents(
+                          inputs,
+                          [event, ...acceptableInputs],
                         );
-                      }
-                      const data = this.data;
-                      return (step as StepHandler<Data>)
-                        .call(
-                          null,
-                          {
-                            get data() {
-                              return data;
-                            },
-                            sendEvent: this.#sendEvent,
-                            requireEvent: this.#requireEvent,
-                          },
-                          // @ts-expect-error IDK why
-                          ...events.sort((a, b) => {
-                            const aIndex = inputs.indexOf(
-                              a.constructor as AnyWorkflowEventConstructor,
-                            );
-                            const bIndex = inputs.indexOf(
-                              b.constructor as AnyWorkflowEventConstructor,
-                            );
-                            return aIndex - bIndex;
-                          }),
-                        )
-                        .then((nextEvent) => {
+                        // remove the event from the queue, in case of infinite loop
+                        events.forEach((event) => {
+                          const protocolIdx = this.#queue.findIndex(
+                            (protocol) =>
+                              protocol.type === "event" &&
+                              protocol.event === event,
+                          );
+                          if (protocolIdx !== -1) {
+                            this.#queue.splice(protocolIdx, 1);
+                          }
+                        });
+                        if (events.length !== inputs.length) {
                           if (this.#verbose) {
                             console.log(
-                              `Step ${step.name} completed, next event is ${nextEvent}`,
+                              `Not enough inputs for step ${step.name}, waiting for more events`,
                             );
                           }
-                          const outputs = outputsMap.get(step) ?? [];
-                          if (
-                            !outputs.some(
-                              (output) => nextEvent.constructor === output,
-                            )
-                          ) {
-                            if (this.#strict) {
-                              const error = Error(
-                                `Step ${step.name} returned an unexpected output event ${nextEvent}`,
+                          // not enough to run the step, push back to the queue
+                          this.#sendEvent(event);
+                          isPendingEvents.add(event);
+                          return null;
+                        }
+                        if (isPendingEvents.has(event)) {
+                          isPendingEvents.delete(event);
+                        }
+                        if (this.#verbose) {
+                          console.log(
+                            `Running step ${step.name} with inputs ${events}`,
+                          );
+                        }
+                        const data = this.data;
+                        return (step as StepHandler<Data>)
+                          .call(
+                            null,
+                            {
+                              get data() {
+                                return data;
+                              },
+                              sendEvent: this.#sendEvent,
+                              requireEvent: this.#requireEvent,
+                            },
+                            // @ts-expect-error IDK why
+                            ...events.sort((a, b) => {
+                              const aIndex = inputs.indexOf(
+                                a.constructor as AnyWorkflowEventConstructor,
                               );
-                              controller.error(error);
-                            } else {
-                              console.warn(
-                                `Step ${step.name} returned an unexpected output event ${nextEvent}`,
+                              const bIndex = inputs.indexOf(
+                                b.constructor as AnyWorkflowEventConstructor,
+                              );
+                              return aIndex - bIndex;
+                            }),
+                          )
+                          .then((nextEvent: void | WorkflowEvent<unknown>) => {
+                            if (nextEvent === undefined) {
+                              return;
+                            }
+                            if (this.#verbose) {
+                              console.log(
+                                `Step ${step.name} completed, next event is ${nextEvent}`,
                               );
                             }
-                          }
-                          if (!(nextEvent instanceof StopEvent)) {
-                            this.#pendingInputQueue.unshift(nextEvent);
-                            this.#sendEvent(nextEvent);
-                          }
-                          return nextEvent;
-                        });
-                    })
-                    .filter((promise) => promise !== null);
+                            const outputs = outputsMap.get(step) ?? [];
+                            if (
+                              !outputs.some(
+                                (output) => nextEvent.constructor === output,
+                              )
+                            ) {
+                              if (this.#strict) {
+                                const error = Error(
+                                  `Step ${step.name} returned an unexpected output event ${nextEvent}`,
+                                );
+                                controller.error(error);
+                              } else {
+                                console.warn(
+                                  `Step ${step.name} returned an unexpected output event ${nextEvent}`,
+                                );
+                              }
+                            }
+                            if (!(nextEvent instanceof StopEvent)) {
+                              this.#pendingInputQueue.unshift(nextEvent);
+                              this.#sendEvent(nextEvent);
+                            }
+                            return nextEvent;
+                          });
+                      })
+                      .filter((promise) => promise !== null);
                   nextEventPromises.forEach((promise) => {
                     pendingTasks.add(promise);
                     promise
@@ -433,6 +434,9 @@ export class WorkflowContext<Start = string, Stop = string, Data = unknown>
                   });
                   Promise.race(nextEventPromises)
                     .then((fastestNextEvent) => {
+                      if (fastestNextEvent === undefined) {
+                        return;
+                      }
                       if (!enqueuedEvents.has(fastestNextEvent)) {
                         controller.enqueue(fastestNextEvent);
                         enqueuedEvents.add(fastestNextEvent);
@@ -441,7 +445,10 @@ export class WorkflowContext<Start = string, Stop = string, Data = unknown>
                     })
                     .then(async (fastestNextEvent) =>
                       Promise.all(nextEventPromises).then((nextEvents) => {
-                        for (const nextEvent of nextEvents) {
+                        const events = nextEvents.filter(
+                          (event) => event !== undefined,
+                        );
+                        for (const nextEvent of events) {
                           // do not enqueue the same event twice
                           if (fastestNextEvent !== nextEvent) {
                             if (!enqueuedEvents.has(nextEvent)) {

packages/workflow/src/workflow.ts

@@ -57,10 +57,7 @@ export class Workflow<ContextData, Start, Stop> {
       AnyWorkflowEventConstructor | StartEventConstructor,
       ...(AnyWorkflowEventConstructor | StopEventConstructor)[],
     ],
-    const Out extends [
-      AnyWorkflowEventConstructor | StopEventConstructor,
-      ...(AnyWorkflowEventConstructor | StopEventConstructor)[],
-    ],
+    const Out extends (AnyWorkflowEventConstructor | StopEventConstructor)[],
   >(
     parameters: StepParameters<In, Out>,
     stepFn: (
@@ -69,9 +66,11 @@ export class Workflow<ContextData, Start, Stop> {
         [K in keyof In]: InstanceType<In[K]>;
       }
     ) => Promise<
-      {
-        [K in keyof Out]: InstanceType<Out[K]>;
-      }[number]
+      Out extends []
+        ? void
+        : {
+            [K in keyof Out]: InstanceType<Out[K]>;
+          }[number]
     >,
   ): this {
     const { inputs, outputs } = parameters;

pnpm-lock.yaml

@@ -127,8 +127,8 @@ importers:
         specifier: ^10.1.0
         version: 10.1.0(react@18.3.1)
       '@llamaindex/chat-ui':
-        specifier: 0.0.5
-        version: 0.0.5(@types/react-dom@18.3.1)(@types/react@18.3.12)(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
+        specifier: 0.0.8
+        version: 0.0.8(@types/react-dom@18.3.1)(@types/react@18.3.12)(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
       '@llamaindex/cloud':
         specifier: workspace:*
         version: link:../../packages/cloud
@@ -173,10 +173,10 @@ importers:
         version: 1.1.3(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
       '@vercel/functions':
         specifier: ^1.5.0
-        version: 1.5.0(@aws-sdk/credential-provider-web-identity@3.679.0(@aws-sdk/client-sts@3.682.0))
+        version: 1.5.0(@aws-sdk/credential-provider-web-identity@3.679.0)
       ai:
         specifier: ^3.4.31
-        version: 3.4.31(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@5.1.9))(svelte@5.1.9)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8)
+        version: 3.4.31(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))(react@18.3.1)(sswr@2.1.0)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8)
       class-variance-authority:
         specifier: ^0.7.0
         version: 0.7.0
@@ -257,7 +257,7 @@ importers:
         version: 1.22.2
       shiki-magic-move:
         specifier: ^0.5.0
-        version: 0.5.0(react@18.3.1)(shiki@1.22.2)(svelte@5.1.9)(vue@3.5.12(typescript@5.6.3))
+        version: 0.5.0(react@18.3.1)(shiki@1.22.2)(vue@3.5.12(typescript@5.6.3))
       swr:
         specifier: ^2.2.5
         version: 2.2.5(react@18.3.1)
@@ -428,7 +428,7 @@ importers:
     dependencies:
       ai:
         specifier: ^3.3.21
-        version: 3.4.31(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@5.1.9))(svelte@5.1.9)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8)
+        version: 3.4.31(openai@4.69.0(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@5.1.9))(svelte@5.1.9)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8)
       llamaindex:
         specifier: workspace:*
         version: link:../../../packages/llamaindex
@@ -584,13 +584,13 @@ importers:
         specifier: ^1.4.1
         version: 1.5.0
       '@llamaindex/core':
-        specifier: ^0.4.7
+        specifier: ^0.4.10
         version: link:../packages/core
       '@llamaindex/readers':
-        specifier: ^1.0.8
+        specifier: ^1.0.11
         version: link:../packages/readers
       '@llamaindex/workflow':
-        specifier: ^0.0.5
+        specifier: ^0.0.6
         version: link:../packages/workflow
       '@notionhq/client':
         specifier: ^2.2.15
@@ -606,7 +606,7 @@ importers:
         version: 2.4.9
       chromadb:
         specifier: ^1.8.1
-        version: 1.9.2(cohere-ai@7.14.0(encoding@0.1.13))(encoding@0.1.13)(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
+        version: 1.9.2(cohere-ai@7.14.0(encoding@0.1.13))(encoding@0.1.13)(openai@4.69.0(encoding@0.1.13))
       commander:
         specifier: ^12.1.0
         version: 12.1.0
@@ -617,7 +617,7 @@ importers:
         specifier: ^1.0.14
         version: 1.0.15
       llamaindex:
-        specifier: ^0.8.10
+        specifier: ^0.8.13
         version: link:../packages/llamaindex
       mongodb:
         specifier: ^6.7.0
@@ -738,7 +738,7 @@ importers:
         version: 1.1.0(@types/react@18.3.12)(react@18.3.1)
       ai:
         specifier: ^3.3.21
-        version: 3.4.31(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@5.1.9))(svelte@5.1.9)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8)
+        version: 3.4.31(openai@4.69.0(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@5.1.9))(svelte@5.1.9)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8)
       class-variance-authority:
         specifier: ^0.7.0
         version: 0.7.0
@@ -1191,8 +1191,8 @@ importers:
   packages/providers/anthropic:
     dependencies:
       '@anthropic-ai/sdk':
-        specifier: 0.27.1
-        version: 0.27.1(encoding@0.1.13)
+        specifier: 0.32.1
+        version: 0.32.1(encoding@0.1.13)
       '@llamaindex/core':
         specifier: workspace:*
         version: link:../../core
@@ -1701,6 +1701,9 @@ packages:
   '@anthropic-ai/sdk@0.27.1':
     resolution: {integrity: sha512-AKFd/E8HO26+DOVPiZpEked3Pm2feA5d4gcX2FcJXr9veDkXbKO90hr2C7N2TL7mPIMwm040ldXlsIZQ416dHg==}
 
+  '@anthropic-ai/sdk@0.32.1':
+    resolution: {integrity: sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==}
+
   '@apidevtools/json-schema-ref-parser@11.7.2':
     resolution: {integrity: sha512-4gY54eEGEstClvEkGnwVkTkrx0sqwemEFG5OSRRn3tD91XH0+Q8XIkYIfo7IwEWPpJZwILb9GUXeShtplRc/eA==}
     engines: {node: '>= 16'}
@@ -3752,8 +3755,8 @@ packages:
   '@leichtgewicht/ip-codec@2.0.5':
     resolution: {integrity: sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==}
 
-  '@llamaindex/chat-ui@0.0.5':
-    resolution: {integrity: sha512-nGlhtAeIlVBtHAsCtBN5f+/iJpIHQ7uFf43ebye/wATf5tT12HUl3tP5O/+XC7nyXXwByjk5KAEN+hk3aC1Cug==}
+  '@llamaindex/chat-ui@0.0.8':
+    resolution: {integrity: sha512-yTWgxDzJD6F8w17CNIIJkSBE+hyd+WRqhF6lSElqo3k7CQyhtYr8cOD7cS6K8gBqaI3Xha5EPqvqV0+enJA0Mg==}
     peerDependencies:
       react: ^18.2.0
 
@@ -13425,6 +13428,18 @@ snapshots:
     transitivePeerDependencies:
       - encoding
 
+  '@anthropic-ai/sdk@0.32.1(encoding@0.1.13)':
+    dependencies:
+      '@types/node': 18.19.64
+      '@types/node-fetch': 2.6.11
+      abort-controller: 3.0.0
+      agentkeepalive: 4.5.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.7.0(encoding@0.1.13)
+    transitivePeerDependencies:
+      - encoding
+
   '@apidevtools/json-schema-ref-parser@11.7.2':
     dependencies:
       '@jsdevtools/ono': 7.1.3
@@ -16519,7 +16534,7 @@ snapshots:
 
   '@leichtgewicht/ip-codec@2.0.5': {}
 
-  '@llamaindex/chat-ui@0.0.5(@types/react-dom@18.3.1)(@types/react@18.3.12)(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
+  '@llamaindex/chat-ui@0.0.8(@types/react-dom@18.3.1)(@types/react@18.3.12)(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
     dependencies:
       '@llamaindex/pdf-viewer': 1.2.0(@types/react@18.3.12)(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
       '@radix-ui/react-collapsible': 1.1.1(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -18601,7 +18616,7 @@ snapshots:
 
   '@upstash/vector@1.1.7': {}
 
-  '@vercel/functions@1.5.0(@aws-sdk/credential-provider-web-identity@3.679.0(@aws-sdk/client-sts@3.682.0))':
+  '@vercel/functions@1.5.0(@aws-sdk/credential-provider-web-identity@3.679.0)':
     optionalDependencies:
       '@aws-sdk/credential-provider-web-identity': 3.679.0(@aws-sdk/client-sts@3.682.0)
 
@@ -18879,7 +18894,7 @@ snapshots:
       clean-stack: 2.2.0
       indent-string: 4.0.0
 
-  ai@3.4.31(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@5.1.9))(svelte@5.1.9)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8):
+  ai@3.4.31(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))(react@18.3.1)(sswr@2.1.0)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8):
     dependencies:
       '@ai-sdk/provider': 0.0.26
       '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8)
@@ -18898,6 +18913,30 @@ snapshots:
       openai: 4.69.0(encoding@0.1.13)(zod@3.23.8)
       react: 18.3.1
       sswr: 2.1.0(svelte@5.1.9)
+      zod: 3.23.8
+    transitivePeerDependencies:
+      - solid-js
+      - vue
+
+  ai@3.4.31(openai@4.69.0(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@5.1.9))(svelte@5.1.9)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8):
+    dependencies:
+      '@ai-sdk/provider': 0.0.26
+      '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8)
+      '@ai-sdk/react': 0.0.70(react@18.3.1)(zod@3.23.8)
+      '@ai-sdk/solid': 0.0.54(zod@3.23.8)
+      '@ai-sdk/svelte': 0.0.57(svelte@5.1.9)(zod@3.23.8)
+      '@ai-sdk/ui-utils': 0.0.50(zod@3.23.8)
+      '@ai-sdk/vue': 0.0.59(vue@3.5.12(typescript@5.6.3))(zod@3.23.8)
+      '@opentelemetry/api': 1.9.0
+      eventsource-parser: 1.1.2
+      json-schema: 0.4.0
+      jsondiffpatch: 0.6.0
+      secure-json-parse: 2.7.0
+      zod-to-json-schema: 3.23.5(zod@3.23.8)
+    optionalDependencies:
+      openai: 4.69.0(zod@3.23.8)
+      react: 18.3.1
+      sswr: 2.1.0(svelte@5.1.9)
       svelte: 5.1.9
       zod: 3.23.8
     transitivePeerDependencies:
@@ -19585,7 +19624,7 @@ snapshots:
     transitivePeerDependencies:
       - encoding
 
-  chromadb@1.9.2(cohere-ai@7.14.0(encoding@0.1.13))(encoding@0.1.13)(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)):
+  chromadb@1.9.2(cohere-ai@7.14.0(encoding@0.1.13))(encoding@0.1.13)(openai@4.69.0(encoding@0.1.13)):
     dependencies:
       cliui: 8.0.1
       isomorphic-fetch: 3.0.0(encoding@0.1.13)
@@ -20672,7 +20711,7 @@ snapshots:
       debug: 4.3.7
       enhanced-resolve: 5.17.1
       eslint: 9.14.0(jiti@2.4.0)
-      eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.13.0(eslint@9.14.0(jiti@2.4.0))(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@9.14.0(jiti@2.4.0))
+      eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.13.0(eslint@9.14.0(jiti@2.4.0))(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.13.0(eslint@9.14.0(jiti@2.4.0))(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@9.14.0(jiti@2.4.0)))(eslint@9.14.0(jiti@2.4.0))
       fast-glob: 3.3.2
       get-tsconfig: 4.8.1
       is-bun-module: 1.1.0
@@ -20685,7 +20724,7 @@ snapshots:
       - eslint-import-resolver-webpack
       - supports-color
 
-  eslint-module-utils@2.12.0(@typescript-eslint/parser@8.13.0(eslint@9.14.0(jiti@2.4.0))(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@9.14.0(jiti@2.4.0)):
+  eslint-module-utils@2.12.0(@typescript-eslint/parser@8.13.0(eslint@9.14.0(jiti@2.4.0))(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.13.0(eslint@9.14.0(jiti@2.4.0))(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@9.14.0(jiti@2.4.0)))(eslint@9.14.0(jiti@2.4.0)):
     dependencies:
       debug: 3.2.7
     optionalDependencies:
@@ -20707,7 +20746,7 @@ snapshots:
       doctrine: 2.1.0
       eslint: 9.14.0(jiti@2.4.0)
       eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.13.0(eslint@9.14.0(jiti@2.4.0))(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@9.14.0(jiti@2.4.0))
+      eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.13.0(eslint@9.14.0(jiti@2.4.0))(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.13.0(eslint@9.14.0(jiti@2.4.0))(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@9.14.0(jiti@2.4.0)))(eslint@9.14.0(jiti@2.4.0))
       hasown: 2.0.2
       is-core-module: 2.15.1
       is-glob: 4.0.3
@@ -24373,6 +24412,21 @@ snapshots:
     transitivePeerDependencies:
       - encoding
 
+  openai@4.69.0(zod@3.23.8):
+    dependencies:
+      '@types/node': 18.19.64
+      '@types/node-fetch': 2.6.11
+      abort-controller: 3.0.0
+      agentkeepalive: 4.5.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.7.0(encoding@0.1.13)
+    optionalDependencies:
+      zod: 3.23.8
+    transitivePeerDependencies:
+      - encoding
+    optional: true
+
   openapi-sampler@1.5.1:
     dependencies:
       '@types/json-schema': 7.0.15
@@ -26244,14 +26298,13 @@ snapshots:
       interpret: 1.4.0
       rechoir: 0.6.2
 
-  shiki-magic-move@0.5.0(react@18.3.1)(shiki@1.22.2)(svelte@5.1.9)(vue@3.5.12(typescript@5.6.3)):
+  shiki-magic-move@0.5.0(react@18.3.1)(shiki@1.22.2)(vue@3.5.12(typescript@5.6.3)):
     dependencies:
       diff-match-patch-es: 0.1.1
       ohash: 1.1.4
     optionalDependencies:
       react: 18.3.1
       shiki: 1.22.2
-      svelte: 5.1.9
       vue: 3.5.12(typescript@5.6.3)
 
   shiki@1.22.2:

unit/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/unit-test
 
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [6d22fa2]
+  - @llamaindex/core@0.4.11
+  - @llamaindex/cloud@2.0.11
+  - llamaindex@0.8.17
+  - @llamaindex/node-parser@0.0.12
+  - @llamaindex/openai@0.1.36
+  - @llamaindex/readers@1.0.13
+
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [e60328b]
+  - @llamaindex/readers@1.0.12
+  - llamaindex@0.8.16
+
+## 0.0.25
+
+### Patch Changes
+
+- Updated dependencies [3d503cb]
+- Updated dependencies [5dae534]
+  - llamaindex@0.8.15
+
+## 0.0.24
+
+### Patch Changes
+
+- Updated dependencies [630b425]
+  - llamaindex@0.8.14
+
+## 0.0.23
+
+### Patch Changes
+
+- ee20c44: feat(workflow): allow send event with no output
+- Updated dependencies [a7b0ac3]
+- Updated dependencies [ee20c44]
+- Updated dependencies [c69605f]
+  - @llamaindex/core@0.4.10
+  - @llamaindex/workflow@0.0.6
+  - llamaindex@0.8.13
+  - @llamaindex/cloud@2.0.10
+  - @llamaindex/node-parser@0.0.11
+  - @llamaindex/openai@0.1.35
+  - @llamaindex/readers@1.0.11
+
 ## 0.0.22
 
 ### Patch Changes

unit/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/unit-test",
   "private": true,
-  "version": "0.0.22",
+  "version": "0.0.27",
   "type": "module",
   "scripts": {
     "test": "vitest run"

unit/workflow/workflow.test.ts

@@ -531,6 +531,21 @@ describe("workflow basic", () => {
     const result = await myWorkflow.run("start");
     expect(result.data).toBe("query result");
   });
+
+  test("allow output with send event", async () => {
+    const myFlow = new Workflow<unknown, string, string>({ verbose: true });
+    myFlow.addStep(
+      {
+        inputs: [StartEvent<string>],
+        outputs: [],
+      },
+      async (context, ev) => {
+        context.sendEvent(new StopEvent(`Hello ${ev.data}!`));
+      },
+    );
+    const result = myFlow.run("world");
+    expect((await result).data).toBe("Hello world!");
+  });
 });
 
 describe("workflow event loop", () => {