Release 0.3.4 (#797)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

apps/docs/CHANGELOG.md

@@ -1,5 +1,24 @@
 # docs
 
+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [1dce275]
+- Updated dependencies [d10533e]
+- Updated dependencies [2008efe]
+- Updated dependencies [5e61934]
+- Updated dependencies [9e74a43]
+- Updated dependencies [ee719a1]
+  - llamaindex@0.3.4
+
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [e8c41c5]
+  - llamaindex@0.3.3
+
 ## 0.0.10
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.10",
+  "version": "0.0.12",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/.env.example

@@ -0,0 +1 @@
+DEBUG=llamaindex

examples/agent/openai-task.ts

@@ -1,63 +1,15 @@
-import { ChatResponseChunk, FunctionTool, OpenAIAgent } from "llamaindex";
+import { ChatResponseChunk, OpenAIAgent } from "llamaindex";
 import { ReadableStream } from "node:stream/web";
-
-const functionTool = FunctionTool.from(
-  () => {
-    console.log("Getting user id...");
-    return crypto.randomUUID();
-  },
-  {
-    name: "get_user_id",
-    description: "Get a random user id",
-  },
-);
-
-const functionTool2 = FunctionTool.from(
-  ({ userId }: { userId: string }) => {
-    console.log("Getting user info...", userId);
-    return `Name: Alex; Address: 1234 Main St, CA; User ID: ${userId}`;
-  },
-  {
-    name: "get_user_info",
-    description: "Get user info",
-    parameters: {
-      type: "object",
-      properties: {
-        userId: {
-          type: "string",
-          description: "The user id",
-        },
-      },
-      required: ["userId"],
-    },
-  },
-);
-
-const functionTool3 = FunctionTool.from(
-  ({ address }: { address: string }) => {
-    console.log("Getting weather...", address);
-    return `${address} is in a sunny location!`;
-  },
-  {
-    name: "get_weather",
-    description: "Get the current weather for a location",
-    parameters: {
-      type: "object",
-      properties: {
-        address: {
-          type: "string",
-          description: "The address",
-        },
-      },
-      required: ["address"],
-    },
-  },
-);
+import {
+  getCurrentIDTool,
+  getUserInfoTool,
+  getWeatherTool,
+} from "./utils/tools";
 
 async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
-    tools: [functionTool, functionTool2, functionTool3],
+    tools: [getCurrentIDTool, getUserInfoTool, getWeatherTool],
   });
 
   const task = await agent.createTask(

examples/agent/openai.ts

@@ -53,7 +53,7 @@ async function main() {
     message: "How much is 5 + 5? then divide by 2",
   });
 
-  console.log(String(response));
+  console.log(response.response.message);
 }
 
 void main().then(() => {

examples/agent/react-task.ts

@@ -0,0 +1,40 @@
+import { ChatResponseChunk, ReActAgent } from "llamaindex";
+import { ReadableStream } from "node:stream/web";
+import {
+  getCurrentIDTool,
+  getUserInfoTool,
+  getWeatherTool,
+} from "./utils/tools";
+
+async function main() {
+  // Create an OpenAIAgent with the function tools
+  const agent = new ReActAgent({
+    tools: [getCurrentIDTool, getUserInfoTool, getWeatherTool],
+  });
+
+  const task = await agent.createTask(
+    "What is my current address weather based on my profile?",
+    true,
+  );
+
+  for await (const stepOutput of task) {
+    const stream = stepOutput.output as ReadableStream<ChatResponseChunk>;
+    if (stepOutput.isLast) {
+      for await (const chunk of stream) {
+        process.stdout.write(chunk.delta);
+      }
+      process.stdout.write("\n");
+    } else {
+      // handing function call
+      console.log("handling function call...");
+      for await (const chunk of stream) {
+        console.log("debug:", JSON.stringify(chunk.raw));
+      }
+    }
+    console.log("---");
+  }
+}
+
+void main().then(() => {
+  console.log("Done");
+});

examples/agent/utils/tools.ts

@@ -0,0 +1,54 @@
+import { FunctionTool } from "llamaindex";
+
+export const getCurrentIDTool = FunctionTool.from(
+  () => {
+    console.log("Getting user id...");
+    return crypto.randomUUID();
+  },
+  {
+    name: "get_user_id",
+    description: "Get a random user id",
+  },
+);
+
+export const getUserInfoTool = FunctionTool.from(
+  ({ userId }: { userId: string }) => {
+    console.log("Getting user info...", userId);
+    return `Name: Alex; Address: 1234 Main St, CA; User ID: ${userId}`;
+  },
+  {
+    name: "get_user_info",
+    description: "Get user info",
+    parameters: {
+      type: "object",
+      properties: {
+        userId: {
+          type: "string",
+          description: "The user id",
+        },
+      },
+      required: ["userId"],
+    },
+  },
+);
+
+export const getWeatherTool = FunctionTool.from(
+  ({ address }: { address: string }) => {
+    console.log("Getting weather...", address);
+    return `${address} is in a sunny location!`;
+  },
+  {
+    name: "get_weather",
+    description: "Get the current weather for a location",
+    parameters: {
+      type: "object",
+      properties: {
+        address: {
+          type: "string",
+          description: "The address",
+        },
+      },
+      required: ["address"],
+    },
+  },
+);

examples/huggingface/chat.ts

@@ -0,0 +1,22 @@
+import { HuggingFaceInferenceAPI } from "llamaindex";
+
+(async () => {
+  if (!process.env.HUGGING_FACE_TOKEN) {
+    throw new Error("Please set the HUGGING_FACE_TOKEN environment variable.");
+  }
+  const hf = new HuggingFaceInferenceAPI({
+    accessToken: process.env.HUGGING_FACE_TOKEN,
+    model: "mistralai/Mixtral-8x7B-Instruct-v0.1",
+  });
+  const result = await hf.chat({
+    messages: [
+      { content: "You want to talk in rhymes.", role: "system" },
+      {
+        content:
+          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
+        role: "user",
+      },
+    ],
+  });
+  console.log(result);
+})();

packages/core/CHANGELOG.md

@@ -1,5 +1,22 @@
 # llamaindex
 
+## 0.3.4
+
+### Patch Changes
+
+- 1dce275: fix: export `StorageContext` on edge runtime
+- d10533e: feat: add hugging face llm
+- 2008efe: feat: add verbose mode to Agent
+- 5e61934: fix: remove clone object in `CallbackManager.dispatchEvent`
+- 9e74a43: feat: add top k to `asQueryEngine`
+- ee719a1: fix: streaming for ReAct Agent
+
+## 0.3.3
+
+### Patch Changes
+
+- e8c41c5: fix: wrong gemini streaming chat response
+
 ## 0.3.2
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.5
+
+### Patch Changes
+
+- Updated dependencies [1dce275]
+- Updated dependencies [d10533e]
+- Updated dependencies [2008efe]
+- Updated dependencies [5e61934]
+- Updated dependencies [9e74a43]
+- Updated dependencies [ee719a1]
+  - llamaindex@0.3.4
+
+## 0.0.4
+
+### Patch Changes
+
+- Updated dependencies [e8c41c5]
+  - llamaindex@0.3.3
+
 ## 0.0.3
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/next-agent-test
 
+## 0.1.5
+
+### Patch Changes
+
+- Updated dependencies [1dce275]
+- Updated dependencies [d10533e]
+- Updated dependencies [2008efe]
+- Updated dependencies [5e61934]
+- Updated dependencies [9e74a43]
+- Updated dependencies [ee719a1]
+  - llamaindex@0.3.4
+
+## 0.1.4
+
+### Patch Changes
+
+- Updated dependencies [e8c41c5]
+  - llamaindex@0.3.3
+
 ## 0.1.3
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.5
+
+### Patch Changes
+
+- Updated dependencies [1dce275]
+- Updated dependencies [d10533e]
+- Updated dependencies [2008efe]
+- Updated dependencies [5e61934]
+- Updated dependencies [9e74a43]
+- Updated dependencies [ee719a1]
+  - llamaindex@0.3.4
+
+## 0.0.4
+
+### Patch Changes
+
+- Updated dependencies [e8c41c5]
+  - llamaindex@0.3.3
+
 ## 0.0.3
 
 ### Patch Changes

packages/core/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/node/react.e2e.ts

@@ -27,3 +27,23 @@ await test("react agent", async (t) => {
     ok(extractText(response.message.content).includes("72"));
   });
 });
+
+await test("react agent stream", async (t) => {
+  await mockLLMEvent(t, "react-agent-stream");
+  await t.test("get weather", async () => {
+    const agent = new ReActAgent({
+      tools: [getWeatherTool],
+    });
+
+    const stream = await agent.chat({
+      stream: true,
+      message: "What is the weather like in San Francisco?",
+    });
+
+    let content = "";
+    for await (const { response } of stream) {
+      content += response.delta;
+    }
+    ok(content.includes("72"));
+  });
+});

packages/core/e2e/node/snapshot/react-agent-stream.snap

@@ -0,0 +1,488 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "role": "system",
+          "content": "You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.\n\n## Tools\nYou have access to a wide variety of tools. You are responsible for using\nthe tools in any sequence you deem appropriate to complete the task at hand.\nThis may require breaking the task into subtasks and using different tools\nto complete each subtask.\n\nYou have access to the following tools:\n- getWeather: Get the weather for a city with schema: {\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\",\"description\":\"The city to get the weather for\"}},\"required\":[\"city\"]}\n\n## Output Format\nTo answer the question, please use the following format.\n\n\"\"\"\nThought: I need to use a tool to help me answer the question.\nAction: tool name (one of getWeather) if using a tool.\nAction Input: the input to the tool, in a JSON format representing the kwargs (e.g. {{\"input\": \"hello world\", \"num_beams\": 5}})\n\"\"\"\n\nPlease ALWAYS start with a Thought.\n\nPlease use a valid JSON format for the Action Input. Do NOT do this {{'input': 'hello world', 'num_beams': 5}}.\n\nIf this format is used, the user will respond in the following format:\n\n\"\"\"\"\nObservation: tool response\n\"\"\"\"\n\nYou should keep repeating the above format until you have enough information\nto answer the question without using any more tools. At that point, you MUST respond\nin the one of the following two formats:\n\n\"\"\"\"\nThought: I can answer without using any more tools.\nAnswer: [your answer here]\n\"\"\"\"\n\n\"\"\"\"\nThought: I cannot answer the question with the provided tools.\nAnswer: Sorry, I cannot answer your query.\n\"\"\"\"\n\n## Current Conversation\nBelow is the current conversation consisting of interleaving human and assistant messages."
+        },
+        {
+          "role": "user",
+          "content": "What is the weather like in San Francisco?"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_1",
+      "messages": [
+        {
+          "role": "system",
+          "content": "You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.\n\n## Tools\nYou have access to a wide variety of tools. You are responsible for using\nthe tools in any sequence you deem appropriate to complete the task at hand.\nThis may require breaking the task into subtasks and using different tools\nto complete each subtask.\n\nYou have access to the following tools:\n- getWeather: Get the weather for a city with schema: {\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\",\"description\":\"The city to get the weather for\"}},\"required\":[\"city\"]}\n\n## Output Format\nTo answer the question, please use the following format.\n\n\"\"\"\nThought: I need to use a tool to help me answer the question.\nAction: tool name (one of getWeather) if using a tool.\nAction Input: the input to the tool, in a JSON format representing the kwargs (e.g. {{\"input\": \"hello world\", \"num_beams\": 5}})\n\"\"\"\n\nPlease ALWAYS start with a Thought.\n\nPlease use a valid JSON format for the Action Input. Do NOT do this {{'input': 'hello world', 'num_beams': 5}}.\n\nIf this format is used, the user will respond in the following format:\n\n\"\"\"\"\nObservation: tool response\n\"\"\"\"\n\nYou should keep repeating the above format until you have enough information\nto answer the question without using any more tools. At that point, you MUST respond\nin the one of the following two formats:\n\n\"\"\"\"\nThought: I can answer without using any more tools.\nAnswer: [your answer here]\n\"\"\"\"\n\n\"\"\"\"\nThought: I cannot answer the question with the provided tools.\nAnswer: Sorry, I cannot answer your query.\n\"\"\"\"\n\n## Current Conversation\nBelow is the current conversation consisting of interleaving human and assistant messages."
+        },
+        {
+          "role": "user",
+          "content": "What is the weather like in San Francisco?"
+        },
+        {
+          "role": "assistant",
+          "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nInput: {\n  city: San Francisco\n}"
+        },
+        {
+          "role": "user",
+          "content": "Observation: The weather in San Francisco is 72 degrees"
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "raw": null,
+        "message": {
+          "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "response": {
+        "raw": null,
+        "message": {
+          "content": "Thought: I can answer without using any more tools.\nAnswer: The weather in San Francisco is 72 degrees.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": [
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "Thought"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": ":"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " I"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " need"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " to"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " use"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " a"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " tool"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " to"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " help"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " me"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " answer"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " the"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " question"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": ".\n"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "Action"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": ":"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " get"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "Weather"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "\n"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "Action"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " Input"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": ":"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " {\""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "city"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "\":"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " \""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "San"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " Francisco"
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "\"}"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "Thought"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": ":"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " I"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " can"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " answer"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " without"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " using"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " any"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " more"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " tools"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": ".\n"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "Answer"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": ":"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " The"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " weather"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " in"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " San"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " Francisco"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " is"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " "
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "72"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " degrees"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "."
+      }
+    }
+  ]
+}

packages/core/jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/core",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "exports": "./src/index.ts",
   "imports": {
     "@llamaindex/env": "jsr:@llamaindex/env@0.0.6"

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "expectedMinorVersion": "3",
   "license": "MIT",
   "type": "module",
@@ -10,6 +10,7 @@
     "@datastax/astra-db-ts": "^1.0.1",
     "@google/generative-ai": "^0.8.0",
     "@grpc/grpc-js": "^1.10.6",
+    "@huggingface/inference": "^2.6.7",
     "@llamaindex/cloud": "0.0.5",
     "@llamaindex/env": "workspace:*",
     "@mistralai/mistralai": "^0.1.3",

packages/core/src/Settings.ts

@@ -55,9 +55,9 @@ class GlobalSettings implements Config {
   get debug() {
     const debug = getEnv("DEBUG");
     return (
-      getEnv("NODE_ENV") === "development" &&
-      Boolean(debug) &&
-      debug?.includes("llamaindex")
+      (Boolean(debug) && debug?.includes("llamaindex")) ||
+      debug === "*" ||
+      debug === "true"
     );
   }
 

packages/core/src/agent/anthropic.ts

@@ -49,6 +49,7 @@ export class AnthropicAgent extends AgentRunner<Anthropic> {
         "tools" in params
           ? params.tools
           : params.toolRetriever.retrieve.bind(params.toolRetriever),
+      verbose: params.verbose ?? false,
     });
   }
 
@@ -94,7 +95,11 @@ export class AnthropicAgent extends AgentRunner<Anthropic> {
       const targetTool = tools.find(
         (tool) => tool.metadata.name === toolCall.name,
       );
-      const toolOutput = await callTool(targetTool, toolCall);
+      const toolOutput = await callTool(
+        targetTool,
+        toolCall,
+        step.context.logger,
+      );
       step.context.store.toolOutputs.push(toolOutput);
       step.context.store.messages = [
         ...step.context.store.messages,

packages/core/src/agent/base.ts

@@ -4,12 +4,14 @@ import {
   pipeline,
   randomUUID,
 } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
 import {
   type ChatEngine,
   type ChatEngineParamsNonStreaming,
   type ChatEngineParamsStreaming,
 } from "../engines/chat/index.js";
 import { wrapEventCaller } from "../internal/context/EventCaller.js";
+import { consoleLogger, emptyLogger } from "../internal/logger.js";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import { isAsyncIterable } from "../internal/utils.js";
 import type {
@@ -66,6 +68,7 @@ export function createTaskOutputStream<
       const enqueueOutput = (
         output: TaskStepOutput<Model, Store, AdditionalMessageOptions>,
       ) => {
+        context.logger.log("Enqueueing output for step(id, %s).", step.id);
         taskOutputs.push(output);
         controller.enqueue(output);
       };
@@ -75,7 +78,9 @@ export function createTaskOutputStream<
         },
       });
 
+      context.logger.log("Starting step(id, %s).", step.id);
       await handler(step, enqueueOutput);
+      context.logger.log("Finished step(id, %s).", step.id);
       // fixme: support multi-thread when there are multiple outputs
       // todo: for now we pretend there is only one task output
       const { isLast, taskStep } = taskOutputs[0];
@@ -87,6 +92,10 @@ export function createTaskOutputStream<
         toolCallCount: 1,
       };
       if (isLast) {
+        context.logger.log(
+          "Final step(id, %s) reached, closing task.",
+          step.id,
+        );
         getCallbackManager().dispatchEvent("agent-end", {
           payload: {
             endStep: step,
@@ -125,6 +134,7 @@ export type AgentRunnerParams<
   tools:
     | BaseToolWithCall[]
     | ((query: MessageContent) => Promise<BaseToolWithCall[]>);
+  verbose: boolean;
 };
 
 export type AgentParamsBase<
@@ -139,6 +149,7 @@ export type AgentParamsBase<
   llm?: AI;
   chatHistory?: ChatMessage<AdditionalMessageOptions>[];
   systemPrompt?: MessageContent;
+  verbose?: boolean;
 };
 
 /**
@@ -218,6 +229,7 @@ export abstract class AgentRunner<
   readonly #systemPrompt: MessageContent | null = null;
   #chatHistory: ChatMessage<AdditionalMessageOptions>[];
   readonly #runner: AgentWorker<AI, Store, AdditionalMessageOptions>;
+  readonly #verbose: boolean;
 
   // create extra store
   abstract createStore(): Store;
@@ -229,14 +241,15 @@ export abstract class AgentRunner<
   protected constructor(
     params: AgentRunnerParams<AI, Store, AdditionalMessageOptions>,
   ) {
-    const { llm, chatHistory, runner, tools } = params;
+    const { llm, chatHistory, systemPrompt, runner, tools, verbose } = params;
     this.#llm = llm;
     this.#chatHistory = chatHistory;
     this.#runner = runner;
-    if (params.systemPrompt) {
-      this.#systemPrompt = params.systemPrompt;
+    if (systemPrompt) {
+      this.#systemPrompt = systemPrompt;
     }
     this.#tools = tools;
+    this.#verbose = verbose;
   }
 
   get llm() {
@@ -247,6 +260,10 @@ export abstract class AgentRunner<
     return this.#chatHistory;
   }
 
+  get verbose(): boolean {
+    return Settings.debug || this.#verbose;
+  }
+
   public reset(): void {
     this.#chatHistory = [];
   }
@@ -270,8 +287,11 @@ export abstract class AgentRunner<
     return task.context.toolCallCount < MAX_TOOL_CALLS;
   }
 
-  // fixme: this shouldn't be async
-  async createTask(message: MessageContent, stream: boolean = false) {
+  createTask(
+    message: MessageContent,
+    stream: boolean = false,
+    verbose: boolean | undefined = undefined,
+  ) {
     const initialMessages = [...this.#chatHistory];
     if (this.#systemPrompt !== null) {
       const systemPrompt = this.#systemPrompt;
@@ -296,6 +316,13 @@ export abstract class AgentRunner<
         toolOutputs: [] as ToolOutput[],
       },
       shouldContinue: AgentRunner.shouldContinue,
+      logger:
+        // disable verbose if explicitly set to false
+        verbose === false
+          ? emptyLogger
+          : verbose || this.verbose
+            ? consoleLogger
+            : emptyLogger,
     });
   }
 
@@ -312,7 +339,7 @@ export abstract class AgentRunner<
     | AgentChatResponse<AdditionalMessageOptions>
     | ReadableStream<AgentStreamChatResponse<AdditionalMessageOptions>>
   > {
-    const task = await this.createTask(params.message, !!params.stream);
+    const task = this.createTask(params.message, !!params.stream);
     const stepOutput = await pipeline(
       task,
       async (

packages/core/src/agent/openai.ts

@@ -46,6 +46,7 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
         "tools" in params
           ? params.tools
           : params.toolRetriever.retrieve.bind(params.toolRetriever),
+      verbose: params.verbose ?? false,
     });
   }
 
@@ -77,7 +78,11 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
         const targetTool = tools.find(
           (tool) => tool.metadata.name === toolCall.name,
         );
-        const toolOutput = await callTool(targetTool, toolCall);
+        const toolOutput = await callTool(
+          targetTool,
+          toolCall,
+          step.context.logger,
+        );
         step.context.store.toolOutputs.push(toolOutput);
         step.context.store.messages = [
           ...step.context.store.messages,
@@ -154,7 +159,11 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
               },
             },
           ];
-          const toolOutput = await callTool(targetTool, toolCall);
+          const toolOutput = await callTool(
+            targetTool,
+            toolCall,
+            step.context.logger,
+          );
           step.context.store.messages = [
             ...step.context.store.messages,
             {

packages/core/src/agent/react.ts

@@ -1,5 +1,4 @@
-import { pipeline, randomUUID } from "@llamaindex/env";
-import { Settings } from "../Settings.js";
+import { randomUUID, ReadableStream } from "@llamaindex/env";
 import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
 import {
   isAsyncIterable,
@@ -13,6 +12,7 @@ import {
 } from "../llm/index.js";
 import { extractText } from "../llm/utils.js";
 import { ObjectRetriever } from "../objects/index.js";
+import { Settings } from "../Settings.js";
 import type {
   BaseTool,
   BaseToolWithCall,
@@ -60,7 +60,7 @@ type ActionReason = BaseReason & {
 type ResponseReason = BaseReason & {
   type: "response";
   thought: string;
-  response: ChatResponse | AsyncIterable<ChatResponseChunk>;
+  response: ChatResponse;
 };
 
 type Reason = ObservationReason | ActionReason | ResponseReason;
@@ -74,16 +74,9 @@ function reasonFormatter(reason: Reason): string | Promise<string> {
         reason.input,
       )}`;
     case "response": {
-      if (isAsyncIterable(reason.response)) {
-        return consumeAsyncIterable(reason.response).then(
-          (message) =>
-            `Thought: ${reason.thought}\nAnswer: ${extractText(message.content)}`,
-        );
-      } else {
-        return `Thought: ${reason.thought}\nAnswer: ${extractText(
-          reason.response.message.content,
-        )}`;
-      }
+      return `Thought: ${reason.thought}\nAnswer: ${extractText(
+        reason.response.message.content,
+      )}`;
     }
   }
 }
@@ -146,35 +139,52 @@ function actionInputParser(jsonStr: string): JSONObject {
 
 type ReACTOutputParser = <Options extends object>(
   output: ChatResponse<Options> | AsyncIterable<ChatResponseChunk<Options>>,
+  onResolveType: (
+    type: "action" | "thought" | "answer",
+    response:
+      | ChatResponse<Options>
+      | ReadableStream<ChatResponseChunk<Options>>,
+  ) => void,
 ) => Promise<Reason>;
 
 const reACTOutputParser: ReACTOutputParser = async (
   output,
+  onResolveType,
 ): Promise<Reason> => {
   let reason: Reason | null = null;
 
   if (isAsyncIterable(output)) {
     const [peakStream, finalStream] = createReadableStream(output).tee();
-    const type = await pipeline(peakStream, async (iter) => {
-      let content = "";
-      for await (const chunk of iter) {
-        content += chunk.delta;
-        if (content.includes("Action:")) {
-          return "action";
-        } else if (content.includes("Answer:")) {
-          return "answer";
-        } else if (content.includes("Thought:")) {
-          return "thought";
-        }
+    const reader = peakStream.getReader();
+    let type: "action" | "thought" | "answer" | null = null;
+    let content = "";
+    do {
+      const { done, value } = await reader.read();
+      if (done) {
+        break;
       }
-    });
+      content += value.delta;
+      if (content.includes("Action:")) {
+        type = "action";
+      } else if (content.includes("Answer:")) {
+        type = "answer";
+      }
+    } while (true);
+    if (type === null) {
+      // `Thought:` is always present at the beginning of the output.
+      type = "thought";
+    }
+    reader.releaseLock();
+    if (!type) {
+      throw new Error("Could not determine type of output");
+    }
+    onResolveType(type, finalStream);
     // step 2: do the parsing from content
     switch (type) {
       case "action": {
         // have to consume the stream to get the full content
-        const response = await consumeAsyncIterable(finalStream);
-        const { content } = response;
-        const [thought, action, input] = extractToolUse(content);
+        const response = await consumeAsyncIterable(peakStream, content);
+        const [thought, action, input] = extractToolUse(response.content);
         const jsonStr = extractJsonStr(input);
         let json: JSONObject;
         try {
@@ -192,18 +202,20 @@ const reACTOutputParser: ReACTOutputParser = async (
       }
       case "thought": {
         const thought = "(Implicit) I can answer without any more tools!";
+        const response = await consumeAsyncIterable(peakStream, content);
         reason = {
           type: "response",
           thought,
-          // bypass the response, because here we don't need to do anything with it
-          response: finalStream,
+          response: {
+            raw: peakStream,
+            message: response,
+          },
         };
         break;
       }
       case "answer": {
-        const response = await consumeAsyncIterable(finalStream);
-        const { content } = response;
-        const [thought, answer] = extractFinalResponse(content);
+        const response = await consumeAsyncIterable(peakStream, content);
+        const [thought, answer] = extractFinalResponse(response.content);
         reason = {
           type: "response",
           thought,
@@ -227,7 +239,9 @@ const reACTOutputParser: ReACTOutputParser = async (
       ? "answer"
       : content.includes("Action:")
         ? "action"
-        : "thought";
+        : // `Thought:` is always present at the beginning of the output.
+          "thought";
+    onResolveType(type, output);
 
     // step 2: do the parsing from content
     switch (type) {
@@ -340,6 +354,7 @@ export class ReActAgent extends AgentRunner<LLM, ReACTAgentStore> {
         "tools" in params
           ? params.tools
           : params.toolRetriever.retrieve.bind(params.toolRetriever),
+      verbose: params.verbose ?? false,
     });
   }
 
@@ -366,20 +381,26 @@ export class ReActAgent extends AgentRunner<LLM, ReACTAgentStore> {
       stream,
       messages,
     });
-    const reason = await reACTOutputParser(response);
-    step.context.store.reasons = [...step.context.store.reasons, reason];
-    enqueueOutput({
-      taskStep: step,
-      output: response,
-      isLast: reason.type === "response",
+    const reason = await reACTOutputParser(response, (type, response) => {
+      enqueueOutput({
+        taskStep: step,
+        output: response,
+        isLast: type !== "action",
+      });
     });
+    step.context.logger.log("current reason: %O", reason);
+    step.context.store.reasons = [...step.context.store.reasons, reason];
     if (reason.type === "action") {
       const tool = tools.find((tool) => tool.metadata.name === reason.action);
-      const toolOutput = await callTool(tool, {
-        id: randomUUID(),
-        input: reason.input,
-        name: reason.action,
-      });
+      const toolOutput = await callTool(
+        tool,
+        {
+          id: randomUUID(),
+          input: reason.input,
+          name: reason.action,
+        },
+        step.context.logger,
+      );
       step.context.store.reasons = [
         ...step.context.store.reasons,
         {

packages/core/src/agent/types.ts

@@ -1,4 +1,5 @@
 import { ReadableStream } from "@llamaindex/env";
+import type { Logger } from "../internal/logger.js";
 import type { BaseEvent } from "../internal/type.js";
 import type {
   ChatMessage,
@@ -32,6 +33,7 @@ export type AgentTaskContext<
     toolOutputs: ToolOutput[];
     messages: ChatMessage<AdditionalMessageOptions>[];
   } & Store;
+  logger: Readonly<Logger>;
 };
 
 export type TaskStep<

packages/core/src/agent/utils.ts

@@ -1,4 +1,5 @@
 import { ReadableStream } from "@llamaindex/env";
+import type { Logger } from "../internal/logger.js";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import { isAsyncIterable, prettifyError } from "../internal/utils.js";
 import type {
@@ -13,12 +14,14 @@ import type { BaseTool, JSONObject, JSONValue, ToolOutput } from "../types.js";
 export async function callTool(
   tool: BaseTool | undefined,
   toolCall: ToolCall | PartialToolCall,
+  logger: Logger,
 ): Promise<ToolOutput> {
   const input: JSONObject =
     typeof toolCall.input === "string"
       ? JSON.parse(toolCall.input)
       : toolCall.input;
   if (!tool) {
+    logger.error(`Tool ${toolCall.name} does not exist.`);
     const output = `Tool ${toolCall.name} does not exist.`;
     return {
       tool,
@@ -30,6 +33,9 @@ export async function callTool(
   const call = tool.call;
   let output: JSONValue;
   if (!call) {
+    logger.error(
+      `Tool ${tool.metadata.name} (remote:${toolCall.name}) does not have a implementation.`,
+    );
     output = `Tool ${tool.metadata.name} (remote:${toolCall.name}) does not have a implementation.`;
     return {
       tool,
@@ -45,6 +51,10 @@ export async function callTool(
       },
     });
     output = await call.call(tool, input);
+    logger.log(
+      `Tool ${tool.metadata.name} (remote:${toolCall.name}) succeeded.`,
+    );
+    logger.log(`Output: ${JSON.stringify(output)}`);
     const toolOutput: ToolOutput = {
       tool,
       input,
@@ -60,6 +70,9 @@ export async function callTool(
     return toolOutput;
   } catch (e) {
     output = prettifyError(e);
+    logger.error(
+      `Tool ${tool.metadata.name} (remote:${toolCall.name}) failed: ${output}`,
+    );
   }
   return {
     tool,
@@ -71,16 +84,19 @@ export async function callTool(
 
 export async function consumeAsyncIterable<Options extends object>(
   input: ChatMessage<Options>,
+  previousContent?: string,
 ): Promise<ChatMessage<Options>>;
 export async function consumeAsyncIterable<Options extends object>(
   input: AsyncIterable<ChatResponseChunk<Options>>,
+  previousContent?: string,
 ): Promise<TextChatMessage<Options>>;
 export async function consumeAsyncIterable<Options extends object>(
   input: ChatMessage<Options> | AsyncIterable<ChatResponseChunk<Options>>,
+  previousContent: string = "",
 ): Promise<ChatMessage<Options>> {
   if (isAsyncIterable(input)) {
     const result: ChatMessage<Options> = {
-      content: "",
+      content: previousContent,
       // only assistant will give streaming response
       role: "assistant",
       options: {} as Options,

packages/core/src/callbacks/CallbackManager.ts

@@ -212,10 +212,13 @@ export class CallbackManager implements CallbackManagerMethods {
     if (!handlers) {
       return;
     }
-    const clone = structuredClone(detail);
     queueMicrotask(() => {
       handlers.forEach((handler) =>
-        handler(LlamaIndexCustomEvent.fromEvent(event, clone)),
+        handler(
+          LlamaIndexCustomEvent.fromEvent(event, {
+            ...detail,
+          }),
+        ),
       );
     });
   }

packages/core/src/engines/chat/types.ts

@@ -13,6 +13,11 @@ export interface ChatEngineParamsBase {
    * Optional chat history if you want to customize the chat history.
    */
   chatHistory?: ChatMessage[] | ChatHistory;
+  /**
+   * Optional flag to enable verbose mode.
+   * @default false
+   */
+  verbose?: boolean;
 }
 
 export interface ChatEngineParamsStreaming extends ChatEngineParamsBase {

packages/core/src/index.edge.ts

@@ -27,6 +27,7 @@ export * from "./objects/index.js";
 export * from "./postprocessors/index.js";
 export * from "./prompts/index.js";
 export * from "./selectors/index.js";
+export * from "./storage/StorageContext.js";
 export * from "./synthesizers/index.js";
 export * from "./tools/index.js";
 export * from "./types.js";

packages/core/src/indices/vectorStore/index.ts

@@ -279,18 +279,29 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
     return new VectorIndexRetriever({ index: this, ...options });
   }
 
+  /**
+   * Create a RetrieverQueryEngine.
+   * similarityTopK is only used if no existing retriever is provided.
+   */
   asQueryEngine(options?: {
     retriever?: BaseRetriever;
     responseSynthesizer?: BaseSynthesizer;
     preFilters?: MetadataFilters;
     nodePostprocessors?: BaseNodePostprocessor[];
+    similarityTopK?: number;
   }): QueryEngine & RetrieverQueryEngine {
-    const { retriever, responseSynthesizer } = options ?? {};
-    return new RetrieverQueryEngine(
-      retriever ?? this.asRetriever(),
+    const {
+      retriever,
       responseSynthesizer,
-      options?.preFilters,
-      options?.nodePostprocessors,
+      preFilters,
+      nodePostprocessors,
+      similarityTopK,
+    } = options ?? {};
+    return new RetrieverQueryEngine(
+      retriever ?? this.asRetriever({ similarityTopK }),
+      responseSynthesizer,
+      preFilters,
+      nodePostprocessors,
     );
   }
 

packages/core/src/internal/logger.ts

@@ -0,0 +1,17 @@
+export type Logger = {
+  log: (...args: unknown[]) => void;
+  error: (...args: unknown[]) => void;
+  warn: (...args: unknown[]) => void;
+};
+
+export const emptyLogger: Logger = Object.freeze({
+  log: () => {},
+  error: () => {},
+  warn: () => {},
+});
+
+export const consoleLogger: Logger = Object.freeze({
+  log: console.log.bind(console),
+  error: console.error.bind(console),
+  warn: console.warn.bind(console),
+});

packages/core/src/llm/gemini.ts

@@ -302,12 +302,10 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
   ): GeminiChatStreamResponse {
     const { chat, messageContent } = this.prepareChat(params);
     const result = await chat.sendMessageStream(messageContent);
-    return streamConverter(result.stream, (response) => {
-      return {
-        text: response.text(),
-        raw: response,
-      };
-    });
+    yield* streamConverter(result.stream, (response) => ({
+      delta: response.text(),
+      raw: response,
+    }));
   }
 
   chat(params: GeminiChatParamsStreaming): Promise<GeminiChatStreamResponse>;

packages/core/src/llm/huggingface.ts

@@ -0,0 +1,141 @@
+import {
+  HfInference,
+  type Options as HfInferenceOptions,
+} from "@huggingface/inference";
+import { BaseLLM } from "./base.js";
+import type {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMMetadata,
+  ToolCallLLMMessageOptions,
+} from "./types.js";
+import { streamConverter, wrapLLMEvent } from "./utils.js";
+
+const DEFAULT_PARAMS = {
+  temperature: 0.1,
+  topP: 1,
+  maxTokens: undefined,
+  contextWindow: 3900,
+};
+export type HFConfig = Partial<typeof DEFAULT_PARAMS> &
+  HfInferenceOptions & {
+    model: string;
+    accessToken: string;
+    endpoint?: string;
+  };
+
+/**
+    Wrapper on the Hugging Face's Inference API.
+    API Docs: https://huggingface.co/docs/huggingface.js/inference/README
+    List of tasks with models: huggingface.co/api/tasks 
+    
+    Note that Conversational API is not yet supported by the Inference API.
+    They recommend using the text generation API instead.
+    See: https://github.com/huggingface/huggingface.js/issues/586#issuecomment-2024059308
+ */
+export class HuggingFaceInferenceAPI extends BaseLLM {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  contextWindow: number;
+  hf: HfInference;
+
+  constructor(init: HFConfig) {
+    super();
+    const {
+      model,
+      temperature,
+      topP,
+      maxTokens,
+      contextWindow,
+      accessToken,
+      endpoint,
+      ...hfInferenceOpts
+    } = init;
+    this.hf = new HfInference(accessToken, hfInferenceOpts);
+    this.model = model;
+    this.temperature = temperature ?? DEFAULT_PARAMS.temperature;
+    this.topP = topP ?? DEFAULT_PARAMS.topP;
+    this.maxTokens = maxTokens ?? DEFAULT_PARAMS.maxTokens;
+    this.contextWindow = contextWindow ?? DEFAULT_PARAMS.contextWindow;
+    if (endpoint) this.hf.endpoint(endpoint);
+  }
+
+  get metadata(): LLMMetadata {
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow: this.contextWindow,
+      tokenizer: undefined,
+    };
+  }
+
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  @wrapLLMEvent
+  async chat(
+    params: LLMChatParamsStreaming | LLMChatParamsNonStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk> | ChatResponse<object>> {
+    if (params.stream) return this.streamChat(params);
+    return this.nonStreamChat(params);
+  }
+
+  private messagesToPrompt(messages: ChatMessage<ToolCallLLMMessageOptions>[]) {
+    let prompt = "";
+    for (const message of messages) {
+      if (message.role === "system") {
+        prompt += `<|system|>\n${message.content}</s>\n`;
+      } else if (message.role === "user") {
+        prompt += `<|user|>\n${message.content}</s>\n`;
+      } else if (message.role === "assistant") {
+        prompt += `<|assistant|>\n${message.content}</s>\n`;
+      }
+    }
+    // ensure we start with a system prompt, insert blank if needed
+    if (!prompt.startsWith("<|system|>\n")) {
+      prompt = "<|system|>\n</s>\n" + prompt;
+    }
+    // add final assistant prompt
+    prompt = prompt + "<|assistant|>\n";
+    return prompt;
+  }
+
+  protected async nonStreamChat(
+    params: LLMChatParamsNonStreaming,
+  ): Promise<ChatResponse> {
+    const res = await this.hf.textGeneration({
+      model: this.model,
+      inputs: this.messagesToPrompt(params.messages),
+      parameters: this.metadata,
+    });
+    return {
+      raw: res,
+      message: {
+        content: res.generated_text,
+        role: "assistant",
+      },
+    };
+  }
+
+  protected async *streamChat(
+    params: LLMChatParamsStreaming,
+  ): AsyncIterable<ChatResponseChunk> {
+    const stream = this.hf.textGenerationStream({
+      model: this.model,
+      inputs: this.messagesToPrompt(params.messages),
+      parameters: this.metadata,
+    });
+    yield* streamConverter(stream, (chunk) => ({
+      delta: chunk.token.text,
+      raw: chunk,
+    }));
+  }
+}

packages/core/src/llm/index.ts

@@ -7,6 +7,7 @@ export {
 export { FireworksLLM } from "./fireworks.js";
 export { GEMINI_MODEL, Gemini } from "./gemini.js";
 export { Groq } from "./groq.js";
+export { HuggingFaceInferenceAPI } from "./huggingface.js";
 export {
   ALL_AVAILABLE_MISTRAL_MODELS,
   MistralAI,

packages/edge/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/edge",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "license": "MIT",
   "type": "module",
   "dependencies": {
@@ -9,6 +9,7 @@
     "@datastax/astra-db-ts": "^1.0.1",
     "@google/generative-ai": "^0.8.0",
     "@grpc/grpc-js": "^1.10.6",
+    "@huggingface/inference": "^2.6.7",
     "@llamaindex/cloud": "0.0.5",
     "@llamaindex/env": "workspace:*",
     "@mistralai/mistralai": "^0.1.3",

packages/experimental/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/experimental
 
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [1dce275]
+- Updated dependencies [d10533e]
+- Updated dependencies [2008efe]
+- Updated dependencies [5e61934]
+- Updated dependencies [9e74a43]
+- Updated dependencies [ee719a1]
+  - llamaindex@0.3.4
+
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [e8c41c5]
+  - llamaindex@0.3.3
+
 ## 0.0.19
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.19",
+  "version": "0.0.21",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",