Release 0.7.2 (#1355)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: himself65 <himself65@users.noreply.github.com>

.github/workflows/preview.yml

@@ -25,4 +25,4 @@ jobs:
         run: pnpm run build
 
       - name: Pre Release
-        run: pnpx pkg-pr-new publish ./packages/*
+        run: pnpx pkg-pr-new publish ./packages/* ./packages/providers/*

.github/workflows/test.yml

@@ -89,13 +89,7 @@ jobs:
       - name: Run Type Check
         run: pnpm run type-check
       - name: Run Circular Dependency Check
-        run: pnpm dlx turbo run circular-check
-      - uses: actions/upload-artifact@v3
-        if: failure()
-        with:
-          name: typecheck-build-dist
-          path: ./packages/llamaindex/dist
-          if-no-files-found: error
+        run: pnpm run circular-check
   e2e-llamaindex-examples:
     strategy:
       fail-fast: false
@@ -142,27 +136,36 @@ jobs:
         run: pnpm run build
       - name: Copy examples
         run: rsync -rv --exclude=node_modules ./examples ${{ runner.temp }}
-      - name: Pack @llamaindex/cloud
-        run: pnpm pack --pack-destination ${{ runner.temp }}
-        working-directory: packages/cloud
-      - name: Pack @llamaindex/openai
-        run: pnpm pack --pack-destination ${{ runner.temp }}
-        working-directory: packages/llm/openai
-      - name: Pack @llamaindex/groq
-        run: pnpm pack --pack-destination ${{ runner.temp }}
-        working-directory: packages/llm/groq
-      - name: Pack @llamaindex/core
-        run: pnpm pack --pack-destination ${{ runner.temp }}
-        working-directory: packages/core
-      - name: Pack @llamaindex/env
-        run: pnpm pack --pack-destination ${{ runner.temp }}
-        working-directory: packages/env
-      - name: Pack llamaindex
-        run: pnpm pack --pack-destination ${{ runner.temp }}
-        working-directory: packages/llamaindex
+      - name: Pack packages
+        run: |
+          for dir in packages/*; do
+            if [ -d "$dir" ] && [ -f "$dir/package.json" ] && [[ ! "$dir" =~ autotool ]]; then
+              echo "Packing $dir"
+              pnpm pack --pack-destination ${{ runner.temp }} -C $dir
+            else
+              echo "Skipping $dir, no package.json found"
+            fi
+          done
+      - name: Pack provider packages
+        run: |
+          for dir in packages/providers/*; do
+            if [ -d "$dir" ] && [ -f "$dir/package.json" ]; then
+              echo "Packing $dir"
+              pnpm pack --pack-destination ${{ runner.temp }} -C $dir
+            else
+              echo "Skipping $dir, no package.json found"
+            fi
+          done
       - name: Install
         run: npm add ${{ runner.temp }}/*.tgz
         working-directory: ${{ runner.temp }}/examples
       - name: Run Type Check
         run: npx tsc --project ./tsconfig.json
         working-directory: ${{ runner.temp }}/examples
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: build-dist
+          path: |
+            ${{ runner.temp }}/*.tgz
+          if-no-files-found: error

README.md

@@ -7,7 +7,7 @@
 
 LlamaIndex is a data framework for your LLM application.
 
-Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.
+Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in JS runtime environments with TypeScript support.
 
 Documentation: https://ts.llamaindex.ai/
 
@@ -19,17 +19,36 @@ Try examples online:
 
 LlamaIndex.TS aims to be a lightweight, easy to use set of libraries to help you integrate large language models into your applications with your own data.
 
-## Multiple JS Environment Support
+## Compatibility
+
+### Multiple JS Environment Support
 
 LlamaIndex.TS supports multiple JS environments, including:
 
 - Node.js (18, 20, 22) ✅
 - Deno ✅
 - Bun ✅
-- React Server Components (Next.js) ✅
+- Nitro ✅
+- Vercel Edge Runtime ✅ (with some limitations)
+- Cloudflare Workers ✅ (with some limitations)
 
 For now, browser support is limited due to the lack of support for [AsyncLocalStorage-like APIs](https://github.com/tc39/proposal-async-context)
 
+### Supported LLMs:
+
+- OpenAI LLms
+- Anthropic LLms
+- Groq LLMs
+- Llama2, Llama3, Llama3.1 LLMs
+- MistralAI LLMs
+- Fireworks LLMs
+- DeepSeek LLMs
+- ReplicateAI LLMs
+- TogetherAI LLMs
+- HuggingFace LLms
+- DeepInfra LLMs
+- Gemini LLMs
+
 ## Getting started
 
 ```shell
@@ -77,7 +96,7 @@ See more about [moduleResolution](https://www.typescriptlang.org/docs/handbook/m
 ### Node.js
 
 ```ts
-import fs from "fs/promises";
+import fs from "node:fs/promises";
 import { Document, VectorStoreIndex } from "llamaindex";
 
 async function main() {
@@ -111,9 +130,9 @@ main();
 node --import tsx ./main.ts
 ```
 
-### React Server Component (Next.js, Waku, Redwood.JS...)
+### Next.js
 
-First, you will need to add a llamaindex plugin to your Next.js project.
+You will need to add a llamaindex plugin to your Next.js project.
 
 ```js
 // next.config.js
@@ -124,20 +143,18 @@ module.exports = withLlamaIndex({
 });
 ```
 
-You can combine `ai` with `llamaindex` in Next.js with RSC (React Server Components).
+### React Server Actions
+
+You can combine `ai` with `llamaindex` in Next.js, Waku or Redwood.js with RSC (React Server Components).
 
 ```tsx
-// src/apps/page.tsx
 "use client";
 import { chatWithAgent } from "@/actions";
 import type { JSX } from "react";
-import { useFormState } from "react-dom";
-
-// You can use the Edge runtime in Next.js by adding this line:
-// export const runtime = "edge";
+import { useActionState } from "react";
 
 export default function Home() {
-  const [ui, action] = useFormState<JSX.Element | null>(async () => {
+  const [ui, action] = useActionState<JSX.Element | null>(async () => {
     return chatWithAgent("hello!", []);
   }, null);
   return (
@@ -167,11 +184,13 @@ export async function chatWithAgent(
       // ... adding your tools here
     ],
   });
-  const responseStream = await agent.chat({
-    stream: true,
-    message: question,
-    chatHistory: prevMessages,
-  });
+  const responseStream = await agent.chat(
+    {
+      message: question,
+      chatHistory: prevMessages,
+    },
+    true,
+  );
   const uiStream = createStreamableUI(<div>loading...</div>);
   responseStream
     .pipeTo(
@@ -189,6 +208,48 @@ export async function chatWithAgent(
 }
 ```
 
+### Cloudflare Workers
+
+> [!TIP]
+> Some modules are not supported in Cloudflare Workers which require Node.js APIs.
+
+```ts
+// add `OPENAI_API_KEY` to the `.dev.vars` file
+interface Env {
+  OPENAI_API_KEY: string;
+}
+
+export default {
+  async fetch(
+    request: Request,
+    env: Env,
+    ctx: ExecutionContext,
+  ): Promise<Response> {
+    const { OpenAIAgent, OpenAI } = await import("@llamaindex/openai");
+    const text = await request.text();
+    const agent = new OpenAIAgent({
+      llm: new OpenAI({
+        apiKey: env.OPENAI_API_KEY,
+      }),
+      tools: [],
+    });
+    const responseStream = await agent.chat({
+      stream: true,
+      message: text,
+    });
+    const textEncoder = new TextEncoder();
+    const response = responseStream.pipeThrough<Uint8Array>(
+      new TransformStream({
+        transform: (chunk, controller) => {
+          controller.enqueue(textEncoder.encode(chunk.delta));
+        },
+      }),
+    );
+    return new Response(response);
+  },
+};
+```
+
 ### Vite
 
 We have some wasm dependencies for better performance. You can use `vite-plugin-wasm` to load them.
@@ -204,29 +265,9 @@ export default {
 };
 ```
 
-## Playground
+### Tips when using in non-Node.js environments
 
-Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground
-
-## Core concepts for getting started:
-
-- [Document](/packages/llamaindex/src/Node.ts): A document represents a text file, PDF file or other contiguous piece of data.
-
-- [Node](/packages/llamaindex/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
-
-- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that question. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
-
-- [Indices](/packages/llamaindex/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
-
-- [QueryEngine](/packages/llamaindex/src/engines/query/RetrieverQueryEngine.ts): Query engines are what generate the query you put in and give you back the result. Query engines generally combine a pre-built prompt with selected Nodes from your Index to give the LLM the context it needs to answer your query. To build a query engine from your Index (recommended), use the [`asQueryEngine`](/packages/llamaindex/src/indices/BaseIndex.ts) method on your Index. See all query engines [here](/packages/llamaindex/src/engines/query).
-
-- [ChatEngine](/packages/llamaindex/src/engines/chat/SimpleChatEngine.ts): A ChatEngine helps you build a chatbot that will interact with your Indices. See all chat engines [here](/packages/llamaindex/src/engines/chat).
-
-- [SimplePrompt](/packages/llamaindex/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
-
-## Tips when using in non-Node.js environments
-
-When you are importing `llamaindex` in a non-Node.js environment(such as React Server Components, Cloudflare Workers, etc.)
+When you are importing `llamaindex` in a non-Node.js environment(such as Vercel Edge, Cloudflare Workers, etc.)
 Some classes are not exported from top-level entry file.
 
 The reason is that some classes are only compatible with Node.js runtime,(e.g. `PDFReader`) which uses Node.js specific APIs(like `fs`, `child_process`, `crypto`).
@@ -262,19 +303,31 @@ export async function getDocuments() {
 
 You'll find a complete example with LlamaIndexTS here: https://github.com/run-llama/create_llama_projects/tree/main/nextjs-edge-llamaparse
 
-## Supported LLMs:
+## Playground
 
-- OpenAI GPT-3.5-turbo and GPT-4
-- Anthropic Claude 3 (Opus, Sonnet, and Haiku) and the legacy models (Claude 2 and Instant)
-- Groq LLMs
-- Llama2/3 Chat LLMs (70B, 13B, and 7B parameters)
-- MistralAI Chat LLMs
-- Fireworks Chat LLMs
+Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground
+
+## Core concepts for getting started:
+
+- [Document](/packages/llamaindex/src/Node.ts): A document represents a text file, PDF file or other contiguous piece of data.
+
+- [Node](/packages/llamaindex/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
+
+- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that question. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
+
+- [Indices](/packages/llamaindex/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
+
+- [QueryEngine](/packages/llamaindex/src/engines/query/RetrieverQueryEngine.ts): Query engines are what generate the query you put in and give you back the result. Query engines generally combine a pre-built prompt with selected Nodes from your Index to give the LLM the context it needs to answer your query. To build a query engine from your Index (recommended), use the [`asQueryEngine`](/packages/llamaindex/src/indices/BaseIndex.ts) method on your Index. See all query engines [here](/packages/llamaindex/src/engines/query).
+
+- [ChatEngine](/packages/llamaindex/src/engines/chat/SimpleChatEngine.ts): A ChatEngine helps you build a chatbot that will interact with your Indices. See all chat engines [here](/packages/llamaindex/src/engines/chat).
+
+- [SimplePrompt](/packages/llamaindex/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
 
 ## Contributing:
 
-We are in the very early days of LlamaIndex.TS. If you’re interested in hacking on it with us check out our [contributing guide](/CONTRIBUTING.md)
+Please see our [contributing guide](CONTRIBUTING.md) for more information.
+You are highly encouraged to contribute to LlamaIndex.TS!
 
-## Bugs? Questions?
+## Community
 
 Please join our Discord! https://discord.com/invite/eN6D2HQ4aX

apps/docs/CHANGELOG.md

@@ -1,5 +1,142 @@
 # docs
 
+## 0.0.94
+
+### Patch Changes
+
+- llamaindex@0.7.2
+
+## 0.0.93
+
+### Patch Changes
+
+- Updated dependencies [ae49ff4]
+- Updated dependencies [4c38c1b]
+- Updated dependencies [a75af83]
+- Updated dependencies [a75af83]
+  - llamaindex@0.7.1
+
+## 0.0.92
+
+### Patch Changes
+
+- Updated dependencies [1364e8e]
+- Updated dependencies [3b7736f]
+- Updated dependencies [96fc69c]
+  - llamaindex@0.7.0
+  - @llamaindex/examples@0.0.9
+
+## 0.0.91
+
+### Patch Changes
+
+- Updated dependencies [5729bd9]
+  - llamaindex@0.6.22
+
+## 0.0.90
+
+### Patch Changes
+
+- Updated dependencies [6f75306]
+- Updated dependencies [94cb4ad]
+  - llamaindex@0.6.21
+
+## 0.0.89
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 0.0.88
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
+## 0.0.87
+
+### Patch Changes
+
+- Updated dependencies [5f67820]
+- Updated dependencies [fe08d04]
+  - llamaindex@0.6.18
+
+## 0.0.86
+
+### Patch Changes
+
+- Updated dependencies [ee697fb]
+  - llamaindex@0.6.17
+
+## 0.0.85
+
+### Patch Changes
+
+- Updated dependencies [63e9846]
+- Updated dependencies [6f3a31c]
+  - llamaindex@0.6.16
+
+## 0.0.84
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 0.0.83
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 0.0.82
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 0.0.81
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
+## 0.0.80
+
+### Patch Changes
+
+- Updated dependencies [df441e2]
+  - llamaindex@0.6.11
+
+## 0.0.79
+
+### Patch Changes
+
+- Updated dependencies [ebc5105]
+- Updated dependencies [6cce3b1]
+  - llamaindex@0.6.10
+
+## 0.0.78
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 0.0.77
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
 ## 0.0.76
 
 ### Patch Changes

apps/docs/docs/modules/data_loaders/llama_parse/index.mdx

@@ -13,7 +13,7 @@ Official documentation for LlamaParse can be found [here](https://docs.cloud.lla
 ## Usage
 
 You can then use the `LlamaParseReader` class to load local files and convert them into a parsed document that can be used by LlamaIndex.
-See [LlamaParseReader.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/llamaindex/src/readers/LlamaParseReader.ts) for a list of supported file types:
+See [reader.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/cloud/src/reader.ts) for a list of supported file types:
 
 <CodeBlock language="ts">{CodeSource}</CodeBlock>
 

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.76",
+  "version": "0.0.94",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

apps/docs/turbo.json

@@ -0,0 +1,8 @@
+{
+  "extends": ["//"],
+  "tasks": {
+    "build": {
+      "outputs": ["build/**", ".docusaurus/**"]
+    }
+  }
+}

examples/CHANGELOG.md

@@ -1,5 +1,16 @@
 # examples
 
+## 0.0.9
+
+### Patch Changes
+
+- Updated dependencies [1364e8e]
+- Updated dependencies [96fc69c]
+- Updated dependencies [3b7736f]
+- Updated dependencies [96fc69c]
+  - llamaindex@0.7.0
+  - @llamaindex/core@0.3.0
+
 ## 0.0.8
 
 ### Patch Changes

examples/agent/large_toolcall_with_gpt4o.ts

@@ -13,7 +13,7 @@ import { FunctionTool, OpenAI, ToolCallOptions } from "llamaindex";
   }
 })();
 
-async function callLLM(init: Partial<OpenAI>) {
+async function callLLM(init: { model: string }) {
   const csvData =
     "Country,Average Height (cm)\nNetherlands,156\nDenmark,158\nNorway,160";
 

examples/anthropic/chat_interactive.ts

@@ -18,7 +18,7 @@ import readline from "node:readline/promises";
   });
   const chatEngine = new SimpleChatEngine({
     llm,
-    chatHistory,
+    memory: chatHistory,
   });
   const rl = readline.createInterface({ input, output });
 

examples/astradb/example.ts

@@ -1,6 +1,7 @@
 import {
   AstraDBVectorStore,
   Document,
+  MetadataFilters,
   storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
@@ -42,8 +43,10 @@ async function main() {
     const index = await VectorStoreIndex.fromDocuments(docs, {
       storageContext: ctx,
     });
-
-    const queryEngine = index.asQueryEngine();
+    const preFilters: MetadataFilters = {
+      filters: [{ key: "id", operator: "in", value: [123, 789] }],
+    }; // try changing the filters to see the different results
+    const queryEngine = index.asQueryEngine({ preFilters });
     const response = await queryEngine.query({
       query: "Describe AstraDB.",
     });

examples/chromadb/preFilters.ts

@@ -1,57 +1,83 @@
 import {
   ChromaVectorStore,
   Document,
+  MetadataFilters,
   VectorStoreIndex,
   storageContextFromDefaults,
 } from "llamaindex";
 
-const collectionName = "dog_colors";
+const collectionName = "dogs_with_color";
 
 async function main() {
   try {
-    const docs = [
-      new Document({
-        text: "The dog is brown",
-        metadata: {
-          dogId: "1",
-        },
-      }),
-      new Document({
-        text: "The dog is red",
-        metadata: {
-          dogId: "2",
-        },
-      }),
-    ];
-
-    console.log("Creating ChromaDB vector store");
     const chromaVS = new ChromaVectorStore({ collectionName });
-    const ctx = await storageContextFromDefaults({ vectorStore: chromaVS });
+    const index = await VectorStoreIndex.fromVectorStore(chromaVS);
 
-    console.log("Embedding documents and adding to index");
-    const index = await VectorStoreIndex.fromDocuments(docs, {
-      storageContext: ctx,
-    });
+    const queryFn = async (filters?: MetadataFilters) => {
+      console.log("\nQuerying dogs by filters: ", JSON.stringify(filters));
+      const query = "List all colors of dogs";
+      const queryEngine = index.asQueryEngine({
+        preFilters: filters,
+        similarityTopK: 3,
+      });
+      const response = await queryEngine.query({ query });
+      console.log(response.toString());
+    };
 
-    console.log("Querying index");
-    const queryEngine = index.asQueryEngine({
-      preFilters: {
-        filters: [
-          {
-            key: "dogId",
-            value: "2",
-            operator: "==",
-          },
-        ],
-      },
-    });
-    const response = await queryEngine.query({
-      query: "What is the color of the dog?",
-    });
-    console.log(response.toString());
+    await queryFn(); // red, brown, yellow
+    await queryFn({ filters: [{ key: "dogId", value: "1", operator: "==" }] }); // brown
+    await queryFn({ filters: [{ key: "dogId", value: "1", operator: "!=" }] }); // red, yellow
+    await queryFn({
+      filters: [
+        { key: "dogId", value: "1", operator: "==" },
+        { key: "dogId", value: "3", operator: "==" },
+      ],
+      condition: "or",
+    }); // brown, yellow
+    await queryFn({
+      filters: [{ key: "dogId", value: ["1", "2"], operator: "in" }],
+    }); // red, brown
   } catch (e) {
     console.error(e);
   }
 }
 
-void main();
+async function generate() {
+  const docs = [
+    new Document({
+      id_: "doc1",
+      text: "The dog is brown",
+      metadata: {
+        dogId: "1",
+      },
+    }),
+    new Document({
+      id_: "doc2",
+      text: "The dog is red",
+      metadata: {
+        dogId: "2",
+      },
+    }),
+    new Document({
+      id_: "doc3",
+      text: "The dog is yellow",
+      metadata: {
+        dogId: "3",
+      },
+    }),
+  ];
+
+  console.log("Creating ChromaDB vector store");
+  const chromaVS = new ChromaVectorStore({ collectionName });
+  const ctx = await storageContextFromDefaults({ vectorStore: chromaVS });
+
+  console.log("Embedding documents and adding to index");
+  await VectorStoreIndex.fromDocuments(docs, {
+    storageContext: ctx,
+  });
+}
+
+(async () => {
+  await generate();
+  await main();
+})();

examples/huggingface/embedding.ts

@@ -27,12 +27,10 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const stream = await queryEngine.query(
-    {
-      query: "What did the author do in college?",
-    },
-    true,
-  );
+  const stream = await queryEngine.query({
+    query: "What did the author do in college?",
+    stream: true,
+  });
 
   // Output response
   for await (const chunk of stream) {

examples/huggingface/embeddingApi.ts

@@ -37,12 +37,10 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const stream = await queryEngine.query(
-    {
-      query: "What did the author do in college?",
-    },
-    true,
-  );
+  const stream = await queryEngine.query({
+    query: "What did the author do in college?",
+    stream: true,
+  });
 
   // Output response
   for await (const chunk of stream) {

examples/metadata-filter/preFilters.ts

@@ -0,0 +1,51 @@
+import {
+  Document,
+  MetadataFilters,
+  Settings,
+  SimpleDocumentStore,
+  VectorStoreIndex,
+  storageContextFromDefaults,
+} from "llamaindex";
+
+async function getDataSource() {
+  const docs = [
+    new Document({ text: "The dog is brown", metadata: { dogId: "1" } }),
+    new Document({ text: "The dog is yellow", metadata: { dogId: "2" } }),
+  ];
+  const storageContext = await storageContextFromDefaults({
+    persistDir: "./cache",
+  });
+  const numberOfDocs = Object.keys(
+    (storageContext.docStore as SimpleDocumentStore).toDict(),
+  ).length;
+  if (numberOfDocs === 0) {
+    return await VectorStoreIndex.fromDocuments(docs, { storageContext });
+  }
+  return await VectorStoreIndex.init({
+    storageContext,
+  });
+}
+
+Settings.callbackManager.on("retrieve-end", (event) => {
+  const { nodes, query } = event.detail;
+  console.log(`${query.query} - Number of retrieved nodes:`, nodes.length);
+});
+
+async function main() {
+  const index = await getDataSource();
+  const filters: MetadataFilters = {
+    filters: [{ key: "dogId", value: "2", operator: "==" }],
+  };
+
+  const retriever = index.asRetriever({ similarityTopK: 3, filters });
+  const queryEngine = index.asQueryEngine({
+    similarityTopK: 3,
+    preFilters: filters,
+  });
+
+  console.log("Retriever and query engine should only retrieve 1 node:");
+  await retriever.retrieve({ query: "Retriever: get dog" });
+  await queryEngine.query({ query: "QueryEngine: get dog" });
+}
+
+void main();

examples/multimodal/context.ts

@@ -1,4 +1,5 @@
 // call pnpm tsx multimodal/load.ts first to init the storage
+import { extractText } from "@llamaindex/core/utils";
 import {
   ContextChatEngine,
   NodeWithScore,
@@ -25,8 +26,9 @@ Settings.callbackManager.on("retrieve-end", (event) => {
   const textNodes = nodes.filter(
     (node: NodeWithScore) => node.node.type === ObjectType.TEXT,
   );
+  const text = extractText(query);
   console.log(
-    `Retrieved ${textNodes.length} text nodes and ${imageNodes.length} image nodes for query: ${query}`,
+    `Retrieved ${textNodes.length} text nodes and ${imageNodes.length} image nodes for query: ${text}`,
   );
 });
 

examples/multimodal/rag.ts

@@ -1,3 +1,4 @@
+import { extractText } from "@llamaindex/core/utils";
 import {
   getResponseSynthesizer,
   OpenAI,
@@ -16,7 +17,8 @@ Settings.llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 512 });
 // Update callbackManager
 Settings.callbackManager.on("retrieve-end", (event) => {
   const { nodes, query } = event.detail;
-  console.log(`Retrieved ${nodes.length} nodes for query: ${query}`);
+  const text = extractText(query);
+  console.log(`Retrieved ${nodes.length} nodes for query: ${text}`);
 });
 
 async function main() {
@@ -30,12 +32,10 @@ async function main() {
     responseSynthesizer: getResponseSynthesizer("multi_modal"),
     retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1 } }),
   });
-  const stream = await queryEngine.query(
-    {
-      query: "Tell me more about Vincent van Gogh's famous paintings",
-    },
-    true,
-  );
+  const stream = await queryEngine.query({
+    query: "Tell me more about Vincent van Gogh's famous paintings",
+    stream: true,
+  });
   for await (const chunk of stream) {
     process.stdout.write(chunk.response);
   }

examples/package.json

@@ -1,22 +1,24 @@
 {
   "name": "@llamaindex/examples",
   "private": true,
-  "version": "0.0.8",
+  "version": "0.0.9",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@azure/identity": "^4.4.1",
     "@datastax/astra-db-ts": "^1.4.1",
-    "@llamaindex/core": "^0.2.0",
+    "@llamaindex/core": "^0.3.0",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^3.0.2",
+    "@vercel/postgres": "^0.10.0",
     "@zilliz/milvus2-sdk-node": "^2.4.6",
     "chromadb": "^1.8.1",
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.6.0",
+    "llamaindex": "^0.7.0",
     "mongodb": "^6.7.0",
-    "pathe": "^1.1.2"
+    "pathe": "^1.1.2",
+    "postgres": "^3.4.4"
   },
   "devDependencies": {
     "@types/node": "^22.5.1",

examples/qdrantdb/preFilters.ts

@@ -39,6 +39,12 @@ async function main() {
           dogId: "2",
         },
       }),
+      new Document({
+        text: "The dog is black",
+        metadata: {
+          dogId: "3",
+        },
+      }),
     ];
     console.log("Creating QdrantDB vector store");
     const qdrantVs = new QdrantVectorStore({ url: qdrantUrl, collectionName });
@@ -73,6 +79,42 @@ async function main() {
       query: "What is the color of the dog?",
     });
     console.log("Filter with dogId 2 response:", response.toString());
+
+    console.log("Querying index with dogId !=2: Expected output: Not red");
+    const queryEngineNotDogId2 = index.asQueryEngine({
+      preFilters: {
+        filters: [
+          {
+            key: "dogId",
+            value: "2",
+            operator: "!=",
+          },
+        ],
+      },
+    });
+    const responseNotDogId2 = await queryEngineNotDogId2.query({
+      query: "What is the color of the dog?",
+    });
+    console.log(responseNotDogId2.toString());
+
+    console.log(
+      "Querying index with dogId 2 or 3: Expected output: Red, Black",
+    );
+    const queryEngineIn = index.asQueryEngine({
+      preFilters: {
+        filters: [
+          {
+            key: "dogId",
+            value: ["2", "3"],
+            operator: "in",
+          },
+        ],
+      },
+    });
+    const responseIn = await queryEngineIn.query({
+      query: "List all dogs",
+    });
+    console.log(responseIn.toString());
   } catch (e) {
     console.error(e);
   }

examples/readers/src/llamaparse-json.ts

@@ -1,4 +1,4 @@
-import { createMessageContent } from "@llamaindex/core/utils";
+import { createMessageContent } from "@llamaindex/core/response-synthesizers";
 import {
   Document,
   ImageNode,

examples/vector-store/pg/.env.template

@@ -0,0 +1,9 @@
+# neon template
+PGHOST=
+PGDATABASE=
+PGUSER=
+PGPASSWORD=
+ENDPOINT_ID=
+
+# vercel template
+POSTGRES_URL=

examples/vector-store/pg/load-docs.ts

@@ -1,11 +1,11 @@
 // load-docs.ts
-import fs from "fs/promises";
 import {
   PGVectorStore,
   SimpleDirectoryReader,
   storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
+import fs from "node:fs/promises";
 
 async function getSourceFilenames(sourceDir: string) {
   return await fs

examples/vector-store/pg/neon.ts

@@ -0,0 +1,45 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import dotenv from "dotenv";
+import { Document, PGVectorStore, VectorStoreQueryMode } from "llamaindex";
+import postgres from "postgres";
+
+dotenv.config();
+
+const { PGHOST, PGDATABASE, PGUSER, ENDPOINT_ID } = process.env;
+const PGPASSWORD = decodeURIComponent(process.env.PGPASSWORD!);
+
+const sql = postgres({
+  host: PGHOST,
+  database: PGDATABASE,
+  username: PGUSER,
+  password: PGPASSWORD,
+  port: 5432,
+  ssl: "require",
+  connection: {
+    options: `project=${ENDPOINT_ID}`,
+  },
+});
+
+await sql`CREATE EXTENSION IF NOT EXISTS vector`;
+
+const vectorStore = new PGVectorStore({
+  dimensions: 3,
+  client: sql,
+});
+
+await vectorStore.add([
+  new Document({
+    text: "hello, world",
+    embedding: [1, 2, 3],
+  }),
+]);
+
+const results = await vectorStore.query({
+  mode: VectorStoreQueryMode.DEFAULT,
+  similarityTopK: 1,
+  queryEmbedding: [1, 2, 3],
+});
+
+console.log("result", results);
+
+await sql.end();

examples/vector-store/pg/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "pg-vector-store",
+  "type": "module",
+  "private": true
+}

examples/vector-store/pg/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "types": ["node"],
+    "skipLibCheck": true
+  },
+  "include": ["./**/*.ts"]
+}

examples/vector-store/pg/vercel.ts

@@ -0,0 +1,30 @@
+// https://vercel.com/docs/storage/vercel-postgres/sdk
+import { sql } from "@vercel/postgres";
+import dotenv from "dotenv";
+import { Document, PGVectorStore, VectorStoreQueryMode } from "llamaindex";
+
+dotenv.config();
+
+await sql`CREATE EXTENSION IF NOT EXISTS vector`;
+
+const vectorStore = new PGVectorStore({
+  dimensions: 3,
+  client: sql,
+});
+
+await vectorStore.add([
+  new Document({
+    text: "hello, world",
+    embedding: [1, 2, 3],
+  }),
+]);
+
+const results = await vectorStore.query({
+  mode: VectorStoreQueryMode.DEFAULT,
+  similarityTopK: 1,
+  queryEmbedding: [1, 2, 3],
+});
+
+console.log("result", results);
+
+await sql.end();

examples/vectorIndexCustomize.ts

@@ -25,12 +25,9 @@ async function main() {
     similarityCutoff: 0.7,
   });
   // TODO: cannot pass responseSynthesizer into retriever query engine
-  const queryEngine = new RetrieverQueryEngine(
-    retriever,
-    undefined,
-    undefined,
-    [nodePostprocessor],
-  );
+  const queryEngine = new RetrieverQueryEngine(retriever, undefined, [
+    nodePostprocessor,
+  ]);
 
   const response = await queryEngine.query({
     query: "What did the author do growing up?",

examples/vectorIndexFromVectorStore.ts

@@ -1,4 +1,5 @@
 import {
+  BaseVectorStore,
   getResponseSynthesizer,
   OpenAI,
   OpenAIEmbedding,
@@ -6,7 +7,6 @@ import {
   Settings,
   TextNode,
   VectorIndexRetriever,
-  VectorStore,
   VectorStoreIndex,
   VectorStoreQuery,
   VectorStoreQueryResult,
@@ -24,7 +24,7 @@ Settings.llm = new OpenAI({
  * Please do not use this class in production; it's only for demonstration purposes.
  */
 class PineconeVectorStore<T extends RecordMetadata = RecordMetadata>
-  implements VectorStore
+  implements BaseVectorStore
 {
   storesText = true;
   isEmbeddingQuery = false;
@@ -165,9 +165,7 @@ async function main() {
     });
 
     const responseSynthesizer = getResponseSynthesizer("tree_summarize");
-    return new RetrieverQueryEngine(retriever, responseSynthesizer, {
-      filter,
-    });
+    return new RetrieverQueryEngine(retriever, responseSynthesizer);
   };
 
   // whatever is a key from your metadata

package.json

@@ -12,6 +12,7 @@
     "e2e": "turbo run e2e",
     "test": "turbo run test",
     "type-check": "tsc -b --diagnostics",
+    "circular-check": "madge --circular ./packages/**/**/dist/index.js",
     "release": "pnpm run build:release && changeset publish",
     "release-snapshot": "pnpm run build:release && changeset publish --tag snapshot",
     "new-version": "changeset version && pnpm format:write && pnpm run build:release",
@@ -30,7 +31,7 @@
     "madge": "^8.0.0",
     "prettier": "^3.3.3",
     "prettier-plugin-organize-imports": "^4.0.0",
-    "turbo": "^2.1.0",
+    "turbo": "^2.1.2",
     "typescript": "^5.6.2"
   },
   "packageManager": "pnpm@9.5.0",

packages/autotool/CHANGELOG.md

@@ -1,5 +1,143 @@
 # @llamaindex/autotool
 
+## 4.0.2
+
+### Patch Changes
+
+- llamaindex@0.7.2
+
+## 4.0.1
+
+### Patch Changes
+
+- a75af83: refactor: move some llm and embedding to single package
+- Updated dependencies [ae49ff4]
+- Updated dependencies [4c38c1b]
+- Updated dependencies [a75af83]
+- Updated dependencies [a75af83]
+  - llamaindex@0.7.1
+
+## 4.0.0
+
+### Patch Changes
+
+- Updated dependencies [1364e8e]
+- Updated dependencies [3b7736f]
+- Updated dependencies [96fc69c]
+  - llamaindex@0.7.0
+
+## 3.0.22
+
+### Patch Changes
+
+- Updated dependencies [5729bd9]
+  - llamaindex@0.6.22
+
+## 3.0.21
+
+### Patch Changes
+
+- Updated dependencies [6f75306]
+- Updated dependencies [94cb4ad]
+  - llamaindex@0.6.21
+
+## 3.0.20
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 3.0.19
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
+## 3.0.18
+
+### Patch Changes
+
+- Updated dependencies [5f67820]
+- Updated dependencies [fe08d04]
+  - llamaindex@0.6.18
+
+## 3.0.17
+
+### Patch Changes
+
+- Updated dependencies [ee697fb]
+  - llamaindex@0.6.17
+
+## 3.0.16
+
+### Patch Changes
+
+- Updated dependencies [63e9846]
+- Updated dependencies [6f3a31c]
+  - llamaindex@0.6.16
+
+## 3.0.15
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+
+## 3.0.14
+
+### Patch Changes
+
+- llamaindex@0.6.14
+
+## 3.0.13
+
+### Patch Changes
+
+- llamaindex@0.6.13
+
+## 3.0.12
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+
+## 3.0.11
+
+### Patch Changes
+
+- df441e2: fix: consoleLogger is missing from `@llamaindex/env`
+- Updated dependencies [df441e2]
+  - llamaindex@0.6.11
+
+## 3.0.10
+
+### Patch Changes
+
+- Updated dependencies [ebc5105]
+- Updated dependencies [6cce3b1]
+  - llamaindex@0.6.10
+
+## 3.0.9
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 3.0.8
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
 ## 3.0.7
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,159 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.34
+
+### Patch Changes
+
+- llamaindex@0.7.2
+- @llamaindex/autotool@4.0.2
+
+## 0.0.33
+
+### Patch Changes
+
+- Updated dependencies [ae49ff4]
+- Updated dependencies [4c38c1b]
+- Updated dependencies [a75af83]
+- Updated dependencies [a75af83]
+  - llamaindex@0.7.1
+  - @llamaindex/autotool@4.0.1
+
+## 0.0.32
+
+### Patch Changes
+
+- Updated dependencies [1364e8e]
+- Updated dependencies [3b7736f]
+- Updated dependencies [96fc69c]
+  - llamaindex@0.7.0
+  - @llamaindex/autotool@4.0.0
+
+## 0.0.31
+
+### Patch Changes
+
+- Updated dependencies [5729bd9]
+  - llamaindex@0.6.22
+  - @llamaindex/autotool@3.0.22
+
+## 0.0.30
+
+### Patch Changes
+
+- Updated dependencies [6f75306]
+- Updated dependencies [94cb4ad]
+  - llamaindex@0.6.21
+  - @llamaindex/autotool@3.0.21
+
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+  - @llamaindex/autotool@3.0.20
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+  - @llamaindex/autotool@3.0.19
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [5f67820]
+- Updated dependencies [fe08d04]
+  - llamaindex@0.6.18
+  - @llamaindex/autotool@3.0.18
+
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [ee697fb]
+  - llamaindex@0.6.17
+  - @llamaindex/autotool@3.0.17
+
+## 0.0.25
+
+### Patch Changes
+
+- Updated dependencies [63e9846]
+- Updated dependencies [6f3a31c]
+  - llamaindex@0.6.16
+  - @llamaindex/autotool@3.0.16
+
+## 0.0.24
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+  - @llamaindex/autotool@3.0.15
+
+## 0.0.23
+
+### Patch Changes
+
+- llamaindex@0.6.14
+- @llamaindex/autotool@3.0.14
+
+## 0.0.22
+
+### Patch Changes
+
+- llamaindex@0.6.13
+- @llamaindex/autotool@3.0.13
+
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+  - @llamaindex/autotool@3.0.12
+
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [df441e2]
+  - @llamaindex/autotool@3.0.11
+  - llamaindex@0.6.11
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [ebc5105]
+- Updated dependencies [6cce3b1]
+  - llamaindex@0.6.10
+  - @llamaindex/autotool@3.0.10
+
+## 0.0.18
+
+### Patch Changes
+
+- llamaindex@0.6.9
+- @llamaindex/autotool@3.0.9
+
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+  - @llamaindex/autotool@3.0.8
+
 ## 0.0.16
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.16"
+  "version": "0.0.34"
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,159 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.78
+
+### Patch Changes
+
+- llamaindex@0.7.2
+- @llamaindex/autotool@4.0.2
+
+## 0.1.77
+
+### Patch Changes
+
+- Updated dependencies [ae49ff4]
+- Updated dependencies [4c38c1b]
+- Updated dependencies [a75af83]
+- Updated dependencies [a75af83]
+  - llamaindex@0.7.1
+  - @llamaindex/autotool@4.0.1
+
+## 0.1.76
+
+### Patch Changes
+
+- Updated dependencies [1364e8e]
+- Updated dependencies [3b7736f]
+- Updated dependencies [96fc69c]
+  - llamaindex@0.7.0
+  - @llamaindex/autotool@4.0.0
+
+## 0.1.75
+
+### Patch Changes
+
+- Updated dependencies [5729bd9]
+  - llamaindex@0.6.22
+  - @llamaindex/autotool@3.0.22
+
+## 0.1.74
+
+### Patch Changes
+
+- Updated dependencies [6f75306]
+- Updated dependencies [94cb4ad]
+  - llamaindex@0.6.21
+  - @llamaindex/autotool@3.0.21
+
+## 0.1.73
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+  - @llamaindex/autotool@3.0.20
+
+## 0.1.72
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+  - @llamaindex/autotool@3.0.19
+
+## 0.1.71
+
+### Patch Changes
+
+- Updated dependencies [5f67820]
+- Updated dependencies [fe08d04]
+  - llamaindex@0.6.18
+  - @llamaindex/autotool@3.0.18
+
+## 0.1.70
+
+### Patch Changes
+
+- Updated dependencies [ee697fb]
+  - llamaindex@0.6.17
+  - @llamaindex/autotool@3.0.17
+
+## 0.1.69
+
+### Patch Changes
+
+- Updated dependencies [63e9846]
+- Updated dependencies [6f3a31c]
+  - llamaindex@0.6.16
+  - @llamaindex/autotool@3.0.16
+
+## 0.1.68
+
+### Patch Changes
+
+- Updated dependencies [2a82413]
+  - llamaindex@0.6.15
+  - @llamaindex/autotool@3.0.15
+
+## 0.1.67
+
+### Patch Changes
+
+- llamaindex@0.6.14
+- @llamaindex/autotool@3.0.14
+
+## 0.1.66
+
+### Patch Changes
+
+- llamaindex@0.6.13
+- @llamaindex/autotool@3.0.13
+
+## 0.1.65
+
+### Patch Changes
+
+- Updated dependencies [f7b4e94]
+- Updated dependencies [78037a6]
+- Updated dependencies [1d9e3b1]
+  - llamaindex@0.6.12
+  - @llamaindex/autotool@3.0.12
+
+## 0.1.64
+
+### Patch Changes
+
+- Updated dependencies [df441e2]
+  - @llamaindex/autotool@3.0.11
+  - llamaindex@0.6.11
+
+## 0.1.63
+
+### Patch Changes
+
+- Updated dependencies [ebc5105]
+- Updated dependencies [6cce3b1]
+  - llamaindex@0.6.10
+  - @llamaindex/autotool@3.0.10
+
+## 0.1.62
+
+### Patch Changes
+
+- llamaindex@0.6.9
+- @llamaindex/autotool@3.0.9
+
+## 0.1.61
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+  - @llamaindex/autotool@3.0.8
+
 ## 0.1.60
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.60",
+  "version": "0.1.78",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "3.0.7",
+  "version": "4.0.2",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",
@@ -70,7 +70,7 @@
     "@swc/types": "^0.1.12",
     "@types/json-schema": "^7.0.15",
     "@types/node": "^22.5.1",
-    "bunchee": "5.3.2",
+    "bunchee": "5.5.1",
     "llamaindex": "workspace:*",
     "next": "14.2.11",
     "rollup": "^4.21.2",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,82 @@
 # @llamaindex/cloud
 
+## 1.0.2
+
+### Patch Changes
+
+- Updated dependencies [4ba2cfe]
+  - @llamaindex/env@0.1.15
+  - @llamaindex/core@0.3.2
+
+## 1.0.1
+
+### Patch Changes
+
+- 4c38c1b: fix(cloud): do not detect file type in llama parse
+- 24d065f: Log Parse Job Errors when verbose is enabled
+- a75af83: refactor: move some llm and embedding to single package
+- Updated dependencies [ae49ff4]
+- Updated dependencies [a75af83]
+  - @llamaindex/env@0.1.14
+  - @llamaindex/core@0.3.1
+
+## 1.0.0
+
+### Patch Changes
+
+- Updated dependencies [1364e8e]
+- Updated dependencies [96fc69c]
+  - @llamaindex/core@0.3.0
+
+## 0.2.14
+
+### Patch Changes
+
+- Updated dependencies [5f67820]
+  - @llamaindex/core@0.2.12
+
+## 0.2.13
+
+### Patch Changes
+
+- Updated dependencies [ee697fb]
+  - @llamaindex/core@0.2.11
+
+## 0.2.12
+
+### Patch Changes
+
+- Updated dependencies [3489e7d]
+- Updated dependencies [468bda5]
+  - @llamaindex/core@0.2.10
+
+## 0.2.11
+
+### Patch Changes
+
+- 0b20ff9: fix: package.json format
+
+## 0.2.10
+
+### Patch Changes
+
+- 981811e: fix(cloud): llama parse reader save image incorrectly
+
+## 0.2.9
+
+### Patch Changes
+
+- df441e2: fix: consoleLogger is missing from `@llamaindex/env`
+- Updated dependencies [df441e2]
+  - @llamaindex/core@0.2.8
+  - @llamaindex/env@0.1.13
+
+## 0.2.8
+
+### Patch Changes
+
+- ac41ed3: feat: bump cloud sdk version
+
 ## 0.2.7
 
 ### Patch Changes

packages/cloud/api/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.2.7",
+  "version": "1.0.2",
   "type": "module",
   "license": "MIT",
   "scripts": {
@@ -9,56 +9,54 @@
   },
   "files": [
     "openapi.json",
-    "dist"
+    "./api",
+    "./reader"
   ],
   "exports": {
     "./openapi.json": "./openapi.json",
     "./api": {
       "require": {
-        "types": "./dist/api.d.cts",
-        "default": "./dist/api.cjs"
+        "types": "./api/dist/index.d.cts",
+        "default": "./api/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/api.d.ts",
-        "default": "./dist/api.js"
+        "types": "./api/dist/index.d.ts",
+        "default": "./api/dist/index.js"
       },
       "default": {
-        "types": "./dist/api.d.ts",
-        "default": "./dist/api.js"
+        "types": "./api/dist/index.d.ts",
+        "default": "./api/dist/index.js"
       }
     },
     "./reader": {
       "require": {
-        "types": "./dist/reader.d.cts",
-        "default": "./dist/reader.cjs"
+        "types": "./reader/dist/index.d.cts",
+        "default": "./reader/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/reader.d.ts",
-        "default": "./dist/reader.js"
+        "types": "./reader/dist/index.d.ts",
+        "default": "./reader/dist/index.js"
       },
       "default": {
-        "types": "./dist/reader.d.ts",
-        "default": "./dist/reader.js"
+        "types": "./reader/dist/index.d.ts",
+        "default": "./reader/dist/index.js"
       }
     }
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/himself65/LlamaIndexTS.git",
+    "url": "https://github.com/run-llama/LlamaIndexTS.git",
     "directory": "packages/cloud"
   },
   "devDependencies": {
     "@hey-api/client-fetch": "^0.2.4",
     "@hey-api/openapi-ts": "^0.53.0",
-    "@llamaindex/core": "workspace:^0.2.3",
-    "@llamaindex/env": "workspace:^0.1.11",
-    "bunchee": "5.3.2"
+    "@llamaindex/core": "workspace:*",
+    "@llamaindex/env": "workspace:*",
+    "bunchee": "5.5.1"
   },
   "peerDependencies": {
-    "@llamaindex/core": "workspace:^0.2.3",
-    "@llamaindex/env": "workspace:^0.1.11"
-  },
-  "dependencies": {
-    "magic-bytes.js": "^1.10.0"
+    "@llamaindex/core": "workspace:*",
+    "@llamaindex/env": "workspace:*"
   }
 }

packages/cloud/reader/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/cloud/src/reader.ts

@@ -1,11 +1,10 @@
-import { createClient, createConfig, type Client } from "@hey-api/client-fetch";
+import { type Client, createClient, createConfig } from "@hey-api/client-fetch";
 import { Document, FileReader } from "@llamaindex/core/schema";
-import { fs, getEnv } from "@llamaindex/env";
-import { filetypeinfo } from "magic-bytes.js";
+import { fs, getEnv, path } from "@llamaindex/env";
 import {
-  ParsingService,
   type Body_upload_file_api_v1_parsing_upload_post,
   type ParserLanguages,
+  ParsingService,
 } from "./api";
 import { sleep } from "./utils";
 
@@ -13,99 +12,6 @@ export type Language = ParserLanguages;
 
 export type ResultType = "text" | "markdown" | "json";
 
-const SUPPORT_FILE_EXT: string[] = [
-  ".pdf",
-  // document and presentations
-  ".602",
-  ".abw",
-  ".cgm",
-  ".cwk",
-  ".doc",
-  ".docx",
-  ".docm",
-  ".dot",
-  ".dotm",
-  ".hwp",
-  ".key",
-  ".lwp",
-  ".mw",
-  ".mcw",
-  ".pages",
-  ".pbd",
-  ".ppt",
-  ".pptm",
-  ".pptx",
-  ".pot",
-  ".potm",
-  ".potx",
-  ".rtf",
-  ".sda",
-  ".sdd",
-  ".sdp",
-  ".sdw",
-  ".sgl",
-  ".sti",
-  ".sxi",
-  ".sxw",
-  ".stw",
-  ".sxg",
-  ".txt",
-  ".uof",
-  ".uop",
-  ".uot",
-  ".vor",
-  ".wpd",
-  ".wps",
-  ".xml",
-  ".zabw",
-  ".epub",
-  // images
-  ".jpg",
-  ".jpeg",
-  ".png",
-  ".gif",
-  ".bmp",
-  ".svg",
-  ".tiff",
-  ".webp",
-  // web
-  ".htm",
-  ".html",
-  // spreadsheets
-  ".xlsx",
-  ".xls",
-  ".xlsm",
-  ".xlsb",
-  ".xlw",
-  ".csv",
-  ".dif",
-  ".sylk",
-  ".slk",
-  ".prn",
-  ".numbers",
-  ".et",
-  ".ods",
-  ".fods",
-  ".uos1",
-  ".uos2",
-  ".dbf",
-  ".wk1",
-  ".wk2",
-  ".wk3",
-  ".wk4",
-  ".wks",
-  ".123",
-  ".wq1",
-  ".wq2",
-  ".wb1",
-  ".wb2",
-  ".wb3",
-  ".qpw",
-  ".xlr",
-  ".eth",
-  ".tsv",
-];
-
 //todo: should move into @llamaindex/env
 type WriteStream = {
   write: (text: string) => void;
@@ -170,6 +76,15 @@ export class LlamaParseReader extends FileReader {
   vendorMultimodalModelName?: string | undefined;
   // The API key for the multimodal API. Can also be set as an env variable: LLAMA_CLOUD_VENDOR_MULTIMODAL_API_KEY
   vendorMultimodalApiKey?: string | undefined;
+
+  webhookUrl?: string | undefined;
+  premiumMode?: boolean | undefined;
+  takeScreenshot?: boolean | undefined;
+  disableOcr?: boolean | undefined;
+  disableReconstruction?: boolean | undefined;
+  inputS3Path?: string | undefined;
+  outputS3PathPrefix?: string | undefined;
+
   // numWorkers is implemented in SimpleDirectoryReader
   stdout?: WriteStream | undefined;
 
@@ -230,17 +145,12 @@ export class LlamaParseReader extends FileReader {
 
   // Create a job for the LlamaParse API
   private async createJob(data: Uint8Array): Promise<string> {
-    // Load data, set the mime type
-    const { mime } = await LlamaParseReader.getMimeType(data);
-
     if (this.verbose) {
       console.log("Started uploading the file");
     }
 
     const body = {
-      file: new Blob([data], {
-        type: mime,
-      }),
+      file: new Blob([data]),
       language: this.language,
       parsing_instruction: this.parsingInstruction,
       skip_diagonal_text: this.skipDiagonalText,
@@ -258,13 +168,13 @@ export class LlamaParseReader extends FileReader {
       use_vendor_multimodal_model: this.useVendorMultimodalModel,
       vendor_multimodal_model_name: this.vendorMultimodalModelName,
       vendor_multimodal_api_key: this.vendorMultimodalApiKey,
-      // fixme: does these fields need to be set?
-      webhook_url: undefined,
-      take_screenshot: undefined,
-      disable_ocr: undefined,
-      disable_reconstruction: undefined,
-      input_s3_path: undefined,
-      output_s3_path_prefix: undefined,
+      premium_mode: this.premiumMode,
+      webhook_url: this.webhookUrl,
+      take_screenshot: this.takeScreenshot,
+      disable_ocr: this.disableOcr,
+      disable_reconstruction: this.disableReconstruction,
+      input_s3_path: this.inputS3Path,
+      output_s3_path_prefix: this.outputS3PathPrefix,
     } satisfies {
       [Key in keyof Body_upload_file_api_v1_parsing_upload_post]-?:
         | Body_upload_file_api_v1_parsing_upload_post[Key]
@@ -359,6 +269,11 @@ export class LlamaParseReader extends FileReader {
         }
         tries++;
       } else {
+        if (this.verbose) {
+          console.error(
+            `Recieved Error response ${status} for job ${jobId}.  Got Error Code: ${data.error_code} and Error Message: ${data.error_message}`,
+          );
+        }
         throw new Error(
           `Failed to parse the file: ${jobId}, status: ${status}`,
         );
@@ -501,14 +416,7 @@ export class LlamaParseReader extends FileReader {
     jobId: string,
     imageName: string,
   ): Promise<string> {
-    // Get the full path
-    let imagePath = `${downloadPath}/${jobId}-${imageName}`;
-    // Get a valid image path
-    if (!imagePath.endsWith(".png") && !imagePath.endsWith(".jpg")) {
-      imagePath += ".png";
-    }
-
-    return imagePath;
+    return path.join(downloadPath, `${jobId}-${imageName}`);
   }
 
   private async fetchAndSaveImage(
@@ -529,10 +437,9 @@ export class LlamaParseReader extends FileReader {
     if (response.error) {
       throw new Error(`Failed to download image: ${response.error.detail}`);
     }
-    const arrayBuffer = (await response.data) as ArrayBuffer;
-    const buffer = new Uint8Array(arrayBuffer);
+    const blob = (await response.data) as Blob;
     // Write the image buffer to the specified imagePath
-    await fs.writeFile(imagePath, buffer);
+    await fs.writeFile(imagePath, new Uint8Array(await blob.arrayBuffer()));
   }
 
   // Filters out invalid values (null, undefined, empty string) of specific params.
@@ -563,24 +470,4 @@ export class LlamaParseReader extends FileReader {
         }),
     );
   }
-
-  static async getMimeType(
-    data: Uint8Array,
-  ): Promise<{ mime: string; extension: string }> {
-    const typeinfos = filetypeinfo(data);
-    // find the first type info that matches the supported MIME types
-    // It could be happened that docx file is recognized as zip file, so we need to check the mime type
-    const info = typeinfos.find((info) => {
-      if (info.extension && SUPPORT_FILE_EXT.includes(`.${info.extension}`)) {
-        return info;
-      }
-    });
-    if (!info || !info.mime || !info.extension) {
-      const ext = SUPPORT_FILE_EXT.join(", ");
-      throw new Error(
-        `File has type which does not match supported MIME Types. Supported formats include: ${ext}`,
-      );
-    }
-    return { mime: info.mime, extension: info.extension };
-  }
 }

packages/cloud/turbo.json

@@ -2,7 +2,7 @@
   "extends": ["//"],
   "tasks": {
     "build": {
-      "outputs": ["dist/**", "src/client/**"]
+      "outputs": ["**/dist/**", "src/client/**"]
     }
   }
 }

packages/community/CHANGELOG.md

@@ -1,5 +1,90 @@
 # @llamaindex/community
 
+## 0.0.50
+
+### Patch Changes
+
+- Updated dependencies [4ba2cfe]
+  - @llamaindex/env@0.1.15
+  - @llamaindex/core@0.3.2
+
+## 0.0.49
+
+### Patch Changes
+
+- a75af83: refactor: move some llm and embedding to single package
+- Updated dependencies [ae49ff4]
+- Updated dependencies [a75af83]
+  - @llamaindex/env@0.1.14
+  - @llamaindex/core@0.3.1
+
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [1364e8e]
+- Updated dependencies [96fc69c]
+  - @llamaindex/core@0.3.0
+
+## 0.0.47
+
+### Patch Changes
+
+- Updated dependencies [5f67820]
+  - @llamaindex/core@0.2.12
+
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [ee697fb]
+  - @llamaindex/core@0.2.11
+
+## 0.0.45
+
+### Patch Changes
+
+- Updated dependencies [3489e7d]
+- Updated dependencies [468bda5]
+  - @llamaindex/core@0.2.10
+
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [b17d439]
+  - @llamaindex/core@0.2.9
+
+## 0.0.43
+
+### Patch Changes
+
+- 2774e80: feat: added meta3.2 support via Bedrock including vision, tool call and inference region support
+
+## 0.0.42
+
+### Patch Changes
+
+- df441e2: fix: consoleLogger is missing from `@llamaindex/env`
+- Updated dependencies [df441e2]
+  - @llamaindex/core@0.2.8
+  - @llamaindex/env@0.1.13
+
+## 0.0.41
+
+### Patch Changes
+
+- Updated dependencies [6cce3b1]
+  - @llamaindex/core@0.2.7
+
+## 0.0.40
+
+### Patch Changes
+
+- 50e6b57: feat: add Amazon Bedrock Retriever
+- Updated dependencies [8b7fdba]
+  - @llamaindex/core@0.2.6
+
 ## 0.0.39
 
 ### Patch Changes

packages/community/README.md

@@ -5,8 +5,11 @@
 ## Current Features:
 
 - Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
-- Bedrock support for the Meta LLama 2, 3 and 3.1 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
-- Meta LLama3.1 405b tool call support
+- Bedrock support for the Meta LLama 2, 3, 3.1 and 3.2 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
+- Meta LLama3.1 405b and Llama3.2 tool call support
+- Meta 3.2 11B and 90B vision support
+- Bedrock support for querying Knowledge Base
+- Bedrock: [Supported Regions and models for cross-region inference](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference-support.html)
 
 ## LICENSE
 

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.39",
+  "version": "0.0.50",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -43,9 +43,10 @@
   },
   "devDependencies": {
     "@types/node": "^22.5.1",
-    "bunchee": "5.3.2"
+    "bunchee": "5.5.1"
   },
   "dependencies": {
+    "@aws-sdk/client-bedrock-agent-runtime": "^3.642.0",
     "@aws-sdk/client-bedrock-runtime": "^3.642.0",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*"

packages/community/src/index.ts

@@ -2,4 +2,7 @@ export {
   BEDROCK_MODELS,
   BEDROCK_MODEL_MAX_TOKENS,
   Bedrock,
+  INFERENCE_BEDROCK_MODELS,
+  INFERENCE_TO_BEDROCK_MAP,
 } from "./llm/bedrock/index.js";
+export { AmazonKnowledgeBaseRetriever } from "./retrievers/bedrock.js";

packages/community/src/llm/bedrock/anthropic/utils.ts

@@ -6,7 +6,10 @@ import type {
   MessageContentDetail,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
-import { mapMessageContentToMessageContentDetails } from "../utils";
+import {
+  extractDataUrlComponents,
+  mapMessageContentToMessageContentDetails,
+} from "../utils";
 import type {
   AnthropicContent,
   AnthropicImageContent,
@@ -143,27 +146,6 @@ export const mapTextContent = (text: string): AnthropicTextContent => {
   return { type: "text", text };
 };
 
-export const extractDataUrlComponents = (
-  dataUrl: string,
-): {
-  mimeType: string;
-  base64: string;
-} => {
-  const parts = dataUrl.split(";base64,");
-
-  if (parts.length !== 2 || !parts[0]!.startsWith("data:")) {
-    throw new Error("Invalid data URL");
-  }
-
-  const mimeType = parts[0]!.slice(5);
-  const base64 = parts[1]!;
-
-  return {
-    mimeType,
-    base64,
-  };
-};
-
 export const mapImageContent = (imageUrl: string): AnthropicImageContent => {
   if (!imageUrl.startsWith("data:"))
     throw new Error(

packages/community/src/llm/bedrock/index.ts

@@ -16,7 +16,7 @@ import {
   ToolCallLLM,
   type ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
-import { streamConverter, wrapLLMEvent } from "@llamaindex/core/utils";
+import { streamConverter } from "@llamaindex/core/utils";
 import {
   type BedrockAdditionalChatOptions,
   type BedrockChatStreamResponse,
@@ -24,6 +24,7 @@ import {
 } from "./provider";
 import { mapMessageContentToMessageContentDetails } from "./utils";
 
+import { wrapLLMEvent } from "@llamaindex/core/decorator";
 import { AnthropicProvider } from "./anthropic/provider";
 import { MetaProvider } from "./meta/provider";
 
@@ -46,35 +47,96 @@ export type BedrockChatParamsNonStreaming = LLMChatParamsNonStreaming<
 export type BedrockChatNonStreamResponse =
   ChatResponse<ToolCallLLMMessageOptions>;
 
-export enum BEDROCK_MODELS {
-  AMAZON_TITAN_TG1_LARGE = "amazon.titan-tg1-large",
-  AMAZON_TITAN_TEXT_EXPRESS_V1 = "amazon.titan-text-express-v1",
-  AI21_J2_GRANDE_INSTRUCT = "ai21.j2-grande-instruct",
-  AI21_J2_JUMBO_INSTRUCT = "ai21.j2-jumbo-instruct",
-  AI21_J2_MID = "ai21.j2-mid",
-  AI21_J2_MID_V1 = "ai21.j2-mid-v1",
-  AI21_J2_ULTRA = "ai21.j2-ultra",
-  AI21_J2_ULTRA_V1 = "ai21.j2-ultra-v1",
-  COHERE_COMMAND_TEXT_V14 = "cohere.command-text-v14",
-  ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1",
-  ANTHROPIC_CLAUDE_1 = "anthropic.claude-v1", // EOF: No longer supported
-  ANTHROPIC_CLAUDE_2 = "anthropic.claude-v2",
-  ANTHROPIC_CLAUDE_2_1 = "anthropic.claude-v2:1",
-  ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0",
-  ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0",
-  ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0",
-  ANTHROPIC_CLAUDE_3_5_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0",
-  META_LLAMA2_13B_CHAT = "meta.llama2-13b-chat-v1",
-  META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1",
-  META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0",
-  META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0",
-  META_LLAMA3_1_8B_INSTRUCT = "meta.llama3-1-8b-instruct-v1:0",
-  META_LLAMA3_1_70B_INSTRUCT = "meta.llama3-1-70b-instruct-v1:0",
-  META_LLAMA3_1_405B_INSTRUCT = "meta.llama3-1-405b-instruct-v1:0",
-  MISTRAL_7B_INSTRUCT = "mistral.mistral-7b-instruct-v0:2",
-  MISTRAL_MIXTRAL_7B_INSTRUCT = "mistral.mixtral-8x7b-instruct-v0:1",
-  MISTRAL_MIXTRAL_LARGE_2402 = "mistral.mistral-large-2402-v1:0",
-}
+export const BEDROCK_MODELS = {
+  AMAZON_TITAN_TG1_LARGE: "amazon.titan-tg1-large",
+  AMAZON_TITAN_TEXT_EXPRESS_V1: "amazon.titan-text-express-v1",
+  AI21_J2_GRANDE_INSTRUCT: "ai21.j2-grande-instruct",
+  AI21_J2_JUMBO_INSTRUCT: "ai21.j2-jumbo-instruct",
+  AI21_J2_MID: "ai21.j2-mid",
+  AI21_J2_MID_V1: "ai21.j2-mid-v1",
+  AI21_J2_ULTRA: "ai21.j2-ultra",
+  AI21_J2_ULTRA_V1: "ai21.j2-ultra-v1",
+  COHERE_COMMAND_TEXT_V14: "cohere.command-text-v14",
+  ANTHROPIC_CLAUDE_INSTANT_1: "anthropic.claude-instant-v1",
+  ANTHROPIC_CLAUDE_1: "anthropic.claude-v1", // EOF: No longer supported
+  ANTHROPIC_CLAUDE_2: "anthropic.claude-v2",
+  ANTHROPIC_CLAUDE_2_1: "anthropic.claude-v2:1",
+  ANTHROPIC_CLAUDE_3_SONNET: "anthropic.claude-3-sonnet-20240229-v1:0",
+  ANTHROPIC_CLAUDE_3_HAIKU: "anthropic.claude-3-haiku-20240307-v1:0",
+  ANTHROPIC_CLAUDE_3_OPUS: "anthropic.claude-3-opus-20240229-v1:0",
+  ANTHROPIC_CLAUDE_3_5_SONNET: "anthropic.claude-3-5-sonnet-20240620-v1:0",
+  META_LLAMA2_13B_CHAT: "meta.llama2-13b-chat-v1",
+  META_LLAMA2_70B_CHAT: "meta.llama2-70b-chat-v1",
+  META_LLAMA3_8B_INSTRUCT: "meta.llama3-8b-instruct-v1:0",
+  META_LLAMA3_70B_INSTRUCT: "meta.llama3-70b-instruct-v1:0",
+  META_LLAMA3_1_8B_INSTRUCT: "meta.llama3-1-8b-instruct-v1:0",
+  META_LLAMA3_1_70B_INSTRUCT: "meta.llama3-1-70b-instruct-v1:0",
+  META_LLAMA3_1_405B_INSTRUCT: "meta.llama3-1-405b-instruct-v1:0",
+  META_LLAMA3_2_1B_INSTRUCT: "meta.llama3-2-1b-instruct-v1:0",
+  META_LLAMA3_2_3B_INSTRUCT: "meta.llama3-2-3b-instruct-v1:0",
+  META_LLAMA3_2_11B_INSTRUCT: "meta.llama3-2-11b-instruct-v1:0",
+  META_LLAMA3_2_90B_INSTRUCT: "meta.llama3-2-90b-instruct-v1:0",
+  MISTRAL_7B_INSTRUCT: "mistral.mistral-7b-instruct-v0:2",
+  MISTRAL_MIXTRAL_7B_INSTRUCT: "mistral.mixtral-8x7b-instruct-v0:1",
+  MISTRAL_MIXTRAL_LARGE_2402: "mistral.mistral-large-2402-v1:0",
+};
+export type BEDROCK_MODELS =
+  (typeof BEDROCK_MODELS)[keyof typeof BEDROCK_MODELS];
+
+export const INFERENCE_BEDROCK_MODELS = {
+  US_ANTHROPIC_CLAUDE_3_HAIKU: "us.anthropic.claude-3-haiku-20240307-v1:0",
+  US_ANTHROPIC_CLAUDE_3_OPUS: "us.anthropic.claude-3-opus-20240229-v1:0",
+  US_ANTHROPIC_CLAUDE_3_SONNET: "us.anthropic.claude-3-sonnet-20240229-v1:0",
+  US_ANTHROPIC_CLAUDE_3_5_SONNET:
+    "us.anthropic.claude-3-5-sonnet-20240620-v1:0",
+  US_META_LLAMA_3_2_1B_INSTRUCT: "us.meta.llama3-2-1b-instruct-v1:0",
+  US_META_LLAMA_3_2_3B_INSTRUCT: "us.meta.llama3-2-3b-instruct-v1:0",
+  US_META_LLAMA_3_2_11B_INSTRUCT: "us.meta.llama3-2-11b-instruct-v1:0",
+  US_META_LLAMA_3_2_90B_INSTRUCT: "us.meta.llama3-2-90b-instruct-v1:0",
+
+  EU_ANTHROPIC_CLAUDE_3_HAIKU: "eu.anthropic.claude-3-haiku-20240307-v1:0",
+  EU_ANTHROPIC_CLAUDE_3_SONNET: "eu.anthropic.claude-3-sonnet-20240229-v1:0",
+  EU_ANTHROPIC_CLAUDE_3_5_SONNET:
+    "eu.anthropic.claude-3-5-sonnet-20240620-v1:0",
+  EU_META_LLAMA_3_2_1B_INSTRUCT: "eu.meta.llama3-2-1b-instruct-v1:0",
+  EU_META_LLAMA_3_2_3B_INSTRUCT: "eu.meta.llama3-2-3b-instruct-v1:0",
+};
+
+export type INFERENCE_BEDROCK_MODELS =
+  (typeof INFERENCE_BEDROCK_MODELS)[keyof typeof INFERENCE_BEDROCK_MODELS];
+
+export const INFERENCE_TO_BEDROCK_MAP: Record<
+  INFERENCE_BEDROCK_MODELS,
+  BEDROCK_MODELS
+> = {
+  [INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_HAIKU]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+  [INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_OPUS]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
+  [INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_SONNET]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
+  [INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_5_SONNET]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
+  [INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_1B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
+  [INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_3B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT,
+  [INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_11B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT,
+  [INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_90B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT,
+
+  [INFERENCE_BEDROCK_MODELS.EU_ANTHROPIC_CLAUDE_3_HAIKU]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+  [INFERENCE_BEDROCK_MODELS.EU_ANTHROPIC_CLAUDE_3_SONNET]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
+  [INFERENCE_BEDROCK_MODELS.EU_ANTHROPIC_CLAUDE_3_5_SONNET]:
+    BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
+  [INFERENCE_BEDROCK_MODELS.EU_META_LLAMA_3_2_1B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
+  [INFERENCE_BEDROCK_MODELS.EU_META_LLAMA_3_2_3B_INSTRUCT]:
+    BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT,
+};
 
 /*
  * Values taken from https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html#model-parameters-claude
@@ -108,6 +170,10 @@ const CHAT_ONLY_MODELS = {
   [BEDROCK_MODELS.META_LLAMA3_1_8B_INSTRUCT]: 128000,
   [BEDROCK_MODELS.META_LLAMA3_1_70B_INSTRUCT]: 128000,
   [BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT]: 128000,
+  [BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT]: 131000,
+  [BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT]: 131000,
+  [BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT]: 128000,
+  [BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT]: 128000,
   [BEDROCK_MODELS.MISTRAL_7B_INSTRUCT]: 32000,
   [BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT]: 32000,
   [BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402]: 32000,
@@ -138,17 +204,25 @@ export const STREAMING_MODELS = new Set([
   BEDROCK_MODELS.META_LLAMA3_1_8B_INSTRUCT,
   BEDROCK_MODELS.META_LLAMA3_1_70B_INSTRUCT,
   BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT,
   BEDROCK_MODELS.MISTRAL_7B_INSTRUCT,
   BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT,
   BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402,
 ]);
 
-export const TOOL_CALL_MODELS = [
+export const TOOL_CALL_MODELS: BEDROCK_MODELS[] = [
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
   BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT,
 ];
 
 const getProvider = (model: string): Provider => {
@@ -165,7 +239,7 @@ const getProvider = (model: string): Provider => {
 };
 
 export type BedrockModelParams = {
-  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
+  model: BEDROCK_MODELS | INFERENCE_BEDROCK_MODELS;
   temperature?: number;
   topP?: number;
   maxTokens?: number;
@@ -184,6 +258,10 @@ export const BEDROCK_MODEL_MAX_TOKENS: Partial<Record<BEDROCK_MODELS, number>> =
     [BEDROCK_MODELS.META_LLAMA3_1_8B_INSTRUCT]: 2048,
     [BEDROCK_MODELS.META_LLAMA3_1_70B_INSTRUCT]: 2048,
     [BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_2_11B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_2_90B_INSTRUCT]: 2048,
   };
 
 const DEFAULT_BEDROCK_PARAMS = {
@@ -192,14 +270,15 @@ const DEFAULT_BEDROCK_PARAMS = {
   maxTokens: 1024, // required by anthropic
 };
 
-export type BedrockParams = BedrockModelParams & BedrockRuntimeClientConfig;
+export type BedrockParams = BedrockRuntimeClientConfig & BedrockModelParams;
 
 /**
  * ToolCallLLM for Bedrock
  */
 export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
   private client: BedrockRuntimeClient;
-  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
+  protected actualModel: BEDROCK_MODELS | INFERENCE_BEDROCK_MODELS;
+  model: BEDROCK_MODELS;
   temperature: number;
   topP: number;
   maxTokens?: number;
@@ -216,8 +295,8 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
     ...params
   }: BedrockParams) {
     super();
-
-    this.model = model;
+    this.actualModel = model;
+    this.model = INFERENCE_TO_BEDROCK_MAP[model] ?? model;
     this.provider = getProvider(this.model);
     this.maxTokens = maxTokens ?? DEFAULT_BEDROCK_PARAMS.maxTokens;
     this.temperature = temperature ?? DEFAULT_BEDROCK_PARAMS.temperature;
@@ -240,7 +319,7 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
       temperature: this.temperature,
       topP: this.topP,
       maxTokens: this.maxTokens,
-      contextWindow: BEDROCK_FOUNDATION_LLMS[this.model],
+      contextWindow: BEDROCK_FOUNDATION_LLMS[this.model] ?? 128000,
       tokenizer: undefined,
     };
   }
@@ -255,6 +334,8 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
       params.additionalChatOptions,
     );
     const command = new InvokeModelCommand(input);
+    command.input.modelId = this.actualModel;
+
     const response = await this.client.send(command);
     let options: ToolCallLLMMessageOptions = {};
     if (this.supportToolCall) {
@@ -286,6 +367,8 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
       params.additionalChatOptions,
     );
     const command = new InvokeModelWithResponseStreamCommand(input);
+    command.input.modelId = this.actualModel;
+
     const response = await this.client.send(command);
 
     if (response.body) yield* this.provider.reduceStream(response.body);

packages/community/src/llm/bedrock/meta/provider.ts

@@ -67,21 +67,26 @@ export class MetaProvider extends Provider<MetaStreamEvent> {
     for await (const response of stream) {
       const event = this.getStreamingEventResponse(response);
       const delta = this.getTextFromStreamResponse(response);
+
       // odd quirk of llama3.1, start token is \n\n
       if (
+        !toolId &&
         !event?.generation.trim() &&
         event?.generation_token_count === 1 &&
-        event.prompt_token_count !== null
+        event?.prompt_token_count !== null
       )
         continue;
 
-      if (delta === TOKENS.TOOL_CALL) {
+      if (delta.startsWith(TOKENS.TOOL_CALL)) {
         toolId = randomUUID();
+        const parts = delta.split(TOKENS.TOOL_CALL).filter((part) => part);
+        collecting.push(...parts);
         continue;
       }
 
       let options: undefined | ToolCallLLMMessageOptions = undefined;
       if (toolId && event?.stop_reason === "stop") {
+        if (delta) collecting.push(delta);
         const tool = JSON.parse(collecting.join(""));
         options = {
           toolCall: [
@@ -110,11 +115,18 @@ export class MetaProvider extends Provider<MetaStreamEvent> {
   getRequestBody<T extends ChatMessage>(
     metadata: LLMMetadata,
     messages: T[],
-    tools?: BaseTool[],
+    tools: BaseTool[] = [],
   ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
     let prompt: string = "";
+    let images: string[] = [];
     if (metadata.model.startsWith("meta.llama3")) {
-      prompt = mapChatMessagesToMetaLlama3Messages(messages, tools);
+      const mapped = mapChatMessagesToMetaLlama3Messages({
+        messages,
+        tools,
+        model: metadata.model,
+      });
+      prompt = mapped.prompt;
+      images = mapped.images;
     } else if (metadata.model.startsWith("meta.llama2")) {
       prompt = mapChatMessagesToMetaLlama2Messages(messages);
     } else {
@@ -127,6 +139,7 @@ export class MetaProvider extends Provider<MetaStreamEvent> {
       accept: "application/json",
       body: JSON.stringify({
         prompt,
+        images: images.length ? images : undefined,
         max_gen_len: metadata.maxTokens,
         temperature: metadata.temperature,
         top_p: metadata.topP,

packages/community/src/llm/bedrock/meta/utils.ts

@@ -1,9 +1,12 @@
 import type {
   BaseTool,
   ChatMessage,
+  LLMMetadata,
   MessageContentTextDetail,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
+import { extractDataUrlComponents } from "../utils";
+import { TOKENS } from "./constants";
 import type { MetaMessage } from "./types";
 
 const getToolCallInstructionString = (tool: BaseTool): string => {
@@ -24,7 +27,7 @@ const getToolCallParametersString = (tool: BaseTool): string => {
 
 // ported from https://github.com/meta-llama/llama-agentic-system/blob/main/llama_agentic_system/system_prompt.py
 // NOTE: using json instead of the above xml style tool calling works more reliability
-export const getToolsPrompt = (tools?: BaseTool[]) => {
+export const getToolsPrompt_3_1 = (tools?: BaseTool[]) => {
   if (!tools?.length) return "";
 
   const customToolParams = tools.map((tool) => {
@@ -77,6 +80,46 @@ Reminder:
   `;
 };
 
+export const getToolsPrompt_3_2 = (tools?: BaseTool[]) => {
+  if (!tools?.length) return "";
+  return `
+You are an expert in composing functions. You are given a question and a set of possible functions.
+Based on the question, you will need to make one or more function/tool calls to achieve the purpose.
+If none of the function can be used, point it out. If the given question lacks the parameters required by the function,
+also point it out. You should only return the function call in tools call sections.
+
+If you decide to invoke any of the function(s), you MUST put it in the format of and start with the token: ${TOKENS.TOOL_CALL}:
+{
+  "name": function_name,
+  "parameters": parameters,
+}
+where
+
+{
+  "name": function_name,
+  "parameters": parameters, => a JSON dict with the function argument name as key and function argument value as value.
+}
+
+Here is an example,
+
+{
+  "name": "example_function_name",
+  "parameters": {"example_name": "example_value"}
+}
+
+Reminder:
+- Function calls MUST follow the specified format
+- Required parameters MUST be specified
+- Only call one function at a time
+- You SHOULD NOT include any other text in the response
+- Put the entire function call reply on one line
+
+Here is a list of functions in JSON format that you can invoke.
+
+${JSON.stringify(tools)}
+`;
+};
+
 export const mapChatRoleToMetaRole = (
   role: ChatMessage["role"],
 ): MetaMessage["role"] => {
@@ -125,16 +168,46 @@ export const mapChatMessagesToMetaMessages = <
 /**
  * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
  */
-export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>(
-  messages: T[],
-  tools?: BaseTool[],
-): string => {
+export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>({
+  messages,
+  model,
+  tools,
+}: {
+  messages: T[];
+  model: LLMMetadata["model"];
+  tools?: BaseTool[];
+}): { prompt: string; images: string[] } => {
+  const images: string[] = [];
+  const textMessages: T[] = [];
+
+  messages.forEach((message) => {
+    if (Array.isArray(message.content)) {
+      message.content.forEach((content) => {
+        if (content.type === "image_url") {
+          const { base64 } = extractDataUrlComponents(content.image_url.url);
+          images.push(base64);
+        } else {
+          textMessages.push(message);
+        }
+      });
+    } else {
+      textMessages.push(message);
+    }
+  });
+
   const parts: string[] = [];
-  if (tools?.length) {
+
+  let toolsPrompt = "";
+  if (model.startsWith("meta.llama3-2")) {
+    toolsPrompt = getToolsPrompt_3_2(tools);
+  } else if (model.startsWith("meta.llama3-1")) {
+    toolsPrompt = getToolsPrompt_3_1(tools);
+  }
+  if (toolsPrompt) {
     parts.push(
       "<|begin_of_text|>",
       "<|start_header_id|>system<|end_header_id|>",
-      getToolsPrompt(tools),
+      toolsPrompt,
       "<|eot_id|>",
     );
   }
@@ -154,7 +227,9 @@ export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>(
     ...mapped,
     "<|start_header_id|>assistant<|end_header_id|>",
   );
-  return parts.join("\n");
+
+  const prompt = parts.join("\n");
+  return { prompt, images };
 };
 
 /**

packages/community/src/llm/bedrock/utils.ts

@@ -11,3 +11,24 @@ export const mapMessageContentToMessageContentDetails = (
 
 export const toUtf8 = (input: Uint8Array): string =>
   new TextDecoder("utf-8").decode(input);
+
+export const extractDataUrlComponents = (
+  dataUrl: string,
+): {
+  mimeType: string;
+  base64: string;
+} => {
+  const parts = dataUrl.split(";base64,");
+
+  if (parts.length !== 2 || !parts[0]!.startsWith("data:")) {
+    throw new Error("Invalid data URL");
+  }
+
+  const mimeType = parts[0]!.slice(5);
+  const base64 = parts[1]!;
+
+  return {
+    mimeType,
+    base64,
+  };
+};

packages/community/src/retrievers/bedrock.ts

@@ -0,0 +1,165 @@
+import type { KnowledgeBaseVectorSearchConfiguration } from "@aws-sdk/client-bedrock-agent-runtime";
+import {
+  BedrockAgentRuntimeClient,
+  type BedrockAgentRuntimeClientConfig,
+  type RetrievalFilter,
+  RetrieveCommand,
+  type SearchType,
+} from "@aws-sdk/client-bedrock-agent-runtime";
+import type { QueryBundle } from "@llamaindex/core/query-engine";
+import { BaseRetriever } from "@llamaindex/core/retriever";
+import { Document, type NodeWithScore } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
+
+/**
+ * Interface for the arguments required to initialize an
+ * AmazonKnowledgeBaseRetriever instance.
+ */
+export interface AmazonKnowledgeBaseRetrieverArgs {
+  knowledgeBaseId: string;
+  topK: number;
+  region: string;
+  clientOptions?: BedrockAgentRuntimeClientConfig;
+  filter?: RetrievalFilter;
+  overrideSearchType?: SearchType;
+}
+
+/**
+ * Class for interacting with Amazon Bedrock Knowledge Bases, a RAG workflow oriented service
+ * Extends the BaseRetriever class.
+ * @example
+ * ```typescript
+ * const retriever = new AmazonKnowledgeBaseRetriever({
+ *   topK: 10,
+ *   knowledgeBaseId: "YOUR_KNOWLEDGE_BASE_ID",
+ *   region: "us-east-2",
+ *   clientOptions: {
+ *     credentials: {
+ *       accessKeyId: "YOUR_ACCESS_KEY_ID",
+ *       secretAccessKey: "YOUR_SECRET_ACCESS_KEY",
+ *     },
+ *   },
+ * });
+ *
+ * const docs = await retriever.retrieve({query: "How are clouds formed?"});
+ * ```
+ */
+export class AmazonKnowledgeBaseRetriever extends BaseRetriever {
+  static lc_name() {
+    return "AmazonKnowledgeBaseRetriever";
+  }
+
+  lc_namespace = ["llamaindex", "retrievers", "amazon_bedrock_knowledge_base"];
+
+  knowledgeBaseId: string;
+
+  topK: number;
+
+  bedrockAgentRuntimeClient: BedrockAgentRuntimeClient;
+
+  filter: RetrievalFilter | undefined;
+
+  overrideSearchType: SearchType | undefined;
+
+  constructor({
+    knowledgeBaseId,
+    topK = 10,
+    clientOptions,
+    region,
+    filter,
+    overrideSearchType,
+  }: AmazonKnowledgeBaseRetrieverArgs) {
+    super();
+
+    this.topK = topK;
+    this.filter = filter;
+    this.overrideSearchType = overrideSearchType;
+    this.bedrockAgentRuntimeClient = new BedrockAgentRuntimeClient({
+      region,
+      ...clientOptions,
+    });
+    this.knowledgeBaseId = knowledgeBaseId;
+  }
+
+  /**
+   * Cleans the result text by replacing sequences of whitespace with a
+   * single space and removing ellipses.
+   * @param resText The result text to clean.
+   * @returns The cleaned result text.
+   */
+  cleanResult(resText: string) {
+    const res = resText.replace(/\s+/g, " ").replace(/\.\.\./g, "");
+    return res;
+  }
+
+  async queryKnowledgeBase(
+    query: QueryBundle,
+    topK: number,
+    filter?: RetrievalFilter,
+    overrideSearchType?: SearchType,
+  ): Promise<NodeWithScore[]> {
+    const retrieveCommand = new RetrieveCommand({
+      knowledgeBaseId: this.knowledgeBaseId,
+      retrievalQuery: {
+        text: extractText(query),
+      },
+      retrievalConfiguration: {
+        vectorSearchConfiguration: {
+          numberOfResults: topK,
+          overrideSearchType,
+          filter,
+        } as KnowledgeBaseVectorSearchConfiguration,
+      },
+    });
+
+    const retrieveResponse =
+      await this.bedrockAgentRuntimeClient.send(retrieveCommand);
+
+    return (
+      retrieveResponse.retrievalResults?.map((result) => {
+        let source;
+        switch (result.location?.type) {
+          case "CONFLUENCE":
+            source = result.location?.confluenceLocation?.url;
+            break;
+          case "S3":
+            source = result.location?.s3Location?.uri;
+            break;
+          case "SALESFORCE":
+            source = result.location?.salesforceLocation?.url;
+            break;
+          case "SHAREPOINT":
+            source = result.location?.sharePointLocation?.url;
+            break;
+          case "WEB":
+            source = result.location?.webLocation?.url;
+            break;
+          default:
+            source = result.location?.s3Location?.uri;
+            break;
+        }
+
+        return {
+          node: new Document({
+            text: this.cleanResult(result.content?.text || ""),
+            metadata: {
+              source,
+              score: result.score,
+              ...result.metadata,
+            },
+          }),
+          score: result.score ?? 1.0,
+        };
+      }) ?? []
+    );
+  }
+
+  async _retrieve(query: QueryBundle): Promise<NodeWithScore[]> {
+    return await this.queryKnowledgeBase(
+      query,
+      this.topK,
+      this.filter,
+      this.overrideSearchType,
+    );
+  }
+}

packages/core/CHANGELOG.md

@@ -1,5 +1,76 @@
 # @llamaindex/core
 
+## 0.3.2
+
+### Patch Changes
+
+- Updated dependencies [4ba2cfe]
+  - @llamaindex/env@0.1.15
+
+## 0.3.1
+
+### Patch Changes
+
+- a75af83: refactor: move some llm and embedding to single package
+- Updated dependencies [ae49ff4]
+- Updated dependencies [a75af83]
+  - @llamaindex/env@0.1.14
+
+## 0.3.0
+
+### Minor Changes
+
+- 1364e8e: update metadata extractors to use PromptTemplate
+- 96fc69c: add defaultQuestionExtractPrompt
+
+## 0.2.12
+
+### Patch Changes
+
+- 5f67820: Fix that node parsers generate nodes with UUIDs
+
+## 0.2.11
+
+### Patch Changes
+
+- ee697fb: fix: generate uuid when inserting to Qdrant
+
+## 0.2.10
+
+### Patch Changes
+
+- 3489e7d: fix: num output incorrect in prompt helper
+- 468bda5: fix: correct warning when chunk size smaller than 0
+
+## 0.2.9
+
+### Patch Changes
+
+- b17d439: Fix #1278: resolved issue where the id\_ was not correctly passed as the id when creating a TextNode. As a result, the upsert operation to the vector database was using a generated ID instead of the provided document ID, if available.
+
+## 0.2.8
+
+### Patch Changes
+
+- df441e2: fix: consoleLogger is missing from `@llamaindex/env`
+- Updated dependencies [df441e2]
+  - @llamaindex/env@0.1.13
+
+## 0.2.7
+
+### Patch Changes
+
+- 6cce3b1: feat: support `npm:postgres`
+
+## 0.2.6
+
+### Patch Changes
+
+- 8b7fdba: refactor: move chat engine & retriever into core.
+
+  - `chatHistory` in BaseChatEngine now returns `ChatMessage[] | Promise<ChatMessage[]>`, instead of `BaseMemory`
+  - update `retrieve-end` type
+
 ## 0.2.5
 
 ### Patch Changes

packages/core/agent/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/chat-engine/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/data-structs/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/decorator/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/embeddings/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/global/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/indices/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/llms/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/memory/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/node-parser/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/objects/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/package.json

@@ -1,208 +1,326 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.2.5",
+  "version": "0.3.2",
   "description": "LlamaIndex Core Module",
   "exports": {
-    "./node-parser": {
+    "./agent": {
       "require": {
-        "types": "./dist/node-parser/index.d.cts",
-        "default": "./dist/node-parser/index.cjs"
+        "types": "./agent/dist/index.d.cts",
+        "default": "./agent/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/node-parser/index.d.ts",
-        "default": "./dist/node-parser/index.js"
+        "types": "./agent/dist/index.d.ts",
+        "default": "./agent/dist/index.js"
       },
       "default": {
-        "types": "./dist/node-parser/index.d.ts",
-        "default": "./dist/node-parser/index.js"
+        "types": "./agent/dist/index.d.ts",
+        "default": "./agent/dist/index.js"
+      }
+    },
+    "./objects": {
+      "require": {
+        "types": "./objects/dist/index.d.cts",
+        "default": "./objects/dist/index.cjs"
+      },
+      "import": {
+        "types": "./objects/dist/index.d.ts",
+        "default": "./objects/dist/index.js"
+      },
+      "default": {
+        "types": "./objects/dist/index.d.ts",
+        "default": "./objects/dist/index.js"
+      }
+    },
+    "./node-parser": {
+      "require": {
+        "types": "./node-parser/dist/index.d.cts",
+        "default": "./node-parser/dist/index.cjs"
+      },
+      "import": {
+        "types": "./node-parser/dist/index.d.ts",
+        "default": "./node-parser/dist/index.js"
+      },
+      "default": {
+        "types": "./node-parser/dist/index.d.ts",
+        "default": "./node-parser/dist/index.js"
       }
     },
     "./query-engine": {
       "require": {
-        "types": "./dist/query-engine/index.d.cts",
-        "default": "./dist/query-engine/index.cjs"
+        "types": "./query-engine/dist/index.d.cts",
+        "default": "./query-engine/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/query-engine/index.d.ts",
-        "default": "./dist/query-engine/index.js"
+        "types": "./query-engine/dist/index.d.ts",
+        "default": "./query-engine/dist/index.js"
       },
       "default": {
-        "types": "./dist/query-engine/index.d.ts",
-        "default": "./dist/query-engine/index.js"
+        "types": "./query-engine/dist/index.d.ts",
+        "default": "./query-engine/dist/index.js"
       }
     },
     "./llms": {
       "require": {
-        "types": "./dist/llms/index.d.cts",
-        "default": "./dist/llms/index.cjs"
+        "types": "./llms/dist/index.d.cts",
+        "default": "./llms/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/llms/index.d.ts",
-        "default": "./dist/llms/index.js"
+        "types": "./llms/dist/index.d.ts",
+        "default": "./llms/dist/index.js"
       },
       "default": {
-        "types": "./dist/llms/index.d.ts",
-        "default": "./dist/llms/index.js"
+        "types": "./llms/dist/index.d.ts",
+        "default": "./llms/dist/index.js"
       }
     },
     "./decorator": {
       "require": {
-        "types": "./dist/decorator/index.d.cts",
-        "default": "./dist/decorator/index.cjs"
+        "types": "./decorator/dist/index.d.cts",
+        "default": "./decorator/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/decorator/index.d.ts",
-        "default": "./dist/decorator/index.js"
+        "types": "./decorator/dist/index.d.ts",
+        "default": "./decorator/dist/index.js"
       },
       "default": {
-        "types": "./dist/decorator/index.d.ts",
-        "default": "./dist/decorator/index.js"
+        "types": "./decorator/dist/index.d.ts",
+        "default": "./decorator/dist/index.js"
       }
     },
     "./embeddings": {
       "require": {
-        "types": "./dist/embeddings/index.d.cts",
-        "default": "./dist/embeddings/index.cjs"
+        "types": "./embeddings/dist/index.d.cts",
+        "default": "./embeddings/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/embeddings/index.d.ts",
-        "default": "./dist/embeddings/index.js"
+        "types": "./embeddings/dist/index.d.ts",
+        "default": "./embeddings/dist/index.js"
       },
       "default": {
-        "types": "./dist/embeddings/index.d.ts",
-        "default": "./dist/embeddings/index.js"
+        "types": "./embeddings/dist/index.d.ts",
+        "default": "./embeddings/dist/index.js"
       }
     },
     "./global": {
       "require": {
-        "types": "./dist/global/index.d.cts",
-        "default": "./dist/global/index.cjs"
+        "types": "./global/dist/index.d.cts",
+        "default": "./global/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/global/index.d.ts",
-        "default": "./dist/global/index.js"
+        "types": "./global/dist/index.d.ts",
+        "default": "./global/dist/index.js"
       },
       "default": {
-        "types": "./dist/global/index.d.ts",
-        "default": "./dist/global/index.js"
+        "types": "./global/dist/index.d.ts",
+        "default": "./global/dist/index.js"
       }
     },
     "./schema": {
       "require": {
-        "types": "./dist/schema/index.d.cts",
-        "default": "./dist/schema/index.cjs"
+        "types": "./schema/dist/index.d.cts",
+        "default": "./schema/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/schema/index.d.ts",
-        "default": "./dist/schema/index.js"
+        "types": "./schema/dist/index.d.ts",
+        "default": "./schema/dist/index.js"
       },
       "default": {
-        "types": "./dist/schema/index.d.ts",
-        "default": "./dist/schema/index.js"
+        "types": "./schema/dist/index.d.ts",
+        "default": "./schema/dist/index.js"
       }
     },
     "./utils": {
       "require": {
-        "types": "./dist/utils/index.d.cts",
-        "default": "./dist/utils/index.cjs"
+        "types": "./utils/dist/index.d.cts",
+        "default": "./utils/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/utils/index.d.ts",
-        "default": "./dist/utils/index.js"
+        "types": "./utils/dist/index.d.ts",
+        "default": "./utils/dist/index.js"
       },
       "default": {
-        "types": "./dist/utils/index.d.ts",
-        "default": "./dist/utils/index.js"
+        "types": "./utils/dist/index.d.ts",
+        "default": "./utils/dist/index.js"
       }
     },
     "./prompts": {
       "require": {
-        "types": "./dist/prompts/index.d.cts",
-        "default": "./dist/prompts/index.cjs"
+        "types": "./prompts/dist/index.d.cts",
+        "default": "./prompts/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/prompts/index.d.ts",
-        "default": "./dist/prompts/index.js"
+        "types": "./prompts/dist/index.d.ts",
+        "default": "./prompts/dist/index.js"
       },
       "default": {
-        "types": "./dist/prompts/index.d.ts",
-        "default": "./dist/prompts/index.js"
+        "types": "./prompts/dist/index.d.ts",
+        "default": "./prompts/dist/index.js"
       }
     },
     "./indices": {
       "require": {
-        "types": "./dist/indices/index.d.cts",
-        "default": "./dist/indices/index.cjs"
+        "types": "./indices/dist/index.d.cts",
+        "default": "./indices/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/indices/index.d.ts",
-        "default": "./dist/indices/index.js"
+        "types": "./indices/dist/index.d.ts",
+        "default": "./indices/dist/index.js"
       },
       "default": {
-        "types": "./dist/indices/index.d.ts",
-        "default": "./dist/indices/index.js"
+        "types": "./indices/dist/index.d.ts",
+        "default": "./indices/dist/index.js"
       }
     },
     "./workflow": {
       "require": {
-        "types": "./dist/workflow/index.d.cts",
-        "default": "./dist/workflow/index.cjs"
+        "types": "./workflow/dist/index.d.cts",
+        "default": "./workflow/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/workflow/index.d.ts",
-        "default": "./dist/workflow/index.js"
+        "types": "./workflow/dist/index.d.ts",
+        "default": "./workflow/dist/index.js"
       },
       "default": {
-        "types": "./dist/workflow/index.d.ts",
-        "default": "./dist/workflow/index.js"
+        "types": "./workflow/dist/index.d.ts",
+        "default": "./workflow/dist/index.js"
       }
     },
     "./memory": {
       "require": {
-        "types": "./dist/memory/index.d.cts",
-        "default": "./dist/memory/index.cjs"
+        "types": "./memory/dist/index.d.cts",
+        "default": "./memory/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/memory/index.d.ts",
-        "default": "./dist/memory/index.js"
+        "types": "./memory/dist/index.d.ts",
+        "default": "./memory/dist/index.js"
       },
       "default": {
-        "types": "./dist/memory/index.d.ts",
-        "default": "./dist/memory/index.js"
+        "types": "./memory/dist/index.d.ts",
+        "default": "./memory/dist/index.js"
       }
     },
     "./storage/chat-store": {
       "require": {
-        "types": "./dist/storage/chat-store/index.d.cts",
-        "default": "./dist/storage/chat-store/index.cjs"
+        "types": "./storage/chat-store/dist/index.d.cts",
+        "default": "./storage/chat-store/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/storage/chat-store/index.d.ts",
-        "default": "./dist/storage/chat-store/index.js"
+        "types": "./storage/chat-store/dist/index.d.ts",
+        "default": "./storage/chat-store/dist/index.js"
       },
       "default": {
-        "types": "./dist/storage/chat-store/index.d.ts",
-        "default": "./dist/storage/chat-store/index.js"
+        "types": "./storage/chat-store/dist/index.d.ts",
+        "default": "./storage/chat-store/dist/index.js"
       }
     },
     "./response-synthesizers": {
       "require": {
-        "types": "./dist/response-synthesizers/index.d.cts",
-        "default": "./dist/response-synthesizers/index.cjs"
+        "types": "./response-synthesizers/dist/index.d.cts",
+        "default": "./response-synthesizers/dist/index.cjs"
       },
       "import": {
-        "types": "./dist/response-synthesizers/index.d.ts",
-        "default": "./dist/response-synthesizers/index.js"
+        "types": "./response-synthesizers/dist/index.d.ts",
+        "default": "./response-synthesizers/dist/index.js"
       },
       "default": {
-        "types": "./dist/response-synthesizers/index.d.ts",
-        "default": "./dist/response-synthesizers/index.js"
+        "types": "./response-synthesizers/dist/index.d.ts",
+        "default": "./response-synthesizers/dist/index.js"
+      }
+    },
+    "./chat-engine": {
+      "require": {
+        "types": "./chat-engine/dist/index.d.cts",
+        "default": "./chat-engine/dist/index.cjs"
+      },
+      "import": {
+        "types": "./chat-engine/dist/index.d.ts",
+        "default": "./chat-engine/dist/index.js"
+      },
+      "default": {
+        "types": "./chat-engine/dist/index.d.ts",
+        "default": "./chat-engine/dist/index.js"
+      }
+    },
+    "./retriever": {
+      "require": {
+        "types": "./retriever/dist/index.d.cts",
+        "default": "./retriever/dist/index.cjs"
+      },
+      "import": {
+        "types": "./retriever/dist/index.d.ts",
+        "default": "./retriever/dist/index.js"
+      },
+      "default": {
+        "types": "./retriever/dist/index.d.ts",
+        "default": "./retriever/dist/index.js"
+      }
+    },
+    "./vector-store": {
+      "require": {
+        "types": "./vector-store/dist/index.d.cts",
+        "default": "./vector-store/dist/index.cjs"
+      },
+      "import": {
+        "types": "./vector-store/dist/index.d.ts",
+        "default": "./vector-store/dist/index.js"
+      },
+      "default": {
+        "types": "./vector-store/dist/index.d.ts",
+        "default": "./vector-store/dist/index.js"
+      }
+    },
+    "./tools": {
+      "require": {
+        "types": "./tools/dist/index.d.cts",
+        "default": "./tools/dist/index.cjs"
+      },
+      "import": {
+        "types": "./tools/dist/index.d.ts",
+        "default": "./tools/dist/index.js"
+      },
+      "default": {
+        "types": "./tools/dist/index.d.ts",
+        "default": "./tools/dist/index.js"
+      }
+    },
+    "./data-structs": {
+      "require": {
+        "types": "./data-structs/dist/index.d.cts",
+        "default": "./data-structs/dist/index.cjs"
+      },
+      "import": {
+        "types": "./data-structs/dist/index.d.ts",
+        "default": "./data-structs/dist/index.js"
+      },
+      "default": {
+        "types": "./data-structs/dist/index.d.ts",
+        "default": "./data-structs/dist/index.js"
       }
     }
   },
   "files": [
-    "dist"
+    "./agent",
+    "./objects",
+    "./node-parser",
+    "./query-engine",
+    "./llms",
+    "./decorator",
+    "./embeddings",
+    "./global",
+    "./schema",
+    "./utils",
+    "./prompts",
+    "./indices",
+    "./workflow",
+    "./memory",
+    "./storage",
+    "./response-synthesizers",
+    "./chat-engine",
+    "./retriever",
+    "./vector-store",
+    "./tools",
+    "./data-structs"
   ],
   "scripts": {
     "dev": "bunchee --watch",
@@ -211,12 +329,12 @@
   "repository": {
     "type": "git",
     "directory": "packages/core",
-    "url": "https://github.com/himself65/LlamaIndexTS.git"
+    "url": "https://github.com/run-llama/LlamaIndexTS.git"
   },
   "devDependencies": {
     "@edge-runtime/vm": "^4.0.3",
     "ajv": "^8.17.1",
-    "bunchee": "5.3.2",
+    "bunchee": "5.5.1",
     "happy-dom": "^15.7.4",
     "natural": "^8.0.1",
     "python-format-js": "^1.4.3"
@@ -225,6 +343,7 @@
     "@llamaindex/env": "workspace:*",
     "@types/node": "^22.5.1",
     "magic-bytes.js": "^1.10.0",
-    "zod": "^3.23.8"
+    "zod": "^3.23.8",
+    "zod-to-json-schema": "^3.23.3"
   }
 }

packages/core/prompts/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/query-engine/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/response-synthesizers/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/retriever/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/schema/package.json

@@ -0,0 +1,8 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "private": true
+}

packages/core/src/agent/base.ts

@@ -1,23 +1,21 @@
+import { consoleLogger, emptyLogger, randomUUID } from "@llamaindex/env";
+import {
+  BaseChatEngine,
+  type NonStreamingChatEngineParams,
+  type StreamingChatEngineParams,
+} from "../chat-engine";
+import { wrapEventCaller } from "../decorator";
+import { Settings } from "../global";
 import type {
   BaseToolWithCall,
   ChatMessage,
   LLM,
   MessageContent,
   ToolOutput,
-} from "@llamaindex/core/llms";
-import { BaseMemory } from "@llamaindex/core/memory";
-import { EngineResponse } from "@llamaindex/core/schema";
-import { wrapEventCaller } from "@llamaindex/core/utils";
-import { randomUUID } from "@llamaindex/env";
-import { Settings } from "../Settings.js";
-import {
-  type ChatEngine,
-  type ChatEngineParamsNonStreaming,
-  type ChatEngineParamsStreaming,
-} from "../engines/chat/index.js";
-import { consoleLogger, emptyLogger } from "../internal/logger.js";
-import { isReadableStream } from "../internal/utils.js";
-import { ObjectRetriever } from "../objects/index.js";
+} from "../llms";
+import { BaseMemory } from "../memory";
+import type { ObjectRetriever } from "../objects";
+import { EngineResponse } from "../schema";
 import type {
   AgentTaskContext,
   TaskHandler,
@@ -207,8 +205,7 @@ export abstract class AgentRunner<
   >
     ? AdditionalMessageOptions
     : never,
-> implements ChatEngine
-{
+> extends BaseChatEngine {
   readonly #llm: AI;
   readonly #tools:
     | BaseToolWithCall[]
@@ -259,6 +256,7 @@ export abstract class AgentRunner<
   protected constructor(
     params: AgentRunnerParams<AI, Store, AdditionalMessageOptions>,
   ) {
+    super();
     const { llm, chatHistory, systemPrompt, runner, tools, verbose } = params;
     this.#llm = llm;
     this.#chatHistory = chatHistory;
@@ -345,13 +343,13 @@ export abstract class AgentRunner<
     });
   }
 
-  async chat(params: ChatEngineParamsNonStreaming): Promise<EngineResponse>;
+  async chat(params: NonStreamingChatEngineParams): Promise<EngineResponse>;
   async chat(
-    params: ChatEngineParamsStreaming,
+    params: StreamingChatEngineParams,
   ): Promise<ReadableStream<EngineResponse>>;
   @wrapEventCaller
   async chat(
-    params: ChatEngineParamsNonStreaming | ChatEngineParamsStreaming,
+    params: NonStreamingChatEngineParams | StreamingChatEngineParams,
   ): Promise<EngineResponse | ReadableStream<EngineResponse>> {
     let chatHistory: ChatMessage<AdditionalMessageOptions>[] = [];
 
@@ -374,7 +372,7 @@ export abstract class AgentRunner<
       this.#chatHistory = [...stepOutput.taskStep.context.store.messages];
       if (stepOutput.isLast) {
         const { output } = stepOutput;
-        if (isReadableStream(output)) {
+        if (output instanceof ReadableStream) {
           return output.pipeThrough<EngineResponse>(
             new TransformStream({
               transform(chunk, controller) {

packages/core/src/agent/index.ts

@@ -0,0 +1,11 @@
+export { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
+export { LLMAgent, LLMAgentWorker, type LLMAgentParams } from "./llm.js";
+export type { AgentEndEvent, AgentStartEvent, TaskHandler } from "./types.js";
+export {
+  callTool,
+  consumeAsyncIterable,
+  createReadableStream,
+  stepTools,
+  stepToolsStreaming,
+  validateAgentParams,
+} from "./utils.js";

packages/core/src/agent/llm.ts

@@ -1,6 +1,6 @@
-import type { BaseToolWithCall, LLM } from "@llamaindex/core/llms";
-import { ObjectRetriever } from "../objects/index.js";
-import { Settings } from "../Settings.js";
+import { Settings } from "../global";
+import type { BaseToolWithCall, LLM } from "../llms";
+import { ObjectRetriever } from "../objects";
 import { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
 import { validateAgentParams } from "./utils.js";
 

packages/core/src/agent/types.ts

@@ -1,3 +1,5 @@
+import type { Logger } from "@llamaindex/env";
+import type { UUID } from "../global";
 import type {
   BaseToolWithCall,
   ChatMessage,
@@ -6,9 +8,7 @@ import type {
   LLM,
   MessageContent,
   ToolOutput,
-} from "@llamaindex/core/llms";
-import type { Logger } from "../internal/logger.js";
-import type { UUID } from "../types.js";
+} from "../llms";
 
 export type AgentTaskContext<
   Model extends LLM,

packages/core/src/agent/utils.ts

@@ -1,8 +1,6 @@
-import {
-  type JSONObject,
-  type JSONValue,
-  Settings,
-} from "@llamaindex/core/global";
+import type { Logger } from "@llamaindex/env";
+import { z } from "zod";
+import { type JSONObject, type JSONValue, Settings } from "../global";
 import type {
   BaseTool,
   ChatMessage,
@@ -14,15 +12,13 @@ import type {
   ToolCall,
   ToolCallLLMMessageOptions,
   ToolOutput,
-} from "@llamaindex/core/llms";
-import { baseToolWithCallSchema } from "@llamaindex/core/schema";
-import { z } from "zod";
-import type { Logger } from "../internal/logger.js";
+} from "../llms";
+import { baseToolWithCallSchema } from "../schema";
 import {
   isAsyncIterable,
   prettifyError,
   stringifyJSONToMessageContent,
-} from "../internal/utils.js";
+} from "../utils";
 import type { AgentParamsBase } from "./base.js";
 import type { TaskHandler } from "./types.js";
 

packages/core/src/chat-engine/index.ts

@@ -0,0 +1,36 @@
+import type { ChatMessage, MessageContent } from "../llms";
+import type { BaseMemory } from "../memory";
+import { EngineResponse } from "../schema";
+
+export interface BaseChatEngineParams<
+  AdditionalMessageOptions extends object = object,
+> {
+  message: MessageContent;
+  /**
+   * Optional chat history if you want to customize the chat history.
+   */
+  chatHistory?:
+    | ChatMessage<AdditionalMessageOptions>[]
+    | BaseMemory<AdditionalMessageOptions>;
+}
+
+export interface StreamingChatEngineParams<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatEngineParams<AdditionalMessageOptions> {
+  stream: true;
+}
+
+export interface NonStreamingChatEngineParams<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatEngineParams<AdditionalMessageOptions> {
+  stream?: false;
+}
+
+export abstract class BaseChatEngine {
+  abstract chat(params: NonStreamingChatEngineParams): Promise<EngineResponse>;
+  abstract chat(
+    params: StreamingChatEngineParams,
+  ): Promise<AsyncIterable<EngineResponse>>;
+
+  abstract chatHistory: ChatMessage[] | Promise<ChatMessage[]>;
+}

packages/core/src/data-structs/data-structs.ts

@@ -0,0 +1,67 @@
+import { randomUUID } from "@llamaindex/env";
+import type { UUID } from "../global";
+import { IndexStructType } from "./struct-type";
+
+export abstract class IndexStruct {
+  indexId: string;
+  summary: string | undefined;
+
+  constructor(
+    indexId: UUID = randomUUID(),
+    summary: string | undefined = undefined,
+  ) {
+    this.indexId = indexId;
+    this.summary = summary;
+  }
+
+  toJson(): Record<string, unknown> {
+    return {
+      indexId: this.indexId,
+      summary: this.summary,
+    };
+  }
+
+  getSummary(): string {
+    if (this.summary === undefined) {
+      throw new Error("summary field of the index struct is not set");
+    }
+    return this.summary;
+  }
+}
+
+// A table of keywords mapping keywords to text chunks.
+export class KeywordTable extends IndexStruct {
+  table: Map<string, Set<string>> = new Map();
+  type: IndexStructType = IndexStructType.KEYWORD_TABLE;
+
+  addNode(keywords: string[], nodeId: string): void {
+    keywords.forEach((keyword) => {
+      if (!this.table.has(keyword)) {
+        this.table.set(keyword, new Set());
+      }
+      this.table.get(keyword)!.add(nodeId);
+    });
+  }
+
+  deleteNode(keywords: string[], nodeId: string) {
+    keywords.forEach((keyword) => {
+      if (this.table.has(keyword)) {
+        this.table.get(keyword)!.delete(nodeId);
+      }
+    });
+  }
+
+  toJson(): Record<string, unknown> {
+    return {
+      ...super.toJson(),
+      table: Array.from(this.table.entries()).reduce(
+        (acc, [keyword, nodeIds]) => {
+          acc[keyword] = Array.from(nodeIds);
+          return acc;
+        },
+        {} as Record<string, string[]>,
+      ),
+      type: this.type,
+    };
+  }
+}

packages/core/src/data-structs/index.ts

@@ -0,0 +1,2 @@
+export { IndexStruct, KeywordTable } from "./data-structs";
+export { IndexStructType } from "./struct-type";

packages/core/src/data-structs/struct-type.ts

@@ -0,0 +1,39 @@
+export const IndexStructType = {
+  NODE: "node",
+  TREE: "tree",
+  LIST: "list",
+  KEYWORD_TABLE: "keyword_table",
+  DICT: "dict",
+  SIMPLE_DICT: "simple_dict",
+  WEAVIATE: "weaviate",
+  PINECONE: "pinecone",
+  QDRANT: "qdrant",
+  LANCEDB: "lancedb",
+  MILVUS: "milvus",
+  CHROMA: "chroma",
+  MYSCALE: "myscale",
+  CLICKHOUSE: "clickhouse",
+  VECTOR_STORE: "vector_store",
+  OPENSEARCH: "opensearch",
+  DASHVECTOR: "dashvector",
+  CHATGPT_RETRIEVAL_PLUGIN: "chatgpt_retrieval_plugin",
+  DEEPLAKE: "deeplake",
+  EPSILLA: "epsilla",
+  MULTIMODAL_VECTOR_STORE: "multimodal",
+  SQL: "sql",
+  KG: "kg",
+  SIMPLE_KG: "simple_kg",
+  SIMPLE_LPG: "simple_lpg",
+  NEBULAGRAPH: "nebulagraph",
+  FALKORDB: "falkordb",
+  EMPTY: "empty",
+  COMPOSITE: "composite",
+  PANDAS: "pandas",
+  DOCUMENT_SUMMARY: "document_summary",
+  VECTARA: "vectara",
+  ZILLIZ_CLOUD_PIPELINE: "zilliz_cloud_pipeline",
+  POSTGRESML: "postgresml",
+} as const;
+
+export type IndexStructType =
+  (typeof IndexStructType)[keyof typeof IndexStructType];

packages/core/src/decorator/event-caller.ts

@@ -0,0 +1,48 @@
+import { AsyncLocalStorage } from "@llamaindex/env";
+import { withEventCaller } from "../global";
+import { isAsyncIterable, isIterable } from "../utils";
+
+export function wrapEventCaller<This, Result, Args extends unknown[]>(
+  originalMethod: (this: This, ...args: Args) => Result,
+  context: ClassMethodDecoratorContext<object>,
+) {
+  const name = context.name;
+  context.addInitializer(function () {
+    // @ts-expect-error
+    const fn = this[name].bind(this);
+    // @ts-expect-error
+    this[name] = (...args: unknown[]) => {
+      return withEventCaller(this, () => fn(...args));
+    };
+  });
+  return function (this: This, ...args: Args): Result {
+    const result = originalMethod.call(this, ...args);
+    // patch for iterators because AsyncLocalStorage doesn't work with them
+    if (isAsyncIterable(result)) {
+      const iter = result[Symbol.asyncIterator]();
+      const snapshot = AsyncLocalStorage.snapshot();
+      return (async function* asyncGeneratorWrapper() {
+        while (true) {
+          const { value, done } = await snapshot(() => iter.next());
+          if (done) {
+            break;
+          }
+          yield value;
+        }
+      })() as Result;
+    } else if (isIterable(result)) {
+      const iter = result[Symbol.iterator]();
+      const snapshot = AsyncLocalStorage.snapshot();
+      return (function* generatorWrapper() {
+        while (true) {
+          const { value, done } = snapshot(() => iter.next());
+          if (done) {
+            break;
+          }
+          yield value;
+        }
+      })() as Result;
+    }
+    return result;
+  };
+}

packages/core/src/decorator/index.ts

@@ -1,67 +1,3 @@
-import { getEnv } from "@llamaindex/env";
-import { Settings } from "../global";
-import type { BaseNode } from "../schema/node";
-
-const emitOnce = false;
-
-export function chunkSizeCheck<
-  This extends { id_: string },
-  Args extends any[],
-  Return,
->(
-  contentGetter: (this: This, ...args: Args) => string,
-  _context: ClassMethodDecoratorContext<
-    This,
-    (this: This, ...args: Args) => Return
-  >,
-) {
-  return function (this: This, ...args: Args) {
-    const content = contentGetter.call(this, ...args);
-    const chunkSize = Settings.chunkSize;
-    const enableChunkSizeCheck = getEnv("ENABLE_CHUNK_SIZE_CHECK") === "true";
-    if (
-      enableChunkSizeCheck &&
-      chunkSize !== undefined &&
-      content.length > chunkSize
-    ) {
-      console.warn(
-        `Node (${this.id_}) is larger than chunk size: ${content.length} > ${chunkSize}`,
-      );
-      if (!emitOnce) {
-        console.warn(
-          "Will truncate the content if it is larger than chunk size",
-        );
-        console.warn("If you want to disable this behavior:");
-        console.warn("  1. Set Settings.chunkSize = undefined");
-        console.warn("  2. Set Settings.chunkSize to a larger value");
-        console.warn(
-          "  3. Change the way of splitting content into smaller chunks",
-        );
-      }
-      return content.slice(0, chunkSize);
-    }
-    return content;
-  };
-}
-
-export function lazyInitHash(
-  value: ClassAccessorDecoratorTarget<BaseNode, string>,
-  _context: ClassAccessorDecoratorContext,
-): ClassAccessorDecoratorResult<BaseNode, string> {
-  return {
-    get() {
-      const oldValue = value.get.call(this);
-      if (oldValue === "") {
-        const hash = this.generateHash();
-        value.set.call(this, hash);
-      }
-      return value.get.call(this);
-    },
-    set(newValue: string) {
-      value.set.call(this, newValue);
-    },
-    init(value: string): string {
-      return value;
-    },
-  };
-}
+export { wrapEventCaller } from "./event-caller";
+export { lazyInitHash } from "./lazy-init-hash";
+export { wrapLLMEvent } from "./wrap-llm-event";

packages/core/src/decorator/lazy-init-hash.ts

@@ -0,0 +1,23 @@
+import type { BaseNode } from "../schema";
+
+export function lazyInitHash(
+  value: ClassAccessorDecoratorTarget<BaseNode, string>,
+  _context: ClassAccessorDecoratorContext,
+): ClassAccessorDecoratorResult<BaseNode, string> {
+  return {
+    get() {
+      const oldValue = value.get.call(this);
+      if (oldValue === "") {
+        const hash = this.generateHash();
+        value.set.call(this, hash);
+      }
+      return value.get.call(this);
+    },
+    set(newValue: string) {
+      value.set.call(this, newValue);
+    },
+    init(value: string): string {
+      return value;
+    },
+  };
+}

packages/core/src/global/index.ts

@@ -9,4 +9,9 @@ export type {
   LLMToolResultEvent,
   LlamaIndexEventMaps,
 } from "./settings/callback-manager";
-export type { JSONArray, JSONObject, JSONValue } from "./type";
+export {
+  EventCaller,
+  getEventCaller,
+  withEventCaller,
+} from "./settings/event-caller";
+export type { JSONArray, JSONObject, JSONValue, UUID } from "./type";

packages/core/src/global/settings.ts

@@ -1,4 +1,4 @@
-import type { Tokenizer } from "@llamaindex/env";
+import { getEnv, type Tokenizer } from "@llamaindex/env";
 import type { LLM } from "../llms";
 import {
   type CallbackManager,
@@ -61,4 +61,16 @@ export const Settings = {
   ): Result {
     return withCallbackManager(callbackManager, fn);
   },
+
+  get debug() {
+    let debug = getEnv("DEBUG");
+    if (typeof window !== "undefined") {
+      debug ||= window.localStorage.debug;
+    }
+    return (
+      (Boolean(debug) && debug?.includes("llamaindex")) ||
+      debug === "*" ||
+      debug === "true"
+    );
+  },
 };

packages/core/src/global/settings/callback-manager.ts

@@ -1,4 +1,5 @@
 import { AsyncLocalStorage, CustomEvent } from "@llamaindex/env";
+import type { AgentEndEvent, AgentStartEvent } from "../../agent";
 import type {
   ChatMessage,
   ChatResponse,
@@ -11,9 +12,10 @@ import type {
   SynthesizeEndEvent,
   SynthesizeStartEvent,
 } from "../../response-synthesizers";
+import type { RetrieveEndEvent, RetrieveStartEvent } from "../../retriever";
 import { TextNode } from "../../schema";
-import { EventCaller, getEventCaller } from "../../utils";
 import type { UUID } from "../type";
+import { EventCaller, getEventCaller } from "./event-caller";
 
 export type LLMStartEvent = {
   id: UUID;
@@ -69,6 +71,10 @@ export interface LlamaIndexEventMaps {
   "query-end": QueryEndEvent;
   "synthesize-start": SynthesizeStartEvent;
   "synthesize-end": SynthesizeEndEvent;
+  "retrieve-start": RetrieveStartEvent;
+  "retrieve-end": RetrieveEndEvent;
+  "agent-start": AgentStartEvent;
+  "agent-end": AgentEndEvent;
 }
 
 export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {

packages/core/src/global/settings/event-caller.ts

@@ -0,0 +1,55 @@
+import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
+
+const eventReasonAsyncLocalStorage = new AsyncLocalStorage<EventCaller>();
+
+/**
+ * EventCaller is used to track the caller of an event.
+ */
+export class EventCaller {
+  public readonly id = randomUUID();
+
+  private constructor(
+    public readonly caller: unknown,
+    public readonly parent: EventCaller | null,
+  ) {}
+
+  #computedCallers: unknown[] | null = null;
+
+  public get computedCallers(): unknown[] {
+    if (this.#computedCallers != null) {
+      return this.#computedCallers;
+    }
+    const callers = [this.caller];
+    let parent = this.parent;
+    while (parent != null) {
+      callers.push(parent.caller);
+      parent = parent.parent;
+    }
+    this.#computedCallers = callers;
+    return callers;
+  }
+
+  public static create(
+    caller: unknown,
+    parent: EventCaller | null,
+  ): EventCaller {
+    return new EventCaller(caller, parent);
+  }
+}
+
+export function getEventCaller(): EventCaller | null {
+  return eventReasonAsyncLocalStorage.getStore() ?? null;
+}
+
+/**
+ * @param caller who is calling this function, pass in `this` if it's a class method
+ * @param fn
+ */
+export function withEventCaller<T>(caller: unknown, fn: () => T) {
+  // create a chain of event callers
+  const parentCaller = getEventCaller();
+  return eventReasonAsyncLocalStorage.run(
+    EventCaller.create(caller, parentCaller),
+    fn,
+  );
+}

packages/core/src/indices/prompt-helper.ts

@@ -8,18 +8,16 @@ import {
   Settings,
 } from "../global";
 import type { LLMMetadata } from "../llms";
-import { SentenceSplitter } from "../node-parser";
-import type { PromptTemplate } from "../prompts";
+import { TextSplitter, TokenTextSplitter, truncateText } from "../node-parser";
+import { BasePromptTemplate, PromptTemplate } from "../prompts";
 
 /**
  * Get the empty prompt text given a prompt.
  */
-function getEmptyPromptTxt(prompt: PromptTemplate) {
-  return prompt.format({
-    ...Object.fromEntries(
-      [...prompt.templateVars.keys()].map((key) => [key, ""]),
-    ),
-  });
+function getEmptyPromptTxt(prompt: PromptTemplate): string {
+  return prompt.format(
+    Object.fromEntries([...prompt.templateVars.keys()].map((key) => [key, ""])),
+  );
 }
 
 /**
@@ -35,24 +33,24 @@ export function getBiggestPrompt(prompts: PromptTemplate[]): PromptTemplate {
 }
 
 export type PromptHelperOptions = {
-  contextWindow?: number;
-  numOutput?: number;
-  chunkOverlapRatio?: number;
-  chunkSizeLimit?: number;
-  tokenizer?: Tokenizer;
-  separator?: string;
+  contextWindow?: number | undefined;
+  numOutput?: number | undefined;
+  chunkOverlapRatio?: number | undefined;
+  chunkSizeLimit?: number | undefined;
+  tokenizer?: Tokenizer | undefined;
+  separator?: string | undefined;
 };
 
 /**
  * A collection of helper functions for working with prompts.
  */
 export class PromptHelper {
-  contextWindow = DEFAULT_CONTEXT_WINDOW;
-  numOutput = DEFAULT_NUM_OUTPUTS;
-  chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO;
+  contextWindow: number;
+  numOutput: number;
+  chunkOverlapRatio: number;
   chunkSizeLimit: number | undefined;
   tokenizer: Tokenizer;
-  separator = " ";
+  separator: string;
 
   constructor(options: PromptHelperOptions = {}) {
     const {
@@ -72,68 +70,93 @@ export class PromptHelper {
   }
 
   /**
-   * Given a prompt, return the maximum size of the inputs to the prompt.
-   * @param prompt
-   * @returns
+   * Calculate the available context size based on the number of prompt tokens.
    */
-  private getAvailableContextSize(prompt: PromptTemplate) {
-    const emptyPromptText = getEmptyPromptTxt(prompt);
-    const promptTokens = this.tokenizer.encode(emptyPromptText);
-    const numPromptTokens = promptTokens.length;
-
-    return this.contextWindow - numPromptTokens - this.numOutput;
-  }
-
-  /**
-   * Find the maximum size of each chunk given a prompt.
-   */
-  private getAvailableChunkSize(
-    prompt: PromptTemplate,
-    numChunks = 1,
-    padding = 5,
-  ): number {
-    const availableContextSize = this.getAvailableContextSize(prompt);
-
-    const result = Math.floor(availableContextSize / numChunks) - padding;
-
-    if (this.chunkSizeLimit) {
-      return Math.min(this.chunkSizeLimit, result);
-    } else {
-      return result;
+  #getAvailableContextSize(numPromptTokens: number): number {
+    const contextSizeTokens =
+      this.contextWindow - numPromptTokens - this.numOutput;
+    if (contextSizeTokens < 0) {
+      throw new Error(
+        `Calculated available context size ${contextSizeTokens} is not non-negative.`,
+      );
     }
+    return contextSizeTokens;
   }
 
   /**
-   * Creates a text splitter with the correct chunk sizes and overlaps given a prompt.
+   * Calculate the available chunk size based on the prompt and other parameters.
+   */
+  #getAvailableChunkSize<Template extends BasePromptTemplate>(
+    prompt: Template,
+    numChunks: number = 1,
+    padding: number = 5,
+  ): number {
+    let numPromptTokens = 0;
+
+    if (prompt instanceof PromptTemplate) {
+      numPromptTokens = this.tokenizer.encode(getEmptyPromptTxt(prompt)).length;
+    }
+
+    const availableContextSize = this.#getAvailableContextSize(numPromptTokens);
+    let result = Math.floor(availableContextSize / numChunks) - padding;
+
+    if (this.chunkSizeLimit !== undefined) {
+      result = Math.min(this.chunkSizeLimit, result);
+    }
+
+    return result;
+  }
+
+  /**
+   * Creates a text splitter configured to maximally pack the available context window.
    */
   getTextSplitterGivenPrompt(
-    prompt: PromptTemplate,
-    numChunks = 1,
-    padding = DEFAULT_PADDING,
-  ) {
-    const chunkSize = this.getAvailableChunkSize(prompt, numChunks, padding);
-    if (chunkSize === 0) {
-      throw new Error("Got 0 as available chunk size");
+    prompt: BasePromptTemplate,
+    numChunks: number = 1,
+    padding: number = DEFAULT_PADDING,
+  ): TextSplitter {
+    const chunkSize = this.#getAvailableChunkSize(prompt, numChunks, padding);
+    if (chunkSize <= 0) {
+      throw new TypeError(`Chunk size ${chunkSize} is not positive.`);
     }
-    const chunkOverlap = this.chunkOverlapRatio * chunkSize;
-    return new SentenceSplitter({
+    const chunkOverlap = Math.floor(this.chunkOverlapRatio * chunkSize);
+    return new TokenTextSplitter({
+      separator: this.separator,
       chunkSize,
       chunkOverlap,
-      separator: this.separator,
       tokenizer: this.tokenizer,
     });
   }
 
   /**
-   * Repack resplits the strings based on the optimal text splitter.
+   * Truncate text chunks to fit within the available context window.
+   */
+  truncate(
+    prompt: BasePromptTemplate,
+    textChunks: string[],
+    padding: number = DEFAULT_PADDING,
+  ): string[] {
+    const textSplitter = this.getTextSplitterGivenPrompt(
+      prompt,
+      textChunks.length,
+      padding,
+    );
+    return textChunks.map((chunk) => truncateText(chunk, textSplitter));
+  }
+
+  /**
+   * Repack text chunks to better utilize the available context window.
    */
   repack(
-    prompt: PromptTemplate,
+    prompt: BasePromptTemplate,
     textChunks: string[],
-    padding = DEFAULT_PADDING,
-  ) {
+    padding: number = DEFAULT_PADDING,
+  ): string[] {
     const textSplitter = this.getTextSplitterGivenPrompt(prompt, 1, padding);
-    const combinedStr = textChunks.join("\n\n");
+    const combinedStr = textChunks
+      .map((c) => c.trim())
+      .filter((c) => c.length > 0)
+      .join("\n\n");
     return textSplitter.splitText(combinedStr);
   }
 
@@ -154,7 +177,8 @@ export class PromptHelper {
     } = options ?? {};
     return new PromptHelper({
       contextWindow: metadata.contextWindow,
-      numOutput: metadata.maxTokens ?? DEFAULT_NUM_OUTPUTS,
+      // fixme: numOutput is not in LLMMetadata
+      numOutput: DEFAULT_NUM_OUTPUTS,
       chunkOverlapRatio,
       chunkSizeLimit,
       tokenizer,

packages/core/src/node-parser/index.ts

@@ -13,6 +13,7 @@ export { MetadataAwareTextSplitter, NodeParser, TextSplitter } from "./base";
 export { MarkdownNodeParser } from "./markdown";
 export { SentenceSplitter } from "./sentence-splitter";
 export { SentenceWindowNodeParser } from "./sentence-window";
+export { TokenTextSplitter } from "./token-text-splitter";
 export type { SplitterParams } from "./type";
 export {
   splitByChar,
@@ -20,5 +21,6 @@ export {
   splitByRegex,
   splitBySentenceTokenizer,
   splitBySep,
+  truncateText,
 } from "./utils";
 export type { TextSplitterFn } from "./utils";

packages/core/src/node-parser/token-text-splitter.ts

@@ -0,0 +1,206 @@
+import type { Tokenizer } from "@llamaindex/env";
+import { z } from "zod";
+import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, Settings } from "../global";
+import { MetadataAwareTextSplitter } from "./base";
+import type { SplitterParams } from "./type";
+import { splitByChar, splitBySep } from "./utils";
+
+const DEFAULT_METADATA_FORMAT_LEN = 2;
+
+const tokenTextSplitterSchema = z.object({
+  chunkSize: z.number().positive().default(DEFAULT_CHUNK_SIZE),
+  chunkOverlap: z.number().nonnegative().default(DEFAULT_CHUNK_OVERLAP),
+  separator: z.string().default(" "),
+  backupSeparators: z.array(z.string()).default(["\n"]),
+});
+
+export class TokenTextSplitter extends MetadataAwareTextSplitter {
+  chunkSize: number = DEFAULT_CHUNK_SIZE;
+  chunkOverlap: number = DEFAULT_CHUNK_OVERLAP;
+  separator: string = " ";
+  backupSeparators: string[] = ["\n"];
+  #tokenizer: Tokenizer;
+  #splitFns: Array<(text: string) => string[]> = [];
+
+  constructor(
+    params?: SplitterParams & Partial<z.infer<typeof tokenTextSplitterSchema>>,
+  ) {
+    super();
+
+    if (params) {
+      const parsedParams = tokenTextSplitterSchema.parse(params);
+      this.chunkSize = parsedParams.chunkSize;
+      this.chunkOverlap = parsedParams.chunkOverlap;
+      this.separator = parsedParams.separator;
+      this.backupSeparators = parsedParams.backupSeparators;
+    }
+
+    if (this.chunkOverlap > this.chunkSize) {
+      throw new Error(
+        `Got a larger chunk overlap (${this.chunkOverlap}) than chunk size (${this.chunkSize}), should be smaller.`,
+      );
+    }
+
+    this.#tokenizer = params?.tokenizer ?? Settings.tokenizer;
+
+    const allSeparators = [this.separator, ...this.backupSeparators];
+    this.#splitFns = allSeparators.map((sep) => splitBySep(sep));
+    this.#splitFns.push(splitByChar());
+  }
+
+  /**
+   * Split text into chunks, reserving space required for metadata string.
+   * @param text The text to split.
+   * @param metadata The metadata string.
+   * @returns An array of text chunks.
+   */
+  splitTextMetadataAware(text: string, metadata: string): string[] {
+    const metadataLength =
+      this.tokenSize(metadata) + DEFAULT_METADATA_FORMAT_LEN;
+    const effectiveChunkSize = this.chunkSize - metadataLength;
+
+    if (effectiveChunkSize <= 0) {
+      throw new Error(
+        `Metadata length (${metadataLength}) is longer than chunk size (${this.chunkSize}). ` +
+          `Consider increasing the chunk size or decreasing the size of your metadata to avoid this.`,
+      );
+    } else if (effectiveChunkSize < 50) {
+      console.warn(
+        `Metadata length (${metadataLength}) is close to chunk size (${this.chunkSize}). ` +
+          `Resulting chunks are less than 50 tokens. Consider increasing the chunk size or decreasing the size of your metadata to avoid this.`,
+      );
+    }
+
+    return this._splitText(text, effectiveChunkSize);
+  }
+
+  /**
+   * Split text into chunks.
+   * @param text The text to split.
+   * @returns An array of text chunks.
+   */
+  splitText(text: string): string[] {
+    return this._splitText(text, this.chunkSize);
+  }
+
+  /**
+   * Internal method to split text into chunks up to a specified size.
+   * @param text The text to split.
+   * @param chunkSize The maximum size of each chunk.
+   * @returns An array of text chunks.
+   */
+  private _splitText(text: string, chunkSize: number): string[] {
+    if (text === "") return [text];
+
+    // Dispatch chunking start event
+    Settings.callbackManager.dispatchEvent("chunking-start", { text: [text] });
+
+    const splits = this._split(text, chunkSize);
+    const chunks = this._merge(splits, chunkSize);
+
+    Settings.callbackManager.dispatchEvent("chunking-end", { chunks });
+
+    return chunks;
+  }
+
+  /**
+   * Break text into splits that are smaller than the chunk size.
+   * @param text The text to split.
+   * @param chunkSize The maximum size of each split.
+   * @returns An array of text splits.
+   */
+  private _split(text: string, chunkSize: number): string[] {
+    if (this.tokenSize(text) <= chunkSize) {
+      return [text];
+    }
+
+    for (const splitFn of this.#splitFns) {
+      const splits = splitFn(text);
+      if (splits.length > 1) {
+        const newSplits: string[] = [];
+        for (const split of splits) {
+          const splitLen = this.tokenSize(split);
+          if (splitLen <= chunkSize) {
+            newSplits.push(split);
+          } else {
+            newSplits.push(...this._split(split, chunkSize));
+          }
+        }
+        return newSplits;
+      }
+    }
+
+    return [text];
+  }
+
+  /**
+   * Merge splits into chunks with overlap.
+   * @param splits The array of text splits.
+   * @param chunkSize The maximum size of each chunk.
+   * @returns An array of merged text chunks.
+   */
+  private _merge(splits: string[], chunkSize: number): string[] {
+    const chunks: string[] = [];
+    let currentChunk: string[] = [];
+    let currentLength = 0;
+
+    for (const split of splits) {
+      const splitLength = this.tokenSize(split);
+
+      if (splitLength > chunkSize) {
+        console.warn(
+          `Got a split of size ${splitLength}, larger than chunk size ${chunkSize}.`,
+        );
+      }
+
+      if (currentLength + splitLength > chunkSize) {
+        const chunk = currentChunk.join("").trim();
+        if (chunk) {
+          chunks.push(chunk);
+        }
+
+        currentChunk = [];
+        currentLength = 0;
+
+        const overlapTokens = this.chunkOverlap;
+        const overlapSplits: string[] = [];
+
+        let overlapLength = 0;
+        while (
+          overlapSplits.length < splits.length &&
+          overlapLength < overlapTokens
+        ) {
+          const overlapSplit = currentChunk.shift();
+          if (!overlapSplit) break;
+          overlapSplits.push(overlapSplit);
+          overlapLength += this.tokenSize(overlapSplit);
+        }
+
+        for (const overlapSplit of overlapSplits.reverse()) {
+          currentChunk.push(overlapSplit);
+          currentLength += this.tokenSize(overlapSplit);
+          if (currentLength >= overlapTokens) break;
+        }
+      }
+
+      currentChunk.push(split);
+      currentLength += splitLength;
+    }
+
+    const finalChunk = currentChunk.join("").trim();
+    if (finalChunk) {
+      chunks.push(finalChunk);
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Calculate the number of tokens in the text using the tokenizer.
+   * @param text The text to tokenize.
+   * @returns The number of tokens.
+   */
+  private tokenSize(text: string): number {
+    return this.#tokenizer.encode(text).length;
+  }
+}

packages/core/src/node-parser/utils.ts

@@ -3,7 +3,10 @@ import SentenceTokenizer from "./sentence_tokenizer";
 
 export type TextSplitterFn = (text: string) => string[];
 
-const truncateText = (text: string, textSplitter: TextSplitter): string => {
+export const truncateText = (
+  text: string,
+  textSplitter: TextSplitter,
+): string => {
   const chunks = textSplitter.splitText(text);
   return chunks[0] ?? text;
 };

packages/core/src/objects/index.ts

@@ -0,0 +1,78 @@
+/**
+ * @todo refactor this module, most of the part is broken
+ *  reference
+ *    - https://github.com/run-llama/LlamaIndexTS/pull/531
+ *    - https://github.com/run-llama/LlamaIndexTS/pull/416
+ */
+import type { MessageContent } from "../llms";
+import { BaseRetriever } from "../retriever";
+import { BaseNode, TextNode } from "../schema";
+import { extractText } from "../utils";
+
+// Assuming that necessary interfaces and classes (like OT, TextNode, BaseNode, etc.) are defined elsewhere
+// Import statements (e.g., for TextNode, BaseNode) should be added based on your project's structure
+export abstract class BaseObjectNodeMapping {
+  // TypeScript doesn't support Python's classmethod directly, but we can use static methods as an alternative
+  abstract fromObjects<OT>(objs: OT[], ...args: any[]): BaseObjectNodeMapping;
+
+  // Abstract methods in TypeScript
+  abstract objNodeMapping(): Record<any, any>;
+  abstract toNode(obj: any): TextNode;
+
+  // Concrete methods can be defined as usual
+  validateObject(obj: any): void {}
+
+  // Implementing the add object logic
+  addObj(obj: any): void {
+    this.validateObject(obj);
+    this._addObj(obj);
+  }
+
+  // Abstract method for internal add object logic
+  abstract _addObj(obj: any): void;
+
+  // Implementing toNodes method
+  toNodes(objs: any[]): TextNode[] {
+    return objs.map((obj) => this.toNode(obj));
+  }
+
+  // Abstract method for internal from node logic
+  abstract _fromNode(node: BaseNode): any;
+
+  // Implementing fromNode method
+  fromNode(node: BaseNode): any {
+    const obj = this._fromNode(node);
+    this.validateObject(obj);
+    return obj;
+  }
+
+  // Abstract methods for persistence
+  abstract persist(persistDir: string, objNodeMappingFilename: string): void;
+}
+
+export class ObjectRetriever<T = unknown> {
+  _retriever: BaseRetriever;
+  _objectNodeMapping: BaseObjectNodeMapping;
+
+  constructor(
+    retriever: BaseRetriever,
+    objectNodeMapping: BaseObjectNodeMapping,
+  ) {
+    this._retriever = retriever;
+    this._objectNodeMapping = objectNodeMapping;
+  }
+
+  // In TypeScript, getters are defined like this.
+  get retriever(): BaseRetriever {
+    return this._retriever;
+  }
+
+  // Translating the retrieve method
+  async retrieve(strOrQueryBundle: MessageContent): Promise<T[]> {
+    const nodes = await this.retriever.retrieve({
+      query: extractText(strOrQueryBundle),
+    });
+    const objs = nodes.map((n) => this._objectNodeMapping.fromNode(n.node));
+    return objs;
+  }
+}

packages/core/src/prompts/index.ts

@@ -12,11 +12,15 @@ export {
   defaultCondenseQuestionPrompt,
   defaultContextSystemPrompt,
   defaultKeywordExtractPrompt,
+  defaultNodeTextTemplate,
   defaultQueryKeywordExtractPrompt,
+  defaultQuestionExtractPrompt,
   defaultRefinePrompt,
   defaultSubQuestionPrompt,
   defaultSummaryPrompt,
   defaultTextQAPrompt,
+  defaultTitleCombinePromptTemplate,
+  defaultTitleExtractorPromptTemplate,
   defaultTreeSummarizePrompt,
 } from "./prompt";
 export type {
@@ -25,9 +29,12 @@ export type {
   ContextSystemPrompt,
   KeywordExtractPrompt,
   QueryKeywordExtractPrompt,
+  QuestionExtractPrompt,
   RefinePrompt,
   SubQuestionPrompt,
   SummaryPrompt,
   TextQAPrompt,
+  TitleCombinePrompt,
+  TitleExtractorPrompt,
   TreeSummarizePrompt,
 } from "./prompt";

packages/core/src/prompts/prompt.ts

@@ -13,8 +13,12 @@ export type CondenseQuestionPrompt = PromptTemplate<
   ["chatHistory", "question"]
 >;
 export type ContextSystemPrompt = PromptTemplate<["context"]>;
-export type KeywordExtractPrompt = PromptTemplate<["context"]>;
+export type KeywordExtractPrompt = PromptTemplate<["context", "maxKeywords"]>;
 export type QueryKeywordExtractPrompt = PromptTemplate<["question"]>;
+export type QuestionExtractPrompt = PromptTemplate<["context", "numQuestions"]>;
+export type TitleExtractorPrompt = PromptTemplate<["context"]>;
+export type TitleCombinePrompt = PromptTemplate<["context"]>;
+export type KeywordExtractorPrompt = PromptTemplate<["context", "numKeywords"]>;
 
 export const defaultTextQAPrompt: TextQAPrompt = new PromptTemplate({
   templateVars: ["context", "query"],
@@ -64,11 +68,13 @@ export const defaultRefinePrompt: RefinePrompt = new PromptTemplate({
   templateVars: ["query", "existingAnswer", "context"],
   template: `The original query is as follows: {query}
 We have provided an existing answer: {existingAnswer}
-We have the opportunity to refine the existing answer (only if needed) with some more context below.
+We have the opportunity to refine the existing answer
+(only if needed) with some more context below.
 ------------
 {context}
 ------------
-Given the new context, refine the original answer to better answer the query. If the context isn't useful, return the original answer.
+Given the new context, refine the original answer to better answer the query.
+If the context isn't useful, return the original answer.
 Refined Answer:`,
 });
 
@@ -251,3 +257,55 @@ export const defaultQueryKeywordExtractPrompt = new PromptTemplate({
 }).partialFormat({
   maxKeywords: "10",
 });
+
+export const defaultQuestionExtractPrompt = new PromptTemplate({
+  templateVars: ["numQuestions", "context"],
+  template: `(
+  "Given the contextual informations below, generate {numQuestions} questions this context can provides specific answers to which are unlikely to be found else where. Higher-level summaries of surrounding context may be provided as well. "
+  "Try using these summaries to generate better questions that this context can answer."
+  "---------------------"
+  "{context}"
+  "---------------------"
+  "Provide questions in the following format: 'QUESTIONS: <questions>'"
+)`,
+}).partialFormat({
+  numQuestions: "5",
+});
+
+export const defaultTitleExtractorPromptTemplate = new PromptTemplate({
+  templateVars: ["context"],
+  template: `{context}
+Give a title that summarizes all of the unique entities, titles or themes found in the context. 
+Title: `,
+});
+
+export const defaultTitleCombinePromptTemplate = new PromptTemplate({
+  templateVars: ["context"],
+  template: `{context} 
+Based on the above candidate titles and contents, what is the comprehensive title for this document? 
+Title: `,
+});
+
+export const defaultKeywordExtractorPromptTemplate = new PromptTemplate({
+  templateVars: ["context", "numKeywords"],
+  template: `{context}
+Give {numKeywords} unique keywords for this document. 
+Format as comma separated. 
+Keywords: `,
+}).partialFormat({
+  keywordCount: "5",
+});
+
+export const defaultNodeTextTemplate = new PromptTemplate({
+  templateVars: ["metadataStr", "content"],
+  template: `[Excerpt from document]
+{metadataStr}
+Excerpt:
+-----
+{content}
+-----
+`,
+}).partialFormat({
+  metadataStr: "",
+  content: "",
+});

packages/core/src/query-engine/base.ts

@@ -1,9 +1,9 @@
 import { randomUUID } from "@llamaindex/env";
+import { wrapEventCaller } from "../decorator";
 import { Settings } from "../global";
 import type { MessageContent } from "../llms";
 import { PromptMixin } from "../prompts";
-import { EngineResponse } from "../schema";
-import { wrapEventCaller } from "../utils";
+import { EngineResponse, type NodeWithScore } from "../schema";
 
 /**
  * @link https://docs.llamaindex.ai/en/stable/api_reference/schema/?h=querybundle#llama_index.core.schema.QueryBundle
@@ -18,6 +18,18 @@ export type QueryBundle = {
 
 export type QueryType = string | QueryBundle;
 
+export type BaseQueryParams = {
+  query: QueryType;
+};
+
+export interface StreamingQueryParams extends BaseQueryParams {
+  stream: true;
+}
+
+export interface NonStreamingQueryParams extends BaseQueryParams {
+  stream?: false;
+}
+
 export type QueryFn = (
   strOrQueryBundle: QueryType,
   stream?: boolean,
@@ -28,23 +40,26 @@ export abstract class BaseQueryEngine extends PromptMixin {
     super();
   }
 
-  query(
-    strOrQueryBundle: QueryType,
-    stream: true,
-  ): Promise<AsyncIterable<EngineResponse>>;
-  query(strOrQueryBundle: QueryType, stream?: false): Promise<EngineResponse>;
+  async retrieve(params: QueryType): Promise<NodeWithScore[]> {
+    throw new Error(
+      "This query engine does not support retrieve, use query directly",
+    );
+  }
+
+  query(params: StreamingQueryParams): Promise<AsyncIterable<EngineResponse>>;
+  query(params: NonStreamingQueryParams): Promise<EngineResponse>;
   @wrapEventCaller
   async query(
-    strOrQueryBundle: QueryType,
-    stream = false,
+    params: StreamingQueryParams | NonStreamingQueryParams,
   ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const { stream, query } = params;
     const id = randomUUID();
     const callbackManager = Settings.callbackManager;
     callbackManager.dispatchEvent("query-start", {
       id,
-      query: strOrQueryBundle,
+      query,
     });
-    const response = await this._query(strOrQueryBundle, stream);
+    const response = await this._query(query, stream);
     callbackManager.dispatchEvent("query-end", {
       id,
       response,

packages/core/src/response-synthesizers/factory.ts

@@ -16,11 +16,12 @@ import {
   type NodeWithScore,
   TextNode,
 } from "../schema";
-import { createMessageContent, extractText, streamConverter } from "../utils";
+import { extractText, streamConverter } from "../utils";
 import {
   BaseSynthesizer,
   type BaseSynthesizerOptions,
 } from "./base-synthesizer";
+import { createMessageContent } from "./utils";
 
 const responseModeSchema = z.enum([
   "refine",
@@ -402,27 +403,27 @@ class MultiModal extends BaseSynthesizer {
   }
 }
 
-export function getResponseSynthesizer(
-  mode: ResponseMode,
+const modeToSynthesizer = {
+  compact: CompactAndRefine,
+  refine: Refine,
+  tree_summarize: TreeSummarize,
+  multi_modal: MultiModal,
+} as const;
+
+export function getResponseSynthesizer<Mode extends ResponseMode>(
+  mode: Mode,
   options: BaseSynthesizerOptions & {
     textQATemplate?: TextQAPrompt;
     refineTemplate?: RefinePrompt;
     summaryTemplate?: TreeSummarizePrompt;
     metadataMode?: MetadataMode;
   } = {},
-) {
-  switch (mode) {
-    case "compact": {
-      return new CompactAndRefine(options);
-    }
-    case "refine": {
-      return new Refine(options);
-    }
-    case "tree_summarize": {
-      return new TreeSummarize(options);
-    }
-    case "multi_modal": {
-      return new MultiModal(options);
-    }
+): InstanceType<(typeof modeToSynthesizer)[Mode]> {
+  const Synthesizer: (typeof modeToSynthesizer)[Mode] = modeToSynthesizer[mode];
+  if (!Synthesizer) {
+    throw new Error(`Invalid response mode: ${mode}`);
   }
+  return new Synthesizer(options) as InstanceType<
+    (typeof modeToSynthesizer)[Mode]
+  >;
 }

packages/core/src/response-synthesizers/index.ts

@@ -8,3 +8,4 @@ export type {
   SynthesizeQuery,
   SynthesizeStartEvent,
 } from "./type";
+export { createMessageContent } from "./utils";

packages/core/src/response-synthesizers/utils.ts

@@ -0,0 +1,72 @@
+// eslint-disable-next-line max-params
+import type { MessageContentDetail } from "../llms";
+import type { BasePromptTemplate } from "../prompts";
+import {
+  type BaseNode,
+  ImageNode,
+  MetadataMode,
+  ModalityType,
+  splitNodesByType,
+} from "../schema";
+import { imageToDataUrl } from "../utils";
+
+async function createContentPerModality(
+  prompt: BasePromptTemplate,
+  type: ModalityType,
+  nodes: BaseNode[],
+  extraParams: Record<string, string>,
+  metadataMode: MetadataMode,
+): Promise<MessageContentDetail[]> {
+  switch (type) {
+    case ModalityType.TEXT:
+      return [
+        {
+          type: "text",
+          text: prompt.format({
+            ...extraParams,
+            context: nodes.map((r) => r.getContent(metadataMode)).join("\n\n"),
+          }),
+        },
+      ];
+    case ModalityType.IMAGE:
+      const images: MessageContentDetail[] = await Promise.all(
+        (nodes as ImageNode[]).map(async (node) => {
+          return {
+            type: "image_url",
+            image_url: {
+              url: await imageToDataUrl(node.image),
+            },
+          } satisfies MessageContentDetail;
+        }),
+      );
+      return images;
+    default:
+      return [];
+  }
+}
+
+export async function createMessageContent(
+  prompt: BasePromptTemplate,
+  nodes: BaseNode[],
+  extraParams: Record<string, string> = {},
+  metadataMode: MetadataMode = MetadataMode.NONE,
+): Promise<MessageContentDetail[]> {
+  const content: MessageContentDetail[] = [];
+  const nodeMap = splitNodesByType(nodes);
+  for (const type in nodeMap) {
+    // for each retrieved modality type, create message content
+    const nodes = nodeMap[type as ModalityType];
+    if (nodes) {
+      content.push(
+        ...(await createContentPerModality(
+          prompt,
+          type as ModalityType,
+          nodes,
+          extraParams,
+          metadataMode,
+        )),
+      );
+    }
+  }
+  return content;
+}