Release 0.6.2 (#1217)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: himself65 <himself65@users.noreply.github.com>

.github/workflows/test.yml

@@ -104,6 +104,7 @@ jobs:
           - nextjs-edge-runtime
           - nextjs-node-runtime
           - waku-query-engine
+          - llama-parse-browser
     runs-on: ubuntu-latest
     name: Build LlamaIndex Example (${{ matrix.packages }})
     steps:
@@ -145,6 +146,9 @@ jobs:
       - name: Pack @llamaindex/openai
         run: pnpm pack --pack-destination ${{ runner.temp }}
         working-directory: packages/llm/openai
+      - name: Pack @llamaindex/groq
+        run: pnpm pack --pack-destination ${{ runner.temp }}
+        working-directory: packages/llm/groq
       - name: Pack @llamaindex/core
         run: pnpm pack --pack-destination ${{ runner.temp }}
         working-directory: packages/core

apps/docs/CHANGELOG.md

@@ -1,5 +1,22 @@
 # docs
 
+## 0.0.71
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+
+## 0.0.70
+
+### Patch Changes
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+
 ## 0.0.69
 
 ### Patch Changes

apps/docs/docs/modules/agent/_category_.yml

@@ -1,2 +1,2 @@
 label: "Agents"
-position: 3
+position: 10

apps/docs/docs/modules/chat_engine.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 4
+sidebar_position: 13
 ---
 
 # ChatEngine

apps/docs/docs/modules/data_index.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 4
+sidebar_position: 12
 ---
 
 # Index
@@ -8,6 +8,7 @@ An index is the basic container and organization for your data. LlamaIndex.TS su
 
 - `VectorStoreIndex` - will send the top-k `Node`s to the LLM when generating a response. The default top-k is 2.
 - `SummaryIndex` - will send every `Node` in the index to the LLM in order to generate a response
+- `KeywordTableIndex` extracts and provides keywords from `Node`s to the LLM
 
 ```typescript
 import { Document, VectorStoreIndex } from "llamaindex";

apps/docs/docs/modules/data_loaders/index.mdx

@@ -6,6 +6,19 @@ import CodeSource2 from "!raw-loader!../../../../../examples/readers/src/custom-
 
 Before you can start indexing your documents, you need to load them into memory.
 
+All "basic" data loaders can be seen below, mapped to their respective filetypes in `SimpleDirectoryReader`. More loaders are shown in the sidebar on the left.
+Additionally the following loaders exist without separate documentation:
+
+- `AssemblyAIReader` transcribes audio using [AssemblyAI](https://www.assemblyai.com/).
+  - [AudioTranscriptReader](../../api/classes/AudioTranscriptReader.md): loads entire transcript as a single document.
+  - [AudioTranscriptParagraphsReader](../../api/classes/AudioTranscriptParagraphsReader.md): creates a document per paragraph.
+  - [AudioTranscriptSentencesReader](../../api/classes/AudioTranscriptSentencesReader.md): creates a document per sentence.
+  - [AudioSubtitlesReader](../../api/classes/AudioTranscriptParagraphsReader.md): creates a document containing the subtitles of a transcript.
+- [NotionReader](../../api/classes/NotionReader.md) loads [Notion](https://www.notion.so/) pages.
+- [SimpleMongoReader](../../api/classes/SimpleMongoReader) loads data from a [MongoDB](https://www.mongodb.com/).
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
 ## SimpleDirectoryReader
 
 [![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/run-llama/LlamaIndexTS/tree/main/examples/readers?file=src/simple-directory-reader.ts&title=Simple%20Directory%20Reader)

apps/docs/docs/modules/data_stores/_category_.yml

@@ -0,0 +1,2 @@
+label: "Data Stores"
+position: 2

apps/docs/docs/modules/data_stores/chat_stores/_category_.yml

@@ -0,0 +1 @@
+label: "Chat Stores"

apps/docs/docs/modules/data_stores/chat_stores/index.md

@@ -0,0 +1,13 @@
+# Chat Stores
+
+Chat stores manage chat history by storing sequences of messages in a structured way, ensuring the order of messages is maintained for accurate conversation flow.
+
+## Available Chat Stores
+
+- [SimpleChatStore](../../../api/classes/SimpleChatStore.md): A simple in-memory chat store with support for [persisting](../index.md#local-storage) data to disk.
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [BaseChatStore](../../../api/interfaces/BaseChatStore.md)

apps/docs/docs/modules/data_stores/doc_stores/_category_.yml

@@ -0,0 +1,2 @@
+label: "Document Stores"
+position: 2

apps/docs/docs/modules/data_stores/doc_stores/index.md

@@ -0,0 +1,14 @@
+# Document Stores
+
+Document stores contain ingested document chunks, i.e. [Node](../../documents_and_nodes/index.md)s.
+
+## Available Document Stores
+
+- [SimpleDocumentStore](../../../api/classes/SimpleDocumentStore.md): A simple in-memory document store with support for [persisting](../index.md#local-storage) data to disk.
+- [PostgresDocumentStore](../../../api/classes/PostgresDocumentStore.md): A PostgreSQL document store, see [PostgreSQL Storage](../index.md#postgresql-storage).
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [BaseDocumentStore](../../../api/classes/BaseDocumentStore.md)

apps/docs/docs/modules/data_stores/index.md

@@ -1,7 +1,3 @@
----
-sidebar_position: 7
----
-
 # Storage
 
 Storage in LlamaIndex.TS works automatically once you've configured a
@@ -57,4 +53,4 @@ const index = await VectorStoreIndex.fromDocuments([document], {
 
 ## API Reference
 
-- [StorageContext](../api/interfaces/StorageContext.md)
+- [StorageContext](../../api/interfaces/StorageContext.md)

apps/docs/docs/modules/data_stores/index_stores/_category_.yml

@@ -0,0 +1,2 @@
+label: "Index Stores"
+position: 3

apps/docs/docs/modules/data_stores/index_stores/index.md

@@ -0,0 +1,14 @@
+# Index Stores
+
+Index stores are underlying storage components that contain metadata(i.e. information created when indexing) about the [index](../../data_index.md) itself.
+
+## Available Index Stores
+
+- [SimpleIndexStore](../../../api/classes/SimpleIndexStore.md): A simple in-memory index store with support for [persisting](../index.md#local-storage) data to disk.
+- [PostgresIndexStore](../../../api/classes/PostgresIndexStore.md): A PostgreSQL index store, , see [PostgreSQL Storage](../index.md#postgresql-storage).
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [BaseIndexStore](../../../api/classes/BaseIndexStore.md)

apps/docs/docs/modules/data_stores/kv_stores/_category_.yml

@@ -0,0 +1,2 @@
+label: "Key-Value Stores"
+position: 4

apps/docs/docs/modules/data_stores/kv_stores/index.md

@@ -0,0 +1,14 @@
+# Key-Value Stores
+
+Key-Value Stores represent underlying storage components used in [Document Stores](../doc_stores/index.md) and [Index Stores](../index_stores/index.md)
+
+## Available Key-Value Stores
+
+- [SimpleKVStore](../../../api/classes/SimpleKVStore.md): A simple Key-Value store with support of [persisting](../index.md#local-storage) data to disk.
+- [PostgresKVStore](../../../api/classes/PostgresKVStore.md): A PostgreSQL Key-Value store, see [PostgreSQL Storage](../index.md#postgresql-storage).
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [BaseKVStore](../../../api/classes/BaseKVStore.md)

apps/docs/docs/modules/data_stores/vector_stores/index.md

@@ -0,0 +1,22 @@
+# Vector Stores
+
+Vector stores save embedding vectors of your ingested document chunks.
+
+## Available Vector Stores
+
+Available Vector Stores are shown on the sidebar to the left. Additionally the following integrations exist without separate documentation:
+
+- [SimpleVectorStore](../../../api/classes/SimpleVectorStore.md): A simple in-memory vector store with optional [persistance](../index.md#local-storage) to disk.
+- [AstraDBVectorStore](../../../api/classes/AstraDBVectorStore.md): A cloud-native, scalable Database-as-a-Service built on Apache Cassandra, see [datastax.com](https://www.datastax.com/products/datastax-astra)
+- [ChromaVectorStore](../../../api/classes/ChromaVectorStore.md): An open-source vector database, focused on ease of use and performance, see [trychroma.com](https://www.trychroma.com/)
+- [MilvusVectorStore](../../../api/classes/MilvusVectorStore.md): An open-source, high-performance, highly scalable vector database, see [milvus.io](https://milvus.io/)
+- [MongoDBAtlasVectorSearch](../../../api/classes/MongoDBAtlasVectorSearch.md): A cloud-based vector search solution for MongoDB, see [mongodb.com](https://www.mongodb.com/products/platform/atlas-vector-search)
+- [PGVectorStore](../../../api/classes/PGVectorStore.md): An open-source vector store built on PostgreSQL, see [pgvector Github](https://github.com/pgvector/pgvector)
+- [PineconeVectorStore](../../../api/classes/PineconeVectorStore.md): A managed, cloud-native vector database, see [pinecone.io](https://www.pinecone.io/)
+- [WeaviateVectorStore](../../../api/classes/WeaviateVectorStore.md): An open-source, ai-native vector database, see [weaviate.io](https://weaviate.io/)
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [VectorStoreBase](../../../api/classes/VectorStoreBase.md)

apps/docs/docs/modules/data_stores/vector_stores/qdrant.md

@@ -1,5 +1,7 @@
 # Qdrant Vector Store
 
+[qdrant.tech](https://qdrant.tech/)
+
 To run this example, you need to have a Qdrant instance running. You can run it with Docker:
 
 ```bash
@@ -87,4 +89,4 @@ main().catch(console.error);
 
 ## API Reference
 
-- [QdrantVectorStore](../../api/classes/QdrantVectorStore.md)
+- [QdrantVectorStore](../../../api/classes/QdrantVectorStore.md)

apps/docs/docs/modules/documents_and_nodes/index.md

@@ -1,7 +1,3 @@
----
-sidebar_position: 1
----
-
 # Documents and Nodes
 
 `Document`s and `Node`s are the basic building blocks of any index. While the API for these objects is similar, `Document` objects represent entire files, while `Node`s are smaller pieces of that original document, that are suitable for an LLM and Q&A.

apps/docs/docs/modules/embeddings/_category_.yml

@@ -1,2 +1,2 @@
 label: "Embeddings"
-position: 3
+position: 6

apps/docs/docs/modules/embeddings/available_embeddings/mixedbreadai.md

@@ -7,7 +7,7 @@ To find out more about the latest features, updates, and available models, visit
 ## Table of Contents
 
 1. [Setup](#setup)
-2. [Usage with LlamaIndex](#integration-with-llamaindex)
+2. [Usage with LlamaIndex](#usage-with-llamaindex)
 3. [Embeddings with Custom Parameters](#embeddings-with-custom-parameters)
 
 ## Setup

apps/docs/docs/modules/embeddings/index.md

@@ -16,6 +16,16 @@ Settings.embedModel = new OpenAIEmbedding({
 
 For local embeddings, you can use the [HuggingFace](./available_embeddings/huggingface.md) embedding model.
 
+## Available Embeddings
+
+Most available embeddings are listed in the sidebar on the left.
+Additionally the following integrations exist without separate documentation:
+
+- [ClipEmbedding](../../api/classes/ClipEmbedding.md) using `@xenova/transformers`
+- [FireworksEmbedding](../../api/classes/FireworksEmbedding.md) see [fireworks.ai](https://fireworks.ai/)
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
 ## API Reference
 
 - [OpenAIEmbedding](../../api/classes/OpenAIEmbedding.md)

apps/docs/docs/modules/evaluation/_category_.yml

@@ -1,2 +1,2 @@
 label: "Evaluating"
-position: 3
+position: 9

apps/docs/docs/modules/ingestion_pipeline/_category_.yml

@@ -1,2 +1,2 @@
 label: "Ingestion Pipeline"
-position: 2
+position: 4

apps/docs/docs/modules/llms/_category_.yml

@@ -1,2 +1,2 @@
 label: "LLMs"
-position: 3
+position: 5

apps/docs/docs/modules/llms/available_llms/fireworks.md

@@ -1,6 +1,6 @@
 # Fireworks LLM
 
-Fireworks.ai focus on production use cases for open source LLMs, offering speed and quality.
+[Fireworks.ai](https://fireworks.ai/) focus on production use cases for open source LLMs, offering speed and quality.
 
 ## Usage
 

apps/docs/docs/modules/llms/index.md

@@ -1,7 +1,3 @@
----
-sidebar_position: 3
----
-
 # Large Language Models (LLMs)
 
 The LLM is responsible for reading text and generating natural language responses to queries. By default, LlamaIndex.TS uses `gpt-3.5-turbo`.
@@ -30,6 +26,15 @@ export AZURE_OPENAI_DEPLOYMENT="gpt-4" # or some other deployment name
 
 For local LLMs, currently we recommend the use of [Ollama](./available_llms/ollama.md) LLM.
 
+## Available LLMs
+
+Most available LLMs are listed in the sidebar on the left. Additionally the following integrations exist without separate documentation:
+
+- [HuggingFaceLLM](../../api/classes/HuggingFaceLLM.md) and [HuggingFaceInferenceAPI](../../api/classes/HuggingFaceInferenceAPI.md).
+- [ReplicateLLM](../../api/classes/ReplicateLLM.md) see [replicate.com](https://replicate.com/)
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
 ## API Reference
 
 - [OpenAI](../../api/classes/OpenAI.md)

apps/docs/docs/modules/node_parser.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 4
+sidebar_position: 11
 ---
 
 # NodeParser

apps/docs/docs/modules/node_postprocessors/index.md

@@ -107,3 +107,4 @@ const filteredNodes = processor.postprocessNodes(nodes);
 ## API Reference
 
 - [SimilarityPostprocessor](../../api/classes/SimilarityPostprocessor.md)
+- [MetadataReplacementPostProcessor](../../api/classes/MetadataReplacementPostProcessor.md)

apps/docs/docs/modules/node_postprocessors/mixedbreadiai_reranker.md

@@ -7,7 +7,7 @@ To find out more about the latest features and updates, visit the [mixedbread.ai
 ## Table of Contents
 
 1. [Setup](#setup)
-2. [Usage with LlamaIndex](#integration-with-llamaindex)
+2. [Usage with LlamaIndex](#usage-with-llamaindex)
 3. [Simple Reranking Guide](#simple-reranking-guide)
 4. [Reranking with Objects](#reranking-with-objects)
 

apps/docs/docs/modules/prompt/_category_.yml

@@ -1,2 +1,2 @@
 label: "Prompts"
-position: 0
+position: 7

apps/docs/docs/modules/prompt/index.md

@@ -73,6 +73,5 @@ const response = await queryEngine.query({
 
 ## API Reference
 
-- [TextQaPrompt](../../api/type-aliases/TextQaPrompt.md)
 - [ResponseSynthesizer](../../api/classes/ResponseSynthesizer.md)
 - [CompactAndRefine](../../api/classes/CompactAndRefine.md)

apps/docs/docs/modules/query_engines/_category_.yml

@@ -1,2 +1,2 @@
 label: "Query Engines"
-position: 2
+position: 8

apps/docs/docs/modules/response_synthesizer.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 6
+sidebar_position: 15
 ---
 
 # ResponseSynthesizer

apps/docs/docs/modules/retriever.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 5
+sidebar_position: 14
 ---
 
 # Retriever

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.69",
+  "version": "0.0.71",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
@@ -37,7 +37,7 @@
     "docusaurus-plugin-typedoc": "1.0.5",
     "typedoc": "0.26.6",
     "typedoc-plugin-markdown": "4.2.6",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   },
   "browserslist": {
     "production": [

examples/anthropic/chat_interactive.ts

@@ -1,4 +1,4 @@
-import { Anthropic, SimpleChatEngine, SimpleChatHistory } from "llamaindex";
+import { Anthropic, ChatMemoryBuffer, SimpleChatEngine } from "llamaindex";
 import { stdin as input, stdout as output } from "node:process";
 import readline from "node:readline/promises";
 
@@ -8,8 +8,8 @@ import readline from "node:readline/promises";
     model: "claude-3-opus",
   });
   // chatHistory will store all the messages in the conversation
-  const chatHistory = new SimpleChatHistory({
-    messages: [
+  const chatHistory = new ChatMemoryBuffer({
+    chatHistory: [
       {
         content: "You want to talk in rhymes.",
         role: "system",

examples/chatHistory.ts

@@ -2,10 +2,10 @@ import { stdin as input, stdout as output } from "node:process";
 import readline from "node:readline/promises";
 
 import {
+  ChatSummaryMemoryBuffer,
   OpenAI,
   Settings,
   SimpleChatEngine,
-  SummaryChatHistory,
 } from "llamaindex";
 
 if (process.env.NODE_ENV === "development") {
@@ -18,7 +18,7 @@ async function main() {
   // Set maxTokens to 75% of the context window size of 4096
   // This will trigger the summarizer once the chat history reaches 25% of the context window size (1024 tokens)
   const llm = new OpenAI({ model: "gpt-3.5-turbo", maxTokens: 4096 * 0.75 });
-  const chatHistory = new SummaryChatHistory({ llm });
+  const chatHistory = new ChatSummaryMemoryBuffer({ llm });
   const chatEngine = new SimpleChatEngine({ llm });
   const rl = readline.createInterface({ input, output });
 

examples/groq.ts

@@ -1,12 +1,23 @@
 import fs from "node:fs/promises";
 
-import { Document, Groq, Settings, VectorStoreIndex } from "llamaindex";
+import {
+  Document,
+  Groq,
+  HuggingFaceEmbedding,
+  Settings,
+  VectorStoreIndex,
+} from "llamaindex";
 
 // Update llm to use Groq
 Settings.llm = new Groq({
   apiKey: process.env.GROQ_API_KEY,
 });
 
+// Use HuggingFace for embeddings
+Settings.embedModel = new HuggingFaceEmbedding({
+  modelType: "Xenova/all-mpnet-base-v2",
+});
+
 async function main() {
   // Load essay from abramov.txt in Node
   const path = "node_modules/llamaindex/examples/abramov.txt";

examples/package.json

@@ -21,7 +21,7 @@
   "devDependencies": {
     "@types/node": "^22.5.1",
     "tsx": "^4.19.0",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   },
   "scripts": {
     "lint": "eslint ."

examples/readers/package.json

@@ -23,6 +23,6 @@
   "devDependencies": {
     "@types/node": "^22.5.1",
     "tsx": "^4.19.0",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   }
 }

package.json

@@ -2,9 +2,9 @@
   "name": "@llamaindex/monorepo",
   "private": true,
   "scripts": {
-    "build": "turbo run build --filter=\"!docs\" --filter=\"!*-test\" --filter=\"!*-example\"",
-    "build:release": "turbo run build lint test --filter=\"!docs\" --filter=\"!*-test\" --filter=\"!*-example\"",
-    "dev": "turbo run dev",
+    "build": "turbo run build",
+    "build:release": "turbo run build --filter=\"./packages/*\"",
+    "dev": "turbo run dev --filter=\"./packages/*\"",
     "format": "prettier --ignore-unknown --cache --check .",
     "format:write": "prettier --ignore-unknown --write .",
     "lint": "turbo run lint",
@@ -31,7 +31,7 @@
     "prettier": "^3.3.3",
     "prettier-plugin-organize-imports": "^4.0.0",
     "turbo": "^2.1.0",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   },
   "packageManager": "pnpm@9.5.0",
   "pnpm": {

packages/autotool/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @llamaindex/autotool
 
+## 3.0.2
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+
+## 3.0.1
+
+### Patch Changes
+
+- 1a6137b: feat: experimental support for browser
+
+  If you see bundler issue in next.js edge runtime, please bump to `next@14` latest version.
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+
 ## 3.0.0
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+  - @llamaindex/autotool@3.0.2
+
+## 0.0.10
+
+### Patch Changes
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+  - @llamaindex/autotool@3.0.1
+
 ## 0.0.9
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.9"
+  "version": "0.0.11"
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.55
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+  - @llamaindex/autotool@3.0.2
+
+## 0.1.54
+
+### Patch Changes
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+  - @llamaindex/autotool@3.0.1
+
 ## 0.1.53
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/actions.ts

@@ -5,9 +5,9 @@ import { runWithStreamableUI } from "@/context";
 import "@/tool";
 import { convertTools } from "@llamaindex/autotool";
 import { createStreamableUI } from "ai/rsc";
-import type { JSX } from "react";
+import type { ReactNode } from "react";
 
-export async function chatWithAI(message: string): Promise<JSX.Element> {
+export async function chatWithAI(message: string): Promise<ReactNode> {
   const agent = new OpenAIAgent({
     tools: convertTools("llamaindex"),
   });
@@ -25,7 +25,7 @@ export async function chatWithAI(message: string): Promise<JSX.Element> {
               uiStream.append("\n");
             },
             write: async (message) => {
-              uiStream.append(message.response.delta);
+              uiStream.append(message.response);
             },
             close: () => {
               uiStream.done();

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.53",
+  "version": "0.1.55",
   "scripts": {
     "dev": "next dev",
     "build": "next build",
@@ -32,6 +32,6 @@
     "cross-env": "^7.0.3",
     "postcss": "^8.4.41",
     "tailwindcss": "^3.4.10",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   }
 }

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "3.0.0",
+  "version": "3.0.2",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",
@@ -51,7 +51,7 @@
     "unplugin": "^1.12.2"
   },
   "peerDependencies": {
-    "llamaindex": "^0.6.0",
+    "llamaindex": "workspace:*",
     "openai": "^4",
     "typescript": "^4"
   },
@@ -72,10 +72,10 @@
     "@types/node": "^22.5.1",
     "bunchee": "5.3.2",
     "llamaindex": "workspace:*",
-    "next": "14.2.7",
+    "next": "14.2.11",
     "rollup": "^4.21.2",
     "tsx": "^4.19.0",
-    "typescript": "^5.5.4",
+    "typescript": "^5.6.2",
     "vitest": "^2.0.5",
     "webpack": "^5.94.0"
   }

packages/autotool/src/compiler.ts

@@ -9,7 +9,7 @@ import td from "typedoc";
 import type { SourceMapCompact } from "unplugin";
 import type { InfoString } from "./internal";
 
-export const isToolFile = (url: string) => /tool\.[jt]sx?$/.test(url);
+export const isToolFile = (url: string) => /\.tool\.[jt]sx?$/.test(url);
 export const isJSorTS = (url: string) => /\.m?[jt]sx?$/.test(url);
 
 async function parseRoot(entryPoint: string) {
@@ -28,7 +28,7 @@ async function parseRoot(entryPoint: string) {
   if (project) {
     return app.serializer.projectToObject(project, process.cwd());
   }
-  throw new Error("Failed to parse root");
+  throw new Error(`Failed to parse root ${entryPoint}`);
 }
 
 export async function transformAutoTool(

packages/cloud/CHANGELOG.md

@@ -1,5 +1,29 @@
 # @llamaindex/cloud
 
+## 0.2.6
+
+### Patch Changes
+
+- b42adeb: fix: get job result in llama parse reader
+- Updated dependencies [749b43a]
+  - @llamaindex/core@0.2.2
+
+## 0.2.5
+
+### Patch Changes
+
+- 85c2e19: feat: `@llamaindex/cloud` package update
+
+  - Bump to latest openapi schema
+  - Move LlamaParse class from llamaindex, this will allow you use llamaparse in more non-node.js environment
+
+- Updated dependencies [ac07e3c]
+- Updated dependencies [70ccb4a]
+- Updated dependencies [1a6137b]
+- Updated dependencies [ac07e3c]
+  - @llamaindex/core@0.2.1
+  - @llamaindex/env@0.1.11
+
 ## 0.2.4
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "type": "module",
   "license": "MIT",
   "scripts": {
@@ -26,6 +26,20 @@
         "types": "./dist/api.d.ts",
         "default": "./dist/api.js"
       }
+    },
+    "./reader": {
+      "require": {
+        "types": "./dist/reader.d.cts",
+        "default": "./dist/reader.cjs"
+      },
+      "import": {
+        "types": "./dist/reader.d.ts",
+        "default": "./dist/reader.js"
+      },
+      "default": {
+        "types": "./dist/reader.d.ts",
+        "default": "./dist/reader.js"
+      }
     }
   },
   "repository": {
@@ -36,6 +50,15 @@
   "devDependencies": {
     "@hey-api/client-fetch": "^0.2.4",
     "@hey-api/openapi-ts": "^0.53.0",
+    "@llamaindex/core": "workspace:^0.2.2",
+    "@llamaindex/env": "workspace:^0.1.11",
     "bunchee": "5.3.2"
+  },
+  "peerDependencies": {
+    "@llamaindex/core": "workspace:^0.2.2",
+    "@llamaindex/env": "workspace:^0.1.11"
+  },
+  "dependencies": {
+    "magic-bytes.js": "^1.10.0"
   }
 }

packages/cloud/src/reader.ts

@@ -1,92 +1,17 @@
+import { createClient, createConfig, type Client } from "@hey-api/client-fetch";
 import { Document, FileReader } from "@llamaindex/core/schema";
 import { fs, getEnv } from "@llamaindex/env";
 import { filetypeinfo } from "magic-bytes.js";
+import {
+  ParsingService,
+  type Body_upload_file_api_v1_parsing_upload_post,
+  type ParserLanguages,
+} from "./api";
+import { sleep } from "./utils";
+
+export type Language = ParserLanguages;
 
 export type ResultType = "text" | "markdown" | "json";
-export type Language =
-  | "abq"
-  | "ady"
-  | "af"
-  | "ang"
-  | "ar"
-  | "as"
-  | "ava"
-  | "az"
-  | "be"
-  | "bg"
-  | "bh"
-  | "bho"
-  | "bn"
-  | "bs"
-  | "ch_sim"
-  | "ch_tra"
-  | "che"
-  | "cs"
-  | "cy"
-  | "da"
-  | "dar"
-  | "de"
-  | "en"
-  | "es"
-  | "et"
-  | "fa"
-  | "fr"
-  | "ga"
-  | "gom"
-  | "hi"
-  | "hr"
-  | "hu"
-  | "id"
-  | "inh"
-  | "is"
-  | "it"
-  | "ja"
-  | "kbd"
-  | "kn"
-  | "ko"
-  | "ku"
-  | "la"
-  | "lbe"
-  | "lez"
-  | "lt"
-  | "lv"
-  | "mah"
-  | "mai"
-  | "mi"
-  | "mn"
-  | "mr"
-  | "ms"
-  | "mt"
-  | "ne"
-  | "new"
-  | "nl"
-  | "no"
-  | "oc"
-  | "pi"
-  | "pl"
-  | "pt"
-  | "ro"
-  | "ru"
-  | "rs_cyrillic"
-  | "rs_latin"
-  | "sck"
-  | "sk"
-  | "sl"
-  | "sq"
-  | "sv"
-  | "sw"
-  | "ta"
-  | "tab"
-  | "te"
-  | "th"
-  | "tjk"
-  | "tl"
-  | "tr"
-  | "ug"
-  | "uk"
-  | "ur"
-  | "uz"
-  | "vi";
 
 const SUPPORT_FILE_EXT: string[] = [
   ".pdf",
@@ -181,6 +106,15 @@ const SUPPORT_FILE_EXT: string[] = [
   ".tsv",
 ];
 
+//todo: should move into @llamaindex/env
+type WriteStream = {
+  write: (text: string) => void;
+};
+
+// Do not modify this variable or cause type errors
+// eslint-disable-next-line no-var
+var process: any;
+
 /**
  * Represents a reader for parsing files using the LlamaParse API.
  * See https://github.com/run-llama/llama_parse
@@ -188,8 +122,8 @@ const SUPPORT_FILE_EXT: string[] = [
 export class LlamaParseReader extends FileReader {
   // The API key for the LlamaParse API. Can be set as an environment variable: LLAMA_CLOUD_API_KEY
   apiKey: string;
-  // The base URL of the Llama Parsing API.
-  baseUrl: string = "https://api.cloud.llamaindex.ai/api/parsing";
+  // The base URL of the Llama Cloud Platform.
+  baseUrl: string = "https://api.cloud.llamaindex.ai";
   // The result type for the parser.
   resultType: ResultType = "text";
   // The interval in seconds to check if the parsing is done.
@@ -199,7 +133,7 @@ export class LlamaParseReader extends FileReader {
   // Whether to print the progress of the parsing.
   verbose = true;
   // The language of the text to parse.
-  language: Language = "en";
+  language: ParserLanguages[] = ["en"];
   // The parsing instruction for the parser. Backend default is an empty string.
   parsingInstruction?: string | undefined;
   // Wether to ignore diagonal text (when the text rotation in degrees is not 0, 90, 180 or 270, so not a horizontal or vertical text). Backend default is false.
@@ -237,21 +171,38 @@ export class LlamaParseReader extends FileReader {
   // The API key for the multimodal API. Can also be set as an env variable: LLAMA_CLOUD_VENDOR_MULTIMODAL_API_KEY
   vendorMultimodalApiKey?: string | undefined;
   // numWorkers is implemented in SimpleDirectoryReader
+  stdout?: WriteStream | undefined;
+
+  readonly #client: Client;
 
   constructor(
-    params: Partial<LlamaParseReader> & {
+    params: Partial<Omit<LlamaParseReader, "language" | "apiKey">> & {
+      language?: ParserLanguages | ParserLanguages[] | undefined;
       apiKey?: string | undefined;
     } = {},
   ) {
     super();
     Object.assign(this, params);
-    params.apiKey = params.apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
-    if (!params.apiKey) {
+    this.language = Array.isArray(this.language)
+      ? this.language
+      : [this.language];
+    this.stdout =
+      (params.stdout ?? typeof process !== "undefined")
+        ? process!.stdout
+        : undefined;
+    const apiKey = params.apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
+    if (!apiKey) {
       throw new Error(
         "API Key is required for LlamaParseReader. Please pass the apiKey parameter or set the LLAMA_CLOUD_API_KEY environment variable.",
       );
     }
-    this.apiKey = params.apiKey;
+    this.apiKey = apiKey;
+    if (this.baseUrl.endsWith("/")) {
+      this.baseUrl = this.baseUrl.slice(0, -"/".length);
+    }
+    if (this.baseUrl.endsWith("/api/parsing")) {
+      this.baseUrl = this.baseUrl.slice(0, -"/api/parsing".length);
+    }
 
     if (params.gpt4oMode) {
       params.gpt4oApiKey =
@@ -266,12 +217,21 @@ export class LlamaParseReader extends FileReader {
 
       this.vendorMultimodalApiKey = params.vendorMultimodalApiKey;
     }
+
+    this.#client = createClient(
+      createConfig({
+        headers: {
+          Authorization: `Bearer ${this.apiKey}`,
+        },
+        baseUrl: this.baseUrl,
+      }),
+    );
   }
 
   // Create a job for the LlamaParse API
   private async createJob(
     data: Uint8Array,
-    fileName?: string,
+    fileName: string = "unknown",
   ): Promise<string> {
     // Load data, set the mime type
     const { mime, extension } = await LlamaParseReader.getMimeType(data);
@@ -281,111 +241,123 @@ export class LlamaParseReader extends FileReader {
       console.log(`Starting load for ${name} file`);
     }
 
-    const body = new FormData();
-    body.set("file", new Blob([data], { type: mime }), fileName);
-
-    const LlamaParseBodyParams = {
+    const body = {
+      file: new File([data], fileName, { type: mime }),
       language: this.language,
       parsing_instruction: this.parsingInstruction,
-      skip_diagonal_text: this.skipDiagonalText?.toString(),
-      invalidate_cache: this.invalidateCache?.toString(),
-      do_not_cache: this.doNotCache?.toString(),
-      fast_mode: this.fastMode?.toString(),
-      do_not_unroll_columns: this.doNotUnrollColumns?.toString(),
+      skip_diagonal_text: this.skipDiagonalText,
+      invalidate_cache: this.invalidateCache,
+      do_not_cache: this.doNotCache,
+      fast_mode: this.fastMode,
+      do_not_unroll_columns: this.doNotUnrollColumns,
       page_separator: this.pageSeparator,
       page_prefix: this.pagePrefix,
       page_suffix: this.pageSuffix,
-      gpt4o_mode: this.gpt4oMode?.toString(),
+      gpt4o_mode: this.gpt4oMode,
       gpt4o_api_key: this.gpt4oApiKey,
       bounding_box: this.boundingBox,
       target_pages: this.targetPages,
-      use_vendor_multimodal_model: this.useVendorMultimodalModel?.toString(),
+      use_vendor_multimodal_model: this.useVendorMultimodalModel,
       vendor_multimodal_model_name: this.vendorMultimodalModelName,
       vendor_multimodal_api_key: this.vendorMultimodalApiKey,
-    };
+      // fixme: does these fields need to be set?
+      webhook_url: undefined,
+      take_screenshot: undefined,
+      disable_ocr: undefined,
+      disable_reconstruction: undefined,
+      input_s3_path: undefined,
+      output_s3_path_prefix: undefined,
+    } satisfies {
+      [Key in keyof Body_upload_file_api_v1_parsing_upload_post]-?:
+        | Body_upload_file_api_v1_parsing_upload_post[Key]
+        | undefined;
+    } as unknown as Body_upload_file_api_v1_parsing_upload_post;
 
-    // Filter out params with invalid values that would cause issues on the backend.
-    const filteredParams = this.filterSpecificParams(LlamaParseBodyParams, [
-      "page_separator",
-      "page_prefix",
-      "page_suffix",
-      "bounding_box",
-      "target_pages",
-    ]);
-
-    // Appends body with any defined LlamaParseBodyParams
-    Object.entries(filteredParams).forEach(([key, value]) => {
-      if (value !== undefined) {
-        body.append(key, value);
-      }
-    });
-
-    const headers = {
-      Authorization: `Bearer ${this.apiKey}`,
-    };
-
-    // Send the request, start job
-    const url = `${this.baseUrl}/upload`;
-    const response = await fetch(url, {
+    const response = await ParsingService.uploadFileApiV1ParsingUploadPost({
+      client: this.#client,
+      throwOnError: true,
       signal: AbortSignal.timeout(this.maxTimeout * 1000),
-      method: "POST",
       body,
-      headers,
     });
-    if (!response.ok) {
-      throw new Error(`Failed to parse the file: ${await response.text()}`);
-    }
-    const jsonResponse = await response.json();
-    return jsonResponse.id;
+
+    return response.data.id;
   }
 
   // Get the result of the job
-  private async getJobResult(jobId: string, resultType: string): Promise<any> {
-    const resultUrl = `${this.baseUrl}/job/${jobId}/result/${resultType}`;
-    const statusUrl = `${this.baseUrl}/job/${jobId}`;
-    const headers = { Authorization: `Bearer ${this.apiKey}` };
-
+  private async getJobResult(
+    jobId: string,
+    resultType: "text" | "json" | "markdown",
+  ): Promise<any> {
     const signal = AbortSignal.timeout(this.maxTimeout * 1000);
     let tries = 0;
     while (true) {
-      await new Promise((resolve) =>
-        setTimeout(resolve, this.checkInterval * 1000),
-      );
+      await sleep(this.checkInterval * 1000);
 
       // Check the job status. If unsuccessful response, checks if maximum timeout has been reached. If reached, throws an error
-      const statusResponse = await fetch(statusUrl, {
-        headers,
+      const result = await ParsingService.getJobApiV1ParsingJobJobIdGet({
+        client: this.#client,
+        throwOnError: true,
+        path: {
+          job_id: jobId,
+        },
         signal,
       });
-      if (!statusResponse.ok) {
-        signal.throwIfAborted();
-        if (this.verbose && tries % 10 === 0) {
-          process.stdout.write(".");
-        }
-        tries++;
-        continue;
-      }
+      const { data } = result;
 
-      // If response is succesful, check status of job. Allowed values "PENDING", "SUCCESS", "ERROR", "CANCELED"
-      const statusJson = await statusResponse.json();
-      const status = statusJson.status;
+      const status = (data as Record<string, unknown>)["status"];
       // If job has completed, return the result
       if (status === "SUCCESS") {
-        const resultResponse = await fetch(resultUrl, {
-          headers,
-          signal,
-        });
-        if (!resultResponse.ok) {
-          throw new Error(
-            `Failed to fetch result: ${await resultResponse.text()}`,
-          );
+        let result;
+        switch (resultType) {
+          case "json": {
+            result =
+              await ParsingService.getJobJsonResultApiV1ParsingJobJobIdResultJsonGet(
+                {
+                  client: this.#client,
+                  throwOnError: true,
+                  path: {
+                    job_id: jobId,
+                  },
+                  signal,
+                },
+              );
+            break;
+          }
+          case "markdown": {
+            result =
+              await ParsingService.getJobResultApiV1ParsingJobJobIdResultMarkdownGet(
+                {
+                  client: this.#client,
+                  throwOnError: true,
+                  path: {
+                    job_id: jobId,
+                  },
+                  signal,
+                },
+              );
+            break;
+          }
+          case "text": {
+            result =
+              await ParsingService.getJobTextResultApiV1ParsingJobJobIdResultTextGet(
+                {
+                  client: this.#client,
+                  throwOnError: true,
+                  path: {
+                    job_id: jobId,
+                  },
+                  signal,
+                },
+              );
+            break;
+          }
         }
-        return resultResponse.json();
+        return result.data;
         // If job is still pending, check if maximum timeout has been reached. If reached, throws an error
       } else if (status === "PENDING") {
         signal.throwIfAborted();
         if (this.verbose && tries % 10 === 0) {
-          process.stdout.write(".");
+          this.stdout?.write(".");
         }
         tries++;
       } else {
@@ -408,36 +380,34 @@ export class LlamaParseReader extends FileReader {
     fileContent: Uint8Array,
     fileName?: string,
   ): Promise<Document[]> {
-    let jobId;
-    try {
-      // Creates a job for the file
-      jobId = await this.createJob(fileContent, fileName);
-      if (this.verbose) {
-        console.log(`Started parsing the file under job id ${jobId}`);
-      }
+    return this.createJob(fileContent, fileName)
+      .then(async (jobId) => {
+        if (this.verbose) {
+          console.log(`Started parsing the file under job id ${jobId}`);
+        }
 
-      // Return results as Document objects
-      const jobResults = await this.getJobResult(jobId, this.resultType);
-      const resultText = jobResults[this.resultType];
+        // Return results as Document objects
+        const jobResults = await this.getJobResult(jobId, this.resultType);
+        const resultText = jobResults[this.resultType];
 
-      // Split the text by separator if splitByPage is true
-      if (this.splitByPage) {
-        return this.splitTextBySeparator(resultText);
-      }
+        // Split the text by separator if splitByPage is true
+        if (this.splitByPage) {
+          return this.splitTextBySeparator(resultText);
+        }
 
-      return [
-        new Document({
-          text: resultText,
-        }),
-      ];
-    } catch (e) {
-      console.error(`Error while parsing file under job id ${jobId}`, e);
-      if (this.ignoreErrors) {
-        return [];
-      } else {
-        throw e;
-      }
-    }
+        return [
+          new Document({
+            text: resultText,
+          }),
+        ];
+      })
+      .catch((error) => {
+        if (this.ignoreErrors) {
+          return [];
+        } else {
+          throw error;
+        }
+      });
   }
   /**
    * Loads data from a file and returns an array of JSON objects.
@@ -551,15 +521,20 @@ export class LlamaParseReader extends FileReader {
     imagePath: string,
     jobId: string,
   ): Promise<void> {
-    const headers = { Authorization: `Bearer ${this.apiKey}` };
-    // Construct the image URL
-    const imageUrl = `${this.baseUrl}/job/${jobId}/result/image/${imageName}`;
-    const response = await fetch(imageUrl, { headers });
-    if (!response.ok) {
-      throw new Error(`Failed to download image: ${await response.text()}`);
+    const response =
+      await ParsingService.getJobImageResultApiV1ParsingJobJobIdResultImageNameGet(
+        {
+          client: this.#client,
+          path: {
+            job_id: jobId,
+            name: imageName,
+          },
+        },
+      );
+    if (response.error) {
+      throw new Error(`Failed to download image: ${response.error.detail}`);
     }
-    // Convert the response to an ArrayBuffer and then to a Buffer
-    const arrayBuffer = await response.arrayBuffer();
+    const arrayBuffer = (await response.data) as ArrayBuffer;
     const buffer = new Uint8Array(arrayBuffer);
     // Write the image buffer to the specified imagePath
     await fs.writeFile(imagePath, buffer);

packages/cloud/src/utils.ts

@@ -0,0 +1,3 @@
+export async function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

packages/cloud/tsconfig.json

@@ -8,8 +8,17 @@
     "moduleResolution": "Bundler",
     "skipLibCheck": true,
     "strict": true,
-    "lib": ["DOM", "ESNext"]
+    "lib": ["DOM", "ESNext"],
+    "types": []
   },
   "include": ["./src"],
-  "exclude": ["node_modules"]
+  "exclude": ["node_modules"],
+  "references": [
+    {
+      "path": "../core/tsconfig.json"
+    },
+    {
+      "path": "../env/tsconfig.json"
+    }
+  ]
 }

packages/community/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @llamaindex/community
 
+## 0.0.36
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - @llamaindex/core@0.2.2
+
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [ac07e3c]
+- Updated dependencies [70ccb4a]
+- Updated dependencies [1a6137b]
+- Updated dependencies [ac07e3c]
+  - @llamaindex/core@0.2.1
+  - @llamaindex/env@0.1.11
+
 ## 0.0.34
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.34",
+  "version": "0.0.36",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @llamaindex/core
 
+## 0.2.2
+
+### Patch Changes
+
+- 749b43a: fix: clip embedding transform function
+
+## 0.2.1
+
+### Patch Changes
+
+- ac07e3c: fix: replace instanceof check with `.type` check
+- 70ccb4a: Allow arbitrary types in workflow's StartEvent and StopEvent
+- ac07e3c: fix: add `console.warn` when import dual module
+- Updated dependencies [ac07e3c]
+- Updated dependencies [1a6137b]
+- Updated dependencies [ac07e3c]
+  - @llamaindex/env@0.1.11
+
 ## 0.2.0
 
 ### Minor Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./node-parser": {
@@ -157,6 +157,34 @@
         "types": "./dist/workflow/index.d.ts",
         "default": "./dist/workflow/index.js"
       }
+    },
+    "./memory": {
+      "require": {
+        "types": "./dist/memory/index.d.cts",
+        "default": "./dist/memory/index.cjs"
+      },
+      "import": {
+        "types": "./dist/memory/index.d.ts",
+        "default": "./dist/memory/index.js"
+      },
+      "default": {
+        "types": "./dist/memory/index.d.ts",
+        "default": "./dist/memory/index.js"
+      }
+    },
+    "./storage/chat-store": {
+      "require": {
+        "types": "./dist/storage/chat-store/index.d.cts",
+        "default": "./dist/storage/chat-store/index.cjs"
+      },
+      "import": {
+        "types": "./dist/storage/chat-store/index.d.ts",
+        "default": "./dist/storage/chat-store/index.js"
+      },
+      "default": {
+        "types": "./dist/storage/chat-store/index.d.ts",
+        "default": "./dist/storage/chat-store/index.js"
+      }
     }
   },
   "files": [
@@ -172,8 +200,10 @@
     "url": "https://github.com/himself65/LlamaIndexTS.git"
   },
   "devDependencies": {
+    "@edge-runtime/vm": "^4.0.3",
     "ajv": "^8.17.1",
     "bunchee": "5.3.2",
+    "happy-dom": "^15.7.4",
     "natural": "^8.0.1",
     "python-format-js": "^1.4.3"
   },

packages/core/src/embeddings/base.ts

@@ -23,23 +23,34 @@ export abstract class BaseEmbedding extends TransformComponent {
   embedBatchSize = DEFAULT_EMBED_BATCH_SIZE;
   embedInfo?: EmbeddingInfo;
 
-  constructor() {
-    super(
-      async (
-        nodes: BaseNode[],
-        options?: BaseEmbeddingOptions,
-      ): Promise<BaseNode[]> => {
-        const texts = nodes.map((node) => node.getContent(MetadataMode.EMBED));
+  protected constructor(
+    transformFn?: (
+      nodes: BaseNode[],
+      options?: BaseEmbeddingOptions,
+    ) => Promise<BaseNode[]>,
+  ) {
+    if (transformFn) {
+      super(transformFn);
+    } else {
+      super(
+        async (
+          nodes: BaseNode[],
+          options?: BaseEmbeddingOptions,
+        ): Promise<BaseNode[]> => {
+          const texts = nodes.map((node) =>
+            node.getContent(MetadataMode.EMBED),
+          );
 
-        const embeddings = await this.getTextEmbeddingsBatch(texts, options);
+          const embeddings = await this.getTextEmbeddingsBatch(texts, options);
 
-        for (let i = 0; i < nodes.length; i++) {
-          nodes[i]!.embedding = embeddings[i];
-        }
+          for (let i = 0; i < nodes.length; i++) {
+            nodes[i]!.embedding = embeddings[i];
+          }
 
-        return nodes;
-      },
-    );
+          return nodes;
+        },
+      );
+    }
   }
 
   similarity(

packages/core/src/embeddings/index.ts

@@ -1,4 +1,5 @@
 export { BaseEmbedding, batchEmbeddings } from "./base";
 export type { BaseEmbeddingOptions, EmbeddingInfo } from "./base";
+export { MultiModalEmbedding } from "./muti-model";
 export { truncateMaxTokens } from "./tokenizer";
 export { DEFAULT_SIMILARITY_TOP_K, SimilarityType, similarity } from "./utils";

packages/core/src/embeddings/muti-model.ts

@@ -0,0 +1,81 @@
+import type { MessageContentDetail } from "../llms";
+import {
+  ImageNode,
+  MetadataMode,
+  ModalityType,
+  splitNodesByType,
+  type BaseNode,
+  type ImageType,
+} from "../schema";
+import { extractImage, extractSingleText } from "../utils";
+import {
+  BaseEmbedding,
+  batchEmbeddings,
+  type BaseEmbeddingOptions,
+} from "./base";
+
+/*
+ * Base class for Multi Modal embeddings.
+ */
+export abstract class MultiModalEmbedding extends BaseEmbedding {
+  abstract getImageEmbedding(images: ImageType): Promise<number[]>;
+
+  protected constructor() {
+    super(
+      async (
+        nodes: BaseNode[],
+        options?: BaseEmbeddingOptions,
+      ): Promise<BaseNode[]> => {
+        const nodeMap = splitNodesByType(nodes);
+        const imageNodes = nodeMap[ModalityType.IMAGE] ?? [];
+        const textNodes = nodeMap[ModalityType.TEXT] ?? [];
+
+        const embeddings = await batchEmbeddings(
+          textNodes.map((node) => node.getContent(MetadataMode.EMBED)),
+          this.getTextEmbeddings.bind(this),
+          this.embedBatchSize,
+          options,
+        );
+        for (let i = 0; i < textNodes.length; i++) {
+          textNodes[i]!.embedding = embeddings[i];
+        }
+
+        const imageEmbeddings = await batchEmbeddings(
+          imageNodes.map((n) => (n as ImageNode).image),
+          this.getImageEmbeddings.bind(this),
+          this.embedBatchSize,
+          options,
+        );
+        for (let i = 0; i < imageNodes.length; i++) {
+          imageNodes[i]!.embedding = imageEmbeddings[i];
+        }
+
+        return nodes;
+      },
+    );
+  }
+
+  /**
+   * Optionally override this method to retrieve multiple image embeddings in a single request
+   * @param images
+   */
+  async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
+    return Promise.all(
+      images.map((imgFilePath) => this.getImageEmbedding(imgFilePath)),
+    );
+  }
+
+  async getQueryEmbedding(
+    query: MessageContentDetail,
+  ): Promise<number[] | null> {
+    const image = extractImage(query);
+    if (image) {
+      return await this.getImageEmbedding(image);
+    }
+    const text = extractSingleText(query);
+    if (text) {
+      return await this.getTextEmbedding(text);
+    }
+    return null;
+  }
+}

packages/core/src/memory/base.ts

@@ -0,0 +1,62 @@
+import { Settings } from "../global";
+import type { ChatMessage, MessageContent } from "../llms";
+import { type BaseChatStore, SimpleChatStore } from "../storage/chat-store";
+import { extractText } from "../utils";
+
+export const DEFAULT_TOKEN_LIMIT_RATIO = 0.75;
+export const DEFAULT_CHAT_STORE_KEY = "chat_history";
+
+/**
+ * A ChatMemory is used to keep the state of back and forth chat messages
+ */
+export abstract class BaseMemory<
+  AdditionalMessageOptions extends object = object,
+> {
+  abstract getMessages(
+    input?: MessageContent | undefined,
+  ):
+    | ChatMessage<AdditionalMessageOptions>[]
+    | Promise<ChatMessage<AdditionalMessageOptions>[]>;
+  abstract getAllMessages():
+    | ChatMessage<AdditionalMessageOptions>[]
+    | Promise<ChatMessage<AdditionalMessageOptions>[]>;
+  abstract put(messages: ChatMessage<AdditionalMessageOptions>): void;
+  abstract reset(): void;
+
+  protected _tokenCountForMessages(messages: ChatMessage[]): number {
+    if (messages.length === 0) {
+      return 0;
+    }
+
+    const tokenizer = Settings.tokenizer;
+    const str = messages.map((m) => extractText(m.content)).join(" ");
+    return tokenizer.encode(str).length;
+  }
+}
+
+export abstract class BaseChatStoreMemory<
+  AdditionalMessageOptions extends object = object,
+> extends BaseMemory<AdditionalMessageOptions> {
+  protected constructor(
+    public chatStore: BaseChatStore<AdditionalMessageOptions> = new SimpleChatStore<AdditionalMessageOptions>(),
+    public chatStoreKey: string = DEFAULT_CHAT_STORE_KEY,
+  ) {
+    super();
+  }
+
+  getAllMessages(): ChatMessage<AdditionalMessageOptions>[] {
+    return this.chatStore.getMessages(this.chatStoreKey);
+  }
+
+  put(messages: ChatMessage<AdditionalMessageOptions>) {
+    this.chatStore.addMessage(this.chatStoreKey, messages);
+  }
+
+  set(messages: ChatMessage<AdditionalMessageOptions>[]) {
+    this.chatStore.setMessages(this.chatStoreKey, messages);
+  }
+
+  reset() {
+    this.chatStore.deleteMessages(this.chatStoreKey);
+  }
+}

packages/core/src/memory/chat-memory-buffer.ts

@@ -0,0 +1,65 @@
+import { Settings } from "../global";
+import type { ChatMessage, LLM, MessageContent } from "../llms";
+import { type BaseChatStore } from "../storage/chat-store";
+import { BaseChatStoreMemory, DEFAULT_TOKEN_LIMIT_RATIO } from "./base";
+
+type ChatMemoryBufferOptions<AdditionalMessageOptions extends object = object> =
+  {
+    tokenLimit?: number | undefined;
+    chatStore?: BaseChatStore<AdditionalMessageOptions> | undefined;
+    chatStoreKey?: string | undefined;
+    chatHistory?: ChatMessage<AdditionalMessageOptions>[] | undefined;
+    llm?: LLM<object, AdditionalMessageOptions> | undefined;
+  };
+
+export class ChatMemoryBuffer<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatStoreMemory<AdditionalMessageOptions> {
+  tokenLimit: number;
+
+  constructor(
+    options?: Partial<ChatMemoryBufferOptions<AdditionalMessageOptions>>,
+  ) {
+    super(options?.chatStore, options?.chatStoreKey);
+
+    const llm = options?.llm ?? Settings.llm;
+    const contextWindow = llm.metadata.contextWindow;
+    this.tokenLimit =
+      options?.tokenLimit ??
+      Math.ceil(contextWindow * DEFAULT_TOKEN_LIMIT_RATIO);
+
+    if (options?.chatHistory) {
+      this.chatStore.setMessages(this.chatStoreKey, options.chatHistory);
+    }
+  }
+
+  getMessages(
+    input?: MessageContent | undefined,
+    initialTokenCount: number = 0,
+  ) {
+    const messages = this.getAllMessages();
+
+    if (initialTokenCount > this.tokenLimit) {
+      throw new Error("Initial token count exceeds token limit");
+    }
+
+    let messageCount = messages.length;
+    let currentMessages = messages.slice(-messageCount);
+    let tokenCount = this._tokenCountForMessages(messages) + initialTokenCount;
+
+    while (tokenCount > this.tokenLimit && messageCount > 1) {
+      messageCount -= 1;
+      if (messages.at(-messageCount)!.role === "assistant") {
+        messageCount -= 1;
+      }
+      currentMessages = messages.slice(-messageCount);
+      tokenCount =
+        this._tokenCountForMessages(currentMessages) + initialTokenCount;
+    }
+
+    if (tokenCount > this.tokenLimit && messageCount <= 0) {
+      return [];
+    }
+    return messages.slice(-messageCount);
+  }
+}

packages/core/src/memory/index.ts

@@ -0,0 +1,3 @@
+export { BaseMemory } from "./base";
+export { ChatMemoryBuffer } from "./chat-memory-buffer";
+export { ChatSummaryMemoryBuffer } from "./summary-memory";

packages/core/src/memory/summary-memory.ts

@@ -1,73 +1,11 @@
-import type { ChatMessage, LLM, MessageType } from "@llamaindex/core/llms";
-import {
-  defaultSummaryPrompt,
-  type SummaryPrompt,
-} from "@llamaindex/core/prompts";
-import { extractText, messagesToHistory } from "@llamaindex/core/utils";
-import { tokenizers, type Tokenizer } from "@llamaindex/env";
-import { OpenAI } from "@llamaindex/openai";
+import { type Tokenizer, tokenizers } from "@llamaindex/env";
+import { Settings } from "../global";
+import type { ChatMessage, LLM, MessageType } from "../llms";
+import { defaultSummaryPrompt, type SummaryPrompt } from "../prompts";
+import { extractText, messagesToHistory } from "../utils";
+import { BaseMemory } from "./base";
 
-/**
- * A ChatHistory is used to keep the state of back and forth chat messages
- */
-export abstract class ChatHistory<
-  AdditionalMessageOptions extends object = object,
-> {
-  abstract get messages(): ChatMessage<AdditionalMessageOptions>[];
-  /**
-   * Adds a message to the chat history.
-   * @param message
-   */
-  abstract addMessage(message: ChatMessage<AdditionalMessageOptions>): void;
-
-  /**
-   * Returns the messages that should be used as input to the LLM.
-   */
-  abstract requestMessages(
-    transientMessages?: ChatMessage<AdditionalMessageOptions>[],
-  ): Promise<ChatMessage<AdditionalMessageOptions>[]>;
-
-  /**
-   * Resets the chat history so that it's empty.
-   */
-  abstract reset(): void;
-
-  /**
-   * Returns the new messages since the last call to this function (or since calling the constructor)
-   */
-  abstract newMessages(): ChatMessage<AdditionalMessageOptions>[];
-}
-
-export class SimpleChatHistory extends ChatHistory {
-  messages: ChatMessage[];
-  private messagesBefore: number;
-
-  constructor(init?: { messages?: ChatMessage[] | undefined }) {
-    super();
-    this.messages = init?.messages ?? [];
-    this.messagesBefore = this.messages.length;
-  }
-
-  addMessage(message: ChatMessage) {
-    this.messages.push(message);
-  }
-
-  async requestMessages(transientMessages?: ChatMessage[]) {
-    return [...(transientMessages ?? []), ...this.messages];
-  }
-
-  reset() {
-    this.messages = [];
-  }
-
-  newMessages() {
-    const newMessages = this.messages.slice(this.messagesBefore);
-    this.messagesBefore = this.messages.length;
-    return newMessages;
-  }
-}
-
-export class SummaryChatHistory extends ChatHistory {
+export class ChatSummaryMemoryBuffer extends BaseMemory {
   /**
    * Tokenizer function that converts text to tokens,
    *  this is used to calculate the number of tokens in a message.
@@ -77,20 +15,18 @@ export class SummaryChatHistory extends ChatHistory {
   messages: ChatMessage[];
   summaryPrompt: SummaryPrompt;
   llm: LLM;
-  private messagesBefore: number;
 
-  constructor(init?: Partial<SummaryChatHistory>) {
+  constructor(options?: Partial<ChatSummaryMemoryBuffer>) {
     super();
-    this.messages = init?.messages ?? [];
-    this.messagesBefore = this.messages.length;
-    this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
-    this.llm = init?.llm ?? new OpenAI();
+    this.messages = options?.messages ?? [];
+    this.summaryPrompt = options?.summaryPrompt ?? defaultSummaryPrompt;
+    this.llm = options?.llm ?? Settings.llm;
     if (!this.llm.metadata.maxTokens) {
       throw new Error(
         "LLM maxTokens is not set. Needed so the summarizer ensures the context window size of the LLM.",
       );
     }
-    this.tokenizer = init?.tokenizer ?? tokenizers.tokenizer();
+    this.tokenizer = options?.tokenizer ?? tokenizers.tokenizer();
     this.tokensToSummarize =
       this.llm.metadata.contextWindow - this.llm.metadata.maxTokens;
     if (this.tokensToSummarize < this.llm.metadata.contextWindow * 0.25) {
@@ -128,12 +64,8 @@ export class SummaryChatHistory extends ChatHistory {
     return { content: response.message.content, role: "memory" };
   }
 
-  addMessage(message: ChatMessage) {
-    this.messages.push(message);
-  }
-
   // Find last summary message
-  private getLastSummaryIndex(): number | null {
+  private get lastSummaryIndex(): number | null {
     const reversedMessages = this.messages.slice().reverse();
     const index = reversedMessages.findIndex(
       (message) => message.role === "memory",
@@ -145,7 +77,7 @@ export class SummaryChatHistory extends ChatHistory {
   }
 
   public getLastSummary(): ChatMessage | null {
-    const lastSummaryIndex = this.getLastSummaryIndex();
+    const lastSummaryIndex = this.lastSummaryIndex;
     return lastSummaryIndex ? this.messages[lastSummaryIndex]! : null;
   }
 
@@ -165,7 +97,7 @@ export class SummaryChatHistory extends ChatHistory {
    * If there's a memory, uses all messages after the last summary message.
    */
   private calcConversationMessages(transformSummary?: boolean): ChatMessage[] {
-    const lastSummaryIndex = this.getLastSummaryIndex();
+    const lastSummaryIndex = this.lastSummaryIndex;
     if (!lastSummaryIndex) {
       // there's no memory, so just use all non-system messages
       return this.nonSystemMessages;
@@ -182,18 +114,18 @@ export class SummaryChatHistory extends ChatHistory {
     }
   }
 
-  private calcCurrentRequestMessages(transientMessages?: ChatMessage[]) {
+  private calcCurrentRequestMessages() {
     // TODO: check order: currently, we're sending:
     // system messages first, then transient messages and then the messages that describe the conversation so far
-    return [
-      ...this.systemMessages,
-      ...(transientMessages ? transientMessages : []),
-      ...this.calcConversationMessages(true),
-    ];
+    return [...this.systemMessages, ...this.calcConversationMessages(true)];
   }
 
-  async requestMessages(transientMessages?: ChatMessage[]) {
-    const requestMessages = this.calcCurrentRequestMessages(transientMessages);
+  reset() {
+    this.messages = [];
+  }
+
+  async getMessages(): Promise<ChatMessage[]> {
+    const requestMessages = this.calcCurrentRequestMessages();
 
     // get tokens of current request messages and the transient messages
     const tokens = requestMessages.reduce(
@@ -217,27 +149,16 @@ export class SummaryChatHistory extends ChatHistory {
       // TODO: we still might have too many tokens
       // e.g. too large system messages or transient messages
       // how should we deal with that?
-      return this.calcCurrentRequestMessages(transientMessages);
+      return this.calcCurrentRequestMessages();
     }
     return requestMessages;
   }
 
-  reset() {
-    this.messages = [];
+  async getAllMessages(): Promise<ChatMessage[]> {
+    return this.getMessages();
   }
 
-  newMessages() {
-    const newMessages = this.messages.slice(this.messagesBefore);
-    this.messagesBefore = this.messages.length;
-    return newMessages;
+  put(message: ChatMessage) {
+    this.messages.push(message);
   }
 }
-
-export function getHistory(
-  chatHistory?: ChatMessage[] | ChatHistory,
-): ChatHistory {
-  if (chatHistory instanceof ChatHistory) {
-    return chatHistory;
-  }
-  return new SimpleChatHistory({ messages: chatHistory });
-}

packages/core/src/schema/node.ts

@@ -437,9 +437,16 @@ export function splitNodesByType(nodes: BaseNode[]): NodesByType {
 
   for (const node of nodes) {
     let type: ModalityType;
-    if (node instanceof ImageNode) {
+    if (
+      node.type === ObjectType.IMAGE ||
+      node.type === ObjectType.IMAGE_DOCUMENT
+    ) {
       type = ModalityType.IMAGE;
-    } else if (node instanceof TextNode) {
+    } else if (
+      node.type === ObjectType.TEXT ||
+      node.type === ObjectType.DOCUMENT ||
+      node.type === ObjectType.INDEX
+    ) {
       type = ModalityType.TEXT;
     } else {
       throw new Error(`Unknown node type: ${node.type}`);
@@ -465,28 +472,36 @@ export function buildNodeFromSplits(
   };
 
   textSplits.forEach((textChunk, i) => {
-    if (doc instanceof ImageDocument) {
+    if (
+      doc.type === ObjectType.IMAGE ||
+      doc.type === ObjectType.IMAGE_DOCUMENT
+    ) {
+      const imageDoc = doc as ImageNode;
       const imageNode = new ImageNode({
-        id_: idGenerator(i, doc),
+        id_: idGenerator(i, imageDoc),
         text: textChunk,
-        image: doc.image,
-        embedding: doc.embedding,
-        excludedEmbedMetadataKeys: [...doc.excludedEmbedMetadataKeys],
-        excludedLlmMetadataKeys: [...doc.excludedLlmMetadataKeys],
-        metadataSeparator: doc.metadataSeparator,
-        textTemplate: doc.textTemplate,
+        image: imageDoc.image,
+        embedding: imageDoc.embedding,
+        excludedEmbedMetadataKeys: [...imageDoc.excludedEmbedMetadataKeys],
+        excludedLlmMetadataKeys: [...imageDoc.excludedLlmMetadataKeys],
+        metadataSeparator: imageDoc.metadataSeparator,
+        textTemplate: imageDoc.textTemplate,
         relationships: { ...relationships },
       });
       nodes.push(imageNode);
-    } else if (doc instanceof Document || doc instanceof TextNode) {
+    } else if (
+      doc.type === ObjectType.DOCUMENT ||
+      doc.type === ObjectType.TEXT
+    ) {
+      const textDoc = doc as TextNode;
       const node = new TextNode({
-        id_: idGenerator(i, doc),
+        id_: idGenerator(i, textDoc),
         text: textChunk,
-        embedding: doc.embedding,
-        excludedEmbedMetadataKeys: [...doc.excludedEmbedMetadataKeys],
-        excludedLlmMetadataKeys: [...doc.excludedLlmMetadataKeys],
-        metadataSeparator: doc.metadataSeparator,
-        textTemplate: doc.textTemplate,
+        embedding: textDoc.embedding,
+        excludedEmbedMetadataKeys: [...textDoc.excludedEmbedMetadataKeys],
+        excludedLlmMetadataKeys: [...textDoc.excludedLlmMetadataKeys],
+        metadataSeparator: textDoc.metadataSeparator,
+        textTemplate: textDoc.textTemplate,
         relationships: { ...relationships },
       });
       nodes.push(node);

packages/core/src/storage/chat-store/base-chat-store.ts

@@ -0,0 +1,19 @@
+import type { ChatMessage } from "../../llms";
+
+export abstract class BaseChatStore<
+  AdditionalMessageOptions extends object = object,
+> {
+  abstract setMessages(
+    key: string,
+    messages: ChatMessage<AdditionalMessageOptions>[],
+  ): void;
+  abstract getMessages(key: string): ChatMessage<AdditionalMessageOptions>[];
+  abstract addMessage(
+    key: string,
+    message: ChatMessage<AdditionalMessageOptions>,
+    idx?: number,
+  ): void;
+  abstract deleteMessages(key: string): void;
+  abstract deleteMessage(key: string, idx: number): void;
+  abstract getKeys(): IterableIterator<string>;
+}

packages/core/src/storage/chat-store/index.ts

@@ -0,0 +1,2 @@
+export { BaseChatStore } from "./base-chat-store";
+export { SimpleChatStore } from "./simple-chat-store";

packages/core/src/storage/chat-store/simple-chat-store.ts

@@ -0,0 +1,43 @@
+import type { ChatMessage } from "../../llms";
+import { BaseChatStore } from "./base-chat-store";
+
+export class SimpleChatStore<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatStore<AdditionalMessageOptions> {
+  #store = new Map<string, ChatMessage<AdditionalMessageOptions>[]>();
+  setMessages(key: string, messages: ChatMessage<AdditionalMessageOptions>[]) {
+    this.#store.set(key, messages);
+  }
+
+  getMessages(key: string) {
+    return this.#store.get(key) ?? [];
+  }
+
+  addMessage(
+    key: string,
+    message: ChatMessage<AdditionalMessageOptions>,
+    idx?: number,
+  ) {
+    const messages = this.#store.get(key) ?? [];
+    if (idx === undefined) {
+      messages.push(message);
+    } else {
+      messages.splice(idx, 0, message);
+    }
+    this.#store.set(key, messages);
+  }
+
+  deleteMessages(key: string) {
+    this.#store.delete(key);
+  }
+
+  deleteMessage(key: string, idx: number) {
+    const messages = this.#store.get(key) ?? [];
+    messages.splice(idx, 1);
+    this.#store.set(key, messages);
+  }
+
+  getKeys() {
+    return this.#store.keys();
+  }
+}

packages/core/src/workflow/context.ts

@@ -95,7 +95,7 @@ export class Context {
     this.#streamingQueue.push(event);
   }
 
-  async *streamEvents(): AsyncGenerator<WorkflowEvent, void, undefined> {
+  async *streamEvents(): AsyncGenerator<WorkflowEvent, void, void> {
     while (true) {
       const event = this.#streamingQueue.shift();
       if (event) {

packages/core/src/workflow/events.ts

@@ -14,5 +14,5 @@ export type EventTypes<T extends Record<string, any> = any> = new (
   data: T,
 ) => WorkflowEvent<T>;
 
-export class StartEvent extends WorkflowEvent<{ input: string }> {}
-export class StopEvent extends WorkflowEvent<{ result: string }> {}
+export class StartEvent<T = string> extends WorkflowEvent<{ input: T }> {}
+export class StopEvent<T = string> extends WorkflowEvent<{ result: T }> {}

packages/core/src/workflow/workflow.ts

@@ -62,7 +62,7 @@ export class Workflow {
     return stepInfo.inputs.includes(eventType);
   }
 
-  async *streamEvents(): AsyncGenerator<WorkflowEvent, void, unknown> {
+  async *streamEvents(): AsyncGenerator<WorkflowEvent, void> {
     if (this.#contexts.size > 1) {
       throw new Error(
         "This workflow has multiple concurrent runs in progress and cannot stream events. " +
@@ -131,7 +131,7 @@ export class Workflow {
     }
   }
 
-  async run(event: StartEvent | string): Promise<StopEvent> {
+  async run<T = string>(event: StartEvent<T> | string): Promise<StopEvent> {
     // Validate the workflow before running if #validate is true
     if (this.#validate) {
       this.validate();

packages/core/tests/vitest.workspace.ts

@@ -0,0 +1,19 @@
+import { defineWorkspace } from "vitest/config";
+
+export default defineWorkspace([
+  {
+    test: {
+      environment: "edge-runtime",
+    },
+  },
+  {
+    test: {
+      environment: "happy-dom",
+    },
+  },
+  {
+    test: {
+      environment: "node",
+    },
+  },
+]);

packages/core/tests/workflow.test.ts

@@ -140,4 +140,30 @@ describe("Workflow", () => {
     expect(result.data.result).toBe("Report generated");
     expect(collectedEvents).toHaveLength(1);
   });
+
+  test("run workflow with object-based StartEvent and StopEvent", async () => {
+    const objectFlow = new Workflow({ verbose: true });
+
+    type Person = { name: string; age: number };
+
+    const processObject = vi.fn(async (_context, ev: StartEvent<Person>) => {
+      const { name, age } = ev.data.input;
+      return new StopEvent({
+        result: { greeting: `Hello ${name}, you are ${age} years old!` },
+      });
+    });
+
+    objectFlow.addStep(StartEvent<Person>, processObject);
+
+    const result = await objectFlow.run(
+      new StartEvent<Person>({
+        input: { name: "Alice", age: 30 },
+      }),
+    );
+
+    expect(processObject).toHaveBeenCalledTimes(1);
+    expect(result.data.result).toEqual({
+      greeting: "Hello Alice, you are 30 years old!",
+    });
+  });
 });

packages/env/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @llamaindex/env
 
+## 0.1.11
+
+### Patch Changes
+
+- ac07e3c: fix: replace instanceof check with `.type` check
+- 1a6137b: feat: experimental support for browser
+
+  If you see bundler issue in next.js edge runtime, please bump to `next@14` latest version.
+
+- ac07e3c: fix: add `console.warn` when import dual module
+
 ## 0.1.10
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -36,6 +36,10 @@
         "types": "./dist/type/index.edge-light.d.ts",
         "default": "./dist/index.edge-light.js"
       },
+      "browser": {
+        "types": "./dist/type/index.browser.d.ts",
+        "default": "./dist/index.browser.js"
+      },
       "import": {
         "types": "./dist/type/index.d.ts",
         "default": "./dist/index.js"

packages/env/src/global-check.ts

@@ -0,0 +1,31 @@
+const glo: any =
+  typeof globalThis !== "undefined"
+    ? globalThis
+    : // @ts-expect-error
+      typeof window !== "undefined"
+      ? // @ts-expect-error
+        window
+      : typeof global !== "undefined"
+        ? global
+        : {};
+
+const importIdentifier = "__ $@llamaindex/env$ __";
+
+if (glo[importIdentifier] === true) {
+  /**
+   * Dear reader of this message. Please take this seriously.
+   *
+   * If you see this message, make sure that you only import one version of llamaindex. In many cases,
+   * your package manager installs two versions of llamaindex that are used by different packages within your project.
+   * Another reason for this message is that some parts of your project use the CJS version of llamaindex
+   * and others use the ESM version of llamaindex.
+   *
+   * This often leads to issues that are hard to debug. We often need to perform constructor checks,
+   * e.g. `node instanceof TextNode`. If you imported different versions of llamaindex, it is impossible for us to
+   * do the constructor checks anymore - which might break the functionality of your application.
+   */
+  console.error(
+    "llamaindex was already imported. This breaks constructor checks and will lead to issues!",
+  );
+}
+glo[importIdentifier] = true;

packages/env/src/index.browser.ts

@@ -0,0 +1,16 @@
+/**
+ * Web environment polyfill.
+ *
+ * @module
+ */
+import "./global-check.js";
+export * from "./web-polyfill.js";
+
+export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";
+
+// @ts-expect-error
+if (typeof window === "undefined") {
+  console.warn(
+    "You are not in a browser environment. This module is not supposed to be used in a non-browser environment.",
+  );
+}

packages/env/src/index.edge-light.ts

@@ -3,6 +3,7 @@
  *
  * @module
  */
-export * from "./polyfill.js";
+import "./global-check.js";
+export * from "./node-polyfill.js";
 
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";

packages/env/src/index.ts

@@ -18,7 +18,8 @@ import path from "node:path";
 import { Readable } from "node:stream";
 import { fileURLToPath } from "node:url";
 import { createWriteStream, fs } from "./fs/node.js";
-import type { SHA256 } from "./polyfill.js";
+import "./global-check.js";
+import type { SHA256 } from "./node-polyfill.js";
 
 export function createSHA256(): SHA256 {
   const hash = createHash("sha256");
@@ -33,7 +34,12 @@ export function createSHA256(): SHA256 {
 }
 
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js";
-export { AsyncLocalStorage, CustomEvent, getEnv, setEnvs } from "./utils.js";
+export {
+  AsyncLocalStorage,
+  CustomEvent,
+  getEnv,
+  setEnvs,
+} from "./utils/index.js";
 export {
   createWriteStream,
   EOL,

packages/env/src/index.workerd.ts

@@ -5,9 +5,9 @@
  *
  * @module
  */
-import { INTERNAL_ENV } from "./utils.js";
+import { INTERNAL_ENV } from "./utils/index.js";
 
-export * from "./polyfill.js";
+export * from "./node-polyfill.js";
 
 export function getEnv(name: string): string | undefined {
   return INTERNAL_ENV[name];

packages/env/src/node-polyfill.ts

@@ -46,4 +46,9 @@ export function randomUUID(): string {
   return crypto.randomUUID();
 }
 
-export { AsyncLocalStorage, CustomEvent, getEnv, setEnvs } from "./utils.js";
+export {
+  AsyncLocalStorage,
+  CustomEvent,
+  getEnv,
+  setEnvs,
+} from "./utils/index.js";

packages/env/src/utils/index.web.ts

@@ -0,0 +1,45 @@
+// DO NOT EXPOSE THIS VARIABLE TO PUBLIC, IT IS USED INTERNALLY FOR BROWSER ENVIRONMENT
+export const INTERNAL_ENV: Record<string, string> = {};
+
+export function setEnvs(envs: object): void {
+  Object.assign(INTERNAL_ENV, envs);
+}
+
+export function getEnv(name: string): string | undefined {
+  if (INTERNAL_ENV[name]) {
+    return INTERNAL_ENV[name];
+  }
+}
+
+// Web doesn't have AsyncLocalStorage and there's no alternative way to implement it
+// Wait for https://github.com/tc39/proposal-async-context
+export class AsyncLocalStorage<T> {
+  #store: T = null!;
+  static bind<Func extends (...args: any[]) => any>(fn: Func): Func {
+    return fn;
+  }
+
+  static snapshot(): <R, TArgs extends any[]>(
+    fn: (...args: TArgs) => R,
+    ...args: TArgs
+  ) => R {
+    return (cb: any, ...args: any[]) => cb(...args);
+  }
+
+  getStore() {
+    return this.#store;
+  }
+
+  run<R>(store: T, cb: () => R): R {
+    this.#store = store;
+    if (cb.constructor.name === "AsyncFunction") {
+      console.warn("AsyncLocalStorage is not supported in the web environment");
+      console.warn("Please note that some features may not work as expected");
+    }
+    return cb();
+  }
+}
+
+const defaultCustomEvent = (globalThis as any).CustomEvent;
+
+export { defaultCustomEvent as CustomEvent };

packages/env/src/web-polyfill.ts

@@ -0,0 +1,54 @@
+/**
+ * Polyfill implementation for `@llamaindex/env`.
+ *
+ * The code should be compatible with any JS runtime.
+ *
+ * Sometimes you should overwrite the polyfill with a native implementation.
+ *
+ * @module
+ */
+import { Sha256 } from "@aws-crypto/sha256-js";
+import pathe from "pathe";
+import { fs } from "./fs/memory.js";
+
+export { fs, pathe as path };
+
+export interface SHA256 {
+  update(data: string | Uint8Array): void;
+  // to base64
+  digest(): string;
+}
+
+export const EOL = "\n";
+
+export function ok(value: unknown, message?: string): asserts value {
+  if (!value) {
+    const error = Error(message);
+    error.name = "AssertionError";
+    error.message = message ?? "The expression evaluated to a falsy value.";
+    throw error;
+  }
+}
+
+export function createSHA256(): SHA256 {
+  const sha256 = new Sha256();
+  return {
+    update(data: string | Uint8Array): void {
+      sha256.update(data);
+    },
+    digest() {
+      return globalThis.btoa(sha256.digestSync().toString());
+    },
+  };
+}
+
+export function randomUUID(): string {
+  return crypto.randomUUID();
+}
+
+export {
+  AsyncLocalStorage,
+  CustomEvent,
+  getEnv,
+  setEnvs,
+} from "./utils/index.web.js";

packages/experimental/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/experimental
 
+## 0.0.80
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+
+## 0.0.79
+
+### Patch Changes
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+
 ## 0.0.78
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.78",
+  "version": "0.0.80",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,44 @@
 # llamaindex
 
+## 0.6.2
+
+### Patch Changes
+
+- 749b43a: fix: clip embedding transform function
+- Updated dependencies [b42adeb]
+- Updated dependencies [749b43a]
+  - @llamaindex/cloud@0.2.6
+  - @llamaindex/core@0.2.2
+  - @llamaindex/openai@0.1.4
+  - @llamaindex/groq@0.0.3
+
+## 0.6.1
+
+### Patch Changes
+
+- fbd5e01: refactor: move groq as llm package
+- 6b70c54: feat: update JinaAIEmbedding, support embedding v3
+- 1a6137b: feat: experimental support for browser
+
+  If you see bundler issue in next.js edge runtime, please bump to `next@14` latest version.
+
+- 85c2e19: feat: `@llamaindex/cloud` package update
+
+  - Bump to latest openapi schema
+  - Move LlamaParse class from llamaindex, this will allow you use llamaparse in more non-node.js environment
+
+- Updated dependencies [ac07e3c]
+- Updated dependencies [fbd5e01]
+- Updated dependencies [70ccb4a]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+- Updated dependencies [ac07e3c]
+  - @llamaindex/core@0.2.1
+  - @llamaindex/env@0.1.11
+  - @llamaindex/groq@0.0.2
+  - @llamaindex/cloud@0.2.5
+  - @llamaindex/openai@0.1.3
+
 ## 0.6.0
 
 ### Minor Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.64
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+
+## 0.0.63
+
+### Patch Changes
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+
 ## 0.0.62
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.62",
+  "version": "0.0.64",
   "type": "module",
   "private": true,
   "scripts": {
@@ -16,7 +16,7 @@
     "@cloudflare/workers-types": "^4.20240821.1",
     "@vitest/runner": "1.5.3",
     "@vitest/snapshot": "1.5.3",
-    "typescript": "^5.5.4",
+    "typescript": "^5.6.2",
     "vitest": "1.5.3",
     "wrangler": "^3.73.0"
   },

packages/llamaindex/e2e/examples/llama-parse-browser/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

packages/llamaindex/e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -0,0 +1,15 @@
+# @llamaindex/llama-parse-browser-test
+
+## 0.0.2
+
+### Patch Changes
+
+- Updated dependencies [b42adeb]
+  - @llamaindex/cloud@0.2.6
+
+## 0.0.1
+
+### Patch Changes
+
+- Updated dependencies [85c2e19]
+  - @llamaindex/cloud@0.2.5

packages/llamaindex/e2e/examples/llama-parse-browser/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + TS</title>
+  </head>
+  <body>
+    <div id="app"></div>
+    <script type="module" src="/src/main.ts"></script>
+  </body>
+</html>

packages/llamaindex/e2e/examples/llama-parse-browser/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "@llamaindex/llama-parse-browser-test",
+  "private": true,
+  "version": "0.0.2",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "preview": "vite preview"
+  },
+  "devDependencies": {
+    "typescript": "^5.5.3",
+    "vite": "^5.4.1",
+    "vite-plugin-wasm": "^3.3.0"
+  },
+  "dependencies": {
+    "@llamaindex/cloud": "workspace:*"
+  }
+}

packages/llamaindex/e2e/examples/llama-parse-browser/src/main.ts

@@ -0,0 +1,10 @@
+import { LlamaParseReader } from "@llamaindex/cloud/reader";
+import "./style.css";
+
+new LlamaParseReader();
+
+document.querySelector<HTMLDivElement>("#app")!.innerHTML = `
+  <div>
+    Hello, world!
+  </div>
+`;

packages/llamaindex/e2e/examples/llama-parse-browser/src/style.css

@@ -0,0 +1,96 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+#app {
+  max-width: 1280px;
+  margin: 0 auto;
+  padding: 2rem;
+  text-align: center;
+}
+
+.logo {
+  height: 6em;
+  padding: 1.5em;
+  will-change: filter;
+  transition: filter 300ms;
+}
+.logo:hover {
+  filter: drop-shadow(0 0 2em #646cffaa);
+}
+.logo.vanilla:hover {
+  filter: drop-shadow(0 0 2em #3178c6aa);
+}
+
+.card {
+  padding: 2em;
+}
+
+.read-the-docs {
+  color: #888;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}

packages/llamaindex/e2e/examples/llama-parse-browser/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />