Release 0.5.17 (#1117)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

apps/docs/CHANGELOG.md

@@ -1,5 +1,34 @@
 # docs
 
+## 0.0.58
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.57
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.56
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.55
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
 ## 0.0.54
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.54",
+  "version": "0.0.58",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/weaviate/README.md

@@ -0,0 +1,31 @@
+# Weaviate Vector Store
+
+Here are two sample scripts which work with loading and querying data from a Weaviate Vector Store.
+
+## Prerequisites
+
+- An Weaviate Vector Database
+  - Hosted https://weaviate.io/
+  - Self Hosted https://weaviate.io/developers/weaviate/installation/docker-compose#starter-docker-compose-file
+- An OpenAI API Key
+
+## Setup
+
+1. Set your env variables:
+
+- `WEAVIATE_CLUSTER_URL`: Address of your Weaviate Vector Store (like localhost:8080)
+- `WEAVIATE_API_KEY`: Your Weaviate API key
+- `OPENAI_API_KEY`: Your OpenAI key
+
+2. `cd` Into the `examples` directory
+3. run `npm i`
+
+## Load the data
+
+This sample loads the same dataset of movie reviews as sample dataset
+
+run `npx tsx weaviate/load`
+
+## Use RAG to Query the data
+
+run `npx tsx weaviate/query`

examples/weaviate/load.ts

@@ -0,0 +1,23 @@
+import {
+  PapaCSVReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+  WeaviateVectorStore,
+} from "llamaindex";
+
+const indexName = "MovieReviews";
+
+async function main() {
+  try {
+    const reader = new PapaCSVReader(false);
+    const docs = await reader.loadData("./data/movie_reviews.csv");
+    const vectorStore = new WeaviateVectorStore({ indexName });
+    const storageContext = await storageContextFromDefaults({ vectorStore });
+    await VectorStoreIndex.fromDocuments(docs, { storageContext });
+    console.log("Successfully loaded data into Weaviate");
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+void main();

examples/weaviate/query.ts

@@ -0,0 +1,46 @@
+import { VectorStoreIndex, WeaviateVectorStore } from "llamaindex";
+
+const indexName = "MovieReviews";
+
+async function main() {
+  try {
+    const query = "Get all movie titles.";
+    const vectorStore = new WeaviateVectorStore({ indexName });
+    const index = await VectorStoreIndex.fromVectorStore(vectorStore);
+    const retriever = index.asRetriever({ similarityTopK: 20 });
+
+    const queryEngine = index.asQueryEngine({ retriever });
+    const results = await queryEngine.query({ query });
+    console.log(`Query from ${results.sourceNodes?.length} nodes`);
+    console.log(results.response);
+
+    console.log("\n=====\nQuerying the index with filters");
+    const queryEngineWithFilters = index.asQueryEngine({
+      retriever,
+      preFilters: {
+        filters: [
+          {
+            key: "document_id",
+            value: "./data/movie_reviews.csv_37",
+            operator: "==",
+          },
+          {
+            key: "document_id",
+            value: "./data/movie_reviews.csv_21",
+            operator: "==",
+          },
+        ],
+        condition: "or",
+      },
+    });
+    const resultAfterFilter = await queryEngineWithFilters.query({
+      query: "Get all movie titles.",
+    });
+    console.log(`Query from ${resultAfterFilter.sourceNodes?.length} nodes`);
+    console.log(resultAfterFilter.response);
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+void main();

packages/autotool/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/autotool
 
+## 2.0.1
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
 ## 2.0.0
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.42
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+  - @llamaindex/autotool@2.0.1
+
+## 0.1.41
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - @llamaindex/autotool@2.0.1
+  - llamaindex@0.5.16
+
+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.39
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+  - @llamaindex/autotool@2.0.0
+
 ## 0.1.38
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.38",
+  "version": "0.1.42",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",
@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.5.13",
+    "llamaindex": "^0.5.17",
     "openai": "^4",
     "typescript": "^4"
   },

packages/cloud/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/cloud
 
+## 0.2.2
+
+### Patch Changes
+
+- 58abc57: fix: align version
+
 ## 0.2.1
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @llamaindex/community
 
+## 0.0.29
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - @llamaindex/core@0.1.8
+  - @llamaindex/env@0.1.9
+
 ## 0.0.28
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.28",
+  "version": "0.0.29",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/core
 
+## 0.1.8
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - @llamaindex/env@0.1.9
+
 ## 0.1.7
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./node-parser": {

packages/core/src/schema/index.ts

@@ -1,4 +1,4 @@
 export * from "./node";
-export { TransformComponent } from "./type";
+export { FileReader, TransformComponent, type BaseReader } from "./type";
 export { EngineResponse } from "./type/engine–response";
 export * from "./zod";

packages/core/src/schema/type.ts

@@ -1,5 +1,5 @@
-import { randomUUID } from "@llamaindex/env";
-import type { BaseNode } from "./node";
+import { fs, path, randomUUID } from "@llamaindex/env";
+import type { BaseNode, Document } from "./node";
 
 interface TransformComponentSignature {
   <Options extends Record<string, unknown>>(
@@ -28,3 +28,37 @@ export class TransformComponent {
     return transform;
   }
 }
+
+/**
+ * A reader takes imports data into Document objects.
+ */
+export interface BaseReader {
+  loadData(...args: unknown[]): Promise<Document[]>;
+}
+
+/**
+ * A FileReader takes file paths and imports data into Document objects.
+ */
+export abstract class FileReader implements BaseReader {
+  abstract loadDataAsContent(
+    fileContent: Uint8Array,
+    fileName?: string,
+  ): Promise<Document[]>;
+
+  async loadData(filePath: string): Promise<Document[]> {
+    const fileContent = await fs.readFile(filePath);
+    const fileName = path.basename(filePath);
+    const docs = await this.loadDataAsContent(fileContent, fileName);
+    docs.forEach(FileReader.addMetaData(filePath));
+    return docs;
+  }
+
+  static addMetaData(filePath: string) {
+    return (doc: Document, index: number) => {
+      // generate id as loadDataAsContent is only responsible for the content
+      doc.id_ = `${filePath}_${index + 1}`;
+      doc.metadata["file_path"] = path.resolve(filePath);
+      doc.metadata["file_name"] = path.basename(filePath);
+    };
+  }
+}

packages/env/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/env
 
+## 0.1.9
+
+### Patch Changes
+
+- 58abc57: fix: align version
+
 ## 0.1.8
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.8",
+  "version": "0.1.9",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/experimental/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/experimental
 
+## 0.0.67
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.66
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.65
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.64
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
 ## 0.0.63
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.63",
+  "version": "0.0.67",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,34 @@
 # llamaindex
 
+## 0.5.17
+
+### Patch Changes
+
+- c654398: Implement Weaviate Vector Store in TS
+
+## 0.5.16
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - @llamaindex/cloud@0.2.2
+  - @llamaindex/core@0.1.8
+  - @llamaindex/env@0.1.9
+
+## 0.5.15
+
+### Patch Changes
+
+- 01c184c: Add is_empty operator for filtering vector store
+- 07a275f: chore: bump openai
+
+## 0.5.14
+
+### Patch Changes
+
+- c825a2f: Add gpt-4o-mini to Azure. Add 2024-06-01 API version for Azure
+
 ## 0.5.13
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,34 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.51
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.50
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.49
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
 ## 0.0.47
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.47",
+  "version": "0.0.51",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,34 @@
 # @llamaindex/next-agent-test
 
+## 0.1.51
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.1.50
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.1.49
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.1.48
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
 ## 0.1.47
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.47",
+  "version": "0.1.51",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,34 @@
 # test-edge-runtime
 
+## 0.1.50
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.1.49
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.1.48
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.1.47
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
 ## 0.1.46
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.46",
+  "version": "0.1.50",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,34 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.32
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.31
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.30
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
 ## 0.0.28
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.28",
+  "version": "0.0.32",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,34 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.51
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.50
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.49
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
 ## 0.0.47
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.47",
+  "version": "0.0.51",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.5.13",
+  "version": "0.5.17",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -55,7 +55,7 @@
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.7.0",
     "notion-md-crawler": "^1.0.0",
-    "openai": "^4.52.5",
+    "openai": "^4.55.3",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
     "pg": "^8.12.0",
@@ -65,6 +65,7 @@
     "string-strip-html": "^13.4.8",
     "tiktoken": "^1.0.15",
     "unpdf": "^0.11.0",
+    "weaviate-client": "^3.1.4",
     "wikipedia": "^2.1.2",
     "wink-nlp": "^2.3.0",
     "zod": "^3.23.8"

packages/llamaindex/src/index.edge.ts

@@ -1,6 +1,8 @@
 import type { AgentEndEvent, AgentStartEvent } from "./agent/types.js";
 import type { RetrievalEndEvent, RetrievalStartEvent } from "./llm/types.js";
 
+export * from "@llamaindex/core/schema";
+
 declare module "@llamaindex/core/global" {
   export interface LlamaIndexEventMaps {
     "retrieve-start": RetrievalStartEvent;

packages/llamaindex/src/ingestion/IngestionPipeline.ts

@@ -1,4 +1,4 @@
-import type { TransformComponent } from "@llamaindex/core/schema";
+import type { BaseReader, TransformComponent } from "@llamaindex/core/schema";
 import {
   ModalityType,
   splitNodesByType,
@@ -6,7 +6,6 @@ import {
   type Document,
   type Metadata,
 } from "@llamaindex/core/schema";
-import type { BaseReader } from "../readers/type.js";
 import type { BaseDocumentStore } from "../storage/docStore/types.js";
 import type {
   VectorStore,
@@ -107,6 +106,7 @@ export class IngestionPipeline {
       inputNodes.push(this.documents);
     }
     if (this.reader) {
+      // fixme: empty parameter might cause error
       inputNodes.push(await this.reader.loadData());
     }
     return inputNodes.flat();

packages/llamaindex/src/llm/azure.ts

@@ -18,6 +18,7 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     openAIModel: "gpt-3.5-turbo-16k",
   },
   "gpt-4o": { contextWindow: 128000, openAIModel: "gpt-4o" },
+  "gpt-4o-mini": { contextWindow: 128000, openAIModel: "gpt-4o-mini" },
   "gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
   "gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
   "gpt-4-turbo": {
@@ -40,6 +41,10 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     contextWindow: 128000,
     openAIModel: "gpt-4o-2024-05-13",
   },
+  "gpt-4o-mini-2024-07-18": {
+    contextWindow: 128000,
+    openAIModel: "gpt-4o-mini-2024-07-18",
+  },
 };
 
 const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
@@ -73,6 +78,7 @@ const ALL_AZURE_API_VERSIONS = [
   "2024-03-01-preview",
   "2024-04-01-preview",
   "2024-05-01-preview",
+  "2024-06-01",
 ];
 
 const DEFAULT_API_VERSION = "2023-05-15";

packages/llamaindex/src/llm/openai.ts

@@ -6,6 +6,7 @@ import type {
   ClientOptions as OpenAIClientOptions,
 } from "openai";
 import { AzureOpenAI, OpenAI as OrigOpenAI } from "openai";
+import type { ChatModel } from "openai/resources/chat/chat";
 
 import {
   type BaseTool,
@@ -108,16 +109,24 @@ export const GPT4_MODELS = {
   "gpt-4o-2024-05-13": { contextWindow: 128000 },
   "gpt-4o-mini": { contextWindow: 128000 },
   "gpt-4o-mini-2024-07-18": { contextWindow: 128000 },
+  "gpt-4o-2024-08-06": { contextWindow: 128000 },
+  "gpt-4o-2024-09-14": { contextWindow: 128000 },
+  "gpt-4o-2024-10-14": { contextWindow: 128000 },
+  "gpt-4-0613": { contextWindow: 128000 },
+  "gpt-4-turbo-2024-04-09": { contextWindow: 128000 },
+  "gpt-4-0314": { contextWindow: 128000 },
+  "gpt-4-32k-0314": { contextWindow: 32768 },
 };
 
 // NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
 export const GPT35_MODELS = {
-  "gpt-3.5-turbo": { contextWindow: 4096 },
+  "gpt-3.5-turbo": { contextWindow: 16385 },
   "gpt-3.5-turbo-0613": { contextWindow: 4096 },
-  "gpt-3.5-turbo-16k": { contextWindow: 16384 },
-  "gpt-3.5-turbo-16k-0613": { contextWindow: 16384 },
-  "gpt-3.5-turbo-1106": { contextWindow: 16384 },
-  "gpt-3.5-turbo-0125": { contextWindow: 16384 },
+  "gpt-3.5-turbo-16k": { contextWindow: 16385 },
+  "gpt-3.5-turbo-16k-0613": { contextWindow: 16385 },
+  "gpt-3.5-turbo-1106": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0125": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0301": { contextWindow: 16385 },
 };
 
 /**
@@ -126,7 +135,7 @@ export const GPT35_MODELS = {
 export const ALL_AVAILABLE_OPENAI_MODELS = {
   ...GPT4_MODELS,
   ...GPT35_MODELS,
-};
+} satisfies Record<ChatModel, { contextWindow: number }>;
 
 export function isFunctionCallingModel(llm: LLM): llm is OpenAI {
   let model: string;
@@ -157,8 +166,10 @@ export type OpenAIAdditionalChatOptions = Omit<
 >;
 
 export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
-  // Per completion OpenAI params
-  model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
+  model:
+    | ChatModel
+    // string & {} is a hack to allow any string, but still give autocomplete
+    | (string & {});
   temperature: number;
   topP: number;
   maxTokens?: number;

packages/llamaindex/src/readers/AssemblyAIReader.ts

@@ -1,4 +1,4 @@
-import { Document } from "@llamaindex/core/schema";
+import { type BaseReader, Document } from "@llamaindex/core/schema";
 import { getEnv } from "@llamaindex/env";
 import type {
   BaseServiceParams,
@@ -8,7 +8,6 @@ import type {
   TranscriptSentence,
 } from "assemblyai";
 import { AssemblyAI } from "assemblyai";
-import type { BaseReader } from "./type.js";
 
 type AssemblyAIOptions = Partial<BaseServiceParams>;
 const defaultOptions = {

packages/llamaindex/src/readers/CSVReader.ts

@@ -1,7 +1,6 @@
-import { Document } from "@llamaindex/core/schema";
+import { type BaseReader, Document, FileReader } from "@llamaindex/core/schema";
 import type { ParseConfig } from "papaparse";
 import Papa from "papaparse";
-import { FileReader } from "./type.js";
 
 /**
  * papaparse-based csv parser

packages/llamaindex/src/readers/DiscordReader.ts

@@ -1,5 +1,5 @@
 import { REST, type RESTOptions } from "@discordjs/rest";
-import { Document } from "@llamaindex/core/schema";
+import { Document, type BaseReader } from "@llamaindex/core/schema";
 import { getEnv } from "@llamaindex/env";
 import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
 
@@ -7,7 +7,7 @@ import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
  * Represents a reader for Discord messages using @discordjs/rest
  * See https://github.com/discordjs/discord.js/tree/main/packages/rest
  */
-export class DiscordReader {
+export class DiscordReader implements BaseReader {
   private client: REST;
 
   constructor(

packages/llamaindex/src/readers/DocxReader.ts

@@ -1,6 +1,5 @@
-import { Document } from "@llamaindex/core/schema";
+import { Document, FileReader } from "@llamaindex/core/schema";
 import mammoth from "mammoth";
-import { FileReader } from "./type.js";
 
 export class DocxReader extends FileReader {
   /** DocxParser */

packages/llamaindex/src/readers/HTMLReader.ts

@@ -1,6 +1,4 @@
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
-
+import { Document, FileReader } from "@llamaindex/core/schema";
 /**
  * Extract the significant text from an arbitrary HTML document.
  * The contents of any head, script, style, and xml tags are removed completely.

packages/llamaindex/src/readers/ImageReader.ts

@@ -1,6 +1,5 @@
 import type { Document } from "@llamaindex/core/schema";
-import { ImageDocument } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
+import { FileReader, ImageDocument } from "@llamaindex/core/schema";
 
 /**
  * Reads the content of an image file into a Document object (which stores the image file as a Blob).

packages/llamaindex/src/readers/JSONReader.ts

@@ -1,7 +1,5 @@
 import type { JSONValue } from "@llamaindex/core/global";
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
-
+import { Document, FileReader } from "@llamaindex/core/schema";
 export interface JSONReaderOptions {
   /**
    * Whether to ensure only ASCII characters.

packages/llamaindex/src/readers/LlamaParseReader.ts

@@ -1,7 +1,92 @@
-import { Document } from "@llamaindex/core/schema";
+import { Document, FileReader } from "@llamaindex/core/schema";
 import { fs, getEnv } from "@llamaindex/env";
 import { filetypeinfo } from "magic-bytes.js";
-import { FileReader, type Language, type ResultType } from "./type.js";
+
+export type ResultType = "text" | "markdown" | "json";
+export type Language =
+  | "abq"
+  | "ady"
+  | "af"
+  | "ang"
+  | "ar"
+  | "as"
+  | "ava"
+  | "az"
+  | "be"
+  | "bg"
+  | "bh"
+  | "bho"
+  | "bn"
+  | "bs"
+  | "ch_sim"
+  | "ch_tra"
+  | "che"
+  | "cs"
+  | "cy"
+  | "da"
+  | "dar"
+  | "de"
+  | "en"
+  | "es"
+  | "et"
+  | "fa"
+  | "fr"
+  | "ga"
+  | "gom"
+  | "hi"
+  | "hr"
+  | "hu"
+  | "id"
+  | "inh"
+  | "is"
+  | "it"
+  | "ja"
+  | "kbd"
+  | "kn"
+  | "ko"
+  | "ku"
+  | "la"
+  | "lbe"
+  | "lez"
+  | "lt"
+  | "lv"
+  | "mah"
+  | "mai"
+  | "mi"
+  | "mn"
+  | "mr"
+  | "ms"
+  | "mt"
+  | "ne"
+  | "new"
+  | "nl"
+  | "no"
+  | "oc"
+  | "pi"
+  | "pl"
+  | "pt"
+  | "ro"
+  | "ru"
+  | "rs_cyrillic"
+  | "rs_latin"
+  | "sck"
+  | "sk"
+  | "sl"
+  | "sq"
+  | "sv"
+  | "sw"
+  | "ta"
+  | "tab"
+  | "te"
+  | "th"
+  | "tjk"
+  | "tl"
+  | "tr"
+  | "ug"
+  | "uk"
+  | "ur"
+  | "uz"
+  | "vi";
 
 const SUPPORT_FILE_EXT: string[] = [
   ".pdf",

packages/llamaindex/src/readers/MarkdownReader.ts

@@ -1,5 +1,4 @@
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
+import { Document, FileReader } from "@llamaindex/core/schema";
 
 type MarkdownTuple = [string | null, string];
 

packages/llamaindex/src/readers/NotionReader.ts

@@ -1,7 +1,7 @@
+import type { BaseReader } from "@llamaindex/core/schema";
 import { Document } from "@llamaindex/core/schema";
 import type { Crawler, CrawlerOptions, Page } from "notion-md-crawler";
 import { crawler, pageToString } from "notion-md-crawler";
-import type { BaseReader } from "./type.js";
 
 type NotionReaderOptions = Pick<CrawlerOptions, "client" | "serializers">;
 

packages/llamaindex/src/readers/PDFReader.ts

@@ -1,5 +1,4 @@
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
+import { Document, FileReader } from "@llamaindex/core/schema";
 
 /**
  * Read the text of a PDF

packages/llamaindex/src/readers/SimpleDirectoryReader.edge.ts

@@ -1,8 +1,8 @@
+import type { BaseReader, FileReader } from "@llamaindex/core/schema";
 import { Document } from "@llamaindex/core/schema";
 import { path } from "@llamaindex/env";
 import { walk } from "../storage/FileSystem.js";
 import { TextFileReader } from "./TextFileReader.js";
-import type { BaseReader, FileReader } from "./type.js";
 import pLimit from "./utils.js";
 
 type ReaderCallback = (

packages/llamaindex/src/readers/SimpleDirectoryReader.ts

@@ -1,3 +1,4 @@
+import type { FileReader } from "@llamaindex/core/schema";
 import { Document } from "@llamaindex/core/schema";
 import { PapaCSVReader } from "./CSVReader.js";
 import { DocxReader } from "./DocxReader.js";
@@ -10,7 +11,6 @@ import {
   type SimpleDirectoryReaderLoadDataParams,
 } from "./SimpleDirectoryReader.edge.js";
 import { TextFileReader } from "./TextFileReader.js";
-import type { FileReader } from "./type.js";
 
 export const FILE_EXT_TO_READER: Record<string, FileReader> = {
   txt: new TextFileReader(),

packages/llamaindex/src/readers/SimpleMongoReader.ts

@@ -1,7 +1,6 @@
 import type { Metadata } from "@llamaindex/core/schema";
-import { Document } from "@llamaindex/core/schema";
+import { type BaseReader, Document } from "@llamaindex/core/schema";
 import type { MongoClient } from "mongodb";
-import type { BaseReader } from "./type.js";
 
 /**
  * Read in from MongoDB

packages/llamaindex/src/readers/TextFileReader.ts

@@ -1,6 +1,4 @@
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
-
+import { Document, FileReader } from "@llamaindex/core/schema";
 /**
  * Read a .txt file
  */

packages/llamaindex/src/readers/type.ts

@@ -1,124 +0,0 @@
-import type { Document } from "@llamaindex/core/schema";
-import { fs, path } from "@llamaindex/env";
-
-/**
- * A reader takes imports data into Document objects.
- */
-export interface BaseReader {
-  loadData(...args: unknown[]): Promise<Document[]>;
-}
-
-/**
- * A FileReader takes file paths and imports data into Document objects.
- */
-export abstract class FileReader implements BaseReader {
-  abstract loadDataAsContent(
-    fileContent: Uint8Array,
-    fileName?: string,
-  ): Promise<Document[]>;
-
-  async loadData(filePath: string): Promise<Document[]> {
-    const fileContent = await fs.readFile(filePath);
-    const fileName = path.basename(filePath);
-    const docs = await this.loadDataAsContent(fileContent, fileName);
-    docs.forEach(FileReader.addMetaData(filePath));
-    return docs;
-  }
-
-  static addMetaData(filePath: string) {
-    return (doc: Document, index: number) => {
-      // generate id as loadDataAsContent is only responsible for the content
-      doc.id_ = `${filePath}_${index + 1}`;
-      doc.metadata["file_path"] = path.resolve(filePath);
-      doc.metadata["file_name"] = path.basename(filePath);
-    };
-  }
-}
-
-// For LlamaParseReader.ts
-
-export type ResultType = "text" | "markdown" | "json";
-export type Language =
-  | "abq"
-  | "ady"
-  | "af"
-  | "ang"
-  | "ar"
-  | "as"
-  | "ava"
-  | "az"
-  | "be"
-  | "bg"
-  | "bh"
-  | "bho"
-  | "bn"
-  | "bs"
-  | "ch_sim"
-  | "ch_tra"
-  | "che"
-  | "cs"
-  | "cy"
-  | "da"
-  | "dar"
-  | "de"
-  | "en"
-  | "es"
-  | "et"
-  | "fa"
-  | "fr"
-  | "ga"
-  | "gom"
-  | "hi"
-  | "hr"
-  | "hu"
-  | "id"
-  | "inh"
-  | "is"
-  | "it"
-  | "ja"
-  | "kbd"
-  | "kn"
-  | "ko"
-  | "ku"
-  | "la"
-  | "lbe"
-  | "lez"
-  | "lt"
-  | "lv"
-  | "mah"
-  | "mai"
-  | "mi"
-  | "mn"
-  | "mr"
-  | "ms"
-  | "mt"
-  | "ne"
-  | "new"
-  | "nl"
-  | "no"
-  | "oc"
-  | "pi"
-  | "pl"
-  | "pt"
-  | "ro"
-  | "ru"
-  | "rs_cyrillic"
-  | "rs_latin"
-  | "sck"
-  | "sk"
-  | "sl"
-  | "sq"
-  | "sv"
-  | "sw"
-  | "ta"
-  | "tab"
-  | "te"
-  | "th"
-  | "tjk"
-  | "tl"
-  | "tr"
-  | "ug"
-  | "uk"
-  | "ur"
-  | "uz"
-  | "vi";

packages/llamaindex/src/storage/index.ts

@@ -18,3 +18,4 @@ export { PineconeVectorStore } from "./vectorStore/PineconeVectorStore.js";
 export { QdrantVectorStore } from "./vectorStore/QdrantVectorStore.js";
 export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore.js";
 export * from "./vectorStore/types.js";
+export { WeaviateVectorStore } from "./vectorStore/WeaviateVectorStore.js";

packages/llamaindex/src/storage/vectorStore/PGVectorStore.ts

@@ -273,7 +273,7 @@ export class PGVectorStore
       const paramIndex = params.length + 1;
       whereClauses.push(`metadata->>'${filter.key}' = $${paramIndex}`);
       // TODO: support filter with other operators
-      if (!Array.isArray(filter.value)) {
+      if (!Array.isArray(filter.value) && filter.value) {
         params.push(filter.value);
       }
     });

packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts

@@ -36,7 +36,7 @@ type MetadataValue = Record<string, any>;
 
 // Mapping of filter operators to metadata filter functions
 const OPERATOR_TO_FILTER: {
-  [key in FilterOperator]: (
+  [key in FilterOperator]?: (
     { key, value }: MetadataFilter,
     metadata: MetadataValue,
   ) => boolean;
@@ -94,7 +94,20 @@ const buildFilterFn = (
   const queryCondition = condition || "and"; // default to and
 
   const itemFilterFn = (filter: MetadataFilter): boolean => {
-    if (metadata[filter.key] === undefined) return false; // always return false if the metadata key is not present
+    if (filter.operator === FilterOperator.IS_EMPTY) {
+      // for `is_empty` operator, return true if the metadata key is not present or the value is empty
+      const value = metadata[filter.key];
+      return (
+        value === undefined ||
+        value === null ||
+        value === "" ||
+        (Array.isArray(value) && value.length === 0)
+      );
+    }
+    if (metadata[filter.key] === undefined) {
+      // for other operators, always return false if the metadata key is not present
+      return false;
+    }
     const metadataLookupFn = OPERATOR_TO_FILTER[filter.operator];
     if (!metadataLookupFn)
       throw new Error(`Unsupported operator: ${filter.operator}`);

packages/llamaindex/src/storage/vectorStore/WeaviateVectorStore.ts

@@ -0,0 +1,339 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import { BaseNode, MetadataMode, type Metadata } from "@llamaindex/core/schema";
+import weaviate, {
+  Filters,
+  type Collection,
+  type DeleteManyOptions,
+  type FilterValue,
+  type WeaviateClient,
+  type WeaviateNonGenericObject,
+} from "weaviate-client";
+
+import {
+  VectorStoreBase,
+  VectorStoreQueryMode,
+  type IEmbedModel,
+  type MetadataFilter,
+  type MetadataFilters,
+  type VectorStoreNoEmbedModel,
+  type VectorStoreQuery,
+  type VectorStoreQueryResult,
+} from "./types.js";
+import {
+  metadataDictToNode,
+  nodeToMetadata,
+  parseArrayValue,
+  parseNumberValue,
+} from "./utils.js";
+
+const NODE_SCHEMA = [
+  {
+    dataType: ["text"],
+    description: "Text property",
+    name: "text",
+  },
+  {
+    dataType: ["text"],
+    description: "The ref_doc_id of the Node",
+    name: "ref_doc_id",
+  },
+  {
+    dataType: ["text"],
+    description: "node_info (in JSON)",
+    name: "node_info",
+  },
+  {
+    dataType: ["text"],
+    description: "The relationships of the node (in JSON)",
+    name: "relationships",
+  },
+];
+
+const SIMILARITY_KEYS: {
+  [key: string]: "distance" | "score";
+} = {
+  [VectorStoreQueryMode.DEFAULT]: "distance",
+  [VectorStoreQueryMode.HYBRID]: "score",
+};
+
+const buildFilterItem = (
+  collection: Collection,
+  filter: MetadataFilter,
+): FilterValue => {
+  const { key, operator, value } = filter;
+
+  switch (operator) {
+    case "==": {
+      return collection.filter.byProperty(key).equal(value);
+    }
+    case "!=": {
+      return collection.filter.byProperty(key).notEqual(value);
+    }
+    case ">": {
+      return collection.filter
+        .byProperty(key)
+        .greaterThan(parseNumberValue(value));
+    }
+    case "<": {
+      return collection.filter
+        .byProperty(key)
+        .lessThan(parseNumberValue(value));
+    }
+    case ">=": {
+      return collection.filter
+        .byProperty(key)
+        .greaterOrEqual(parseNumberValue(value));
+    }
+    case "<=": {
+      return collection.filter
+        .byProperty(key)
+        .lessOrEqual(parseNumberValue(value));
+    }
+    case "any": {
+      return collection.filter
+        .byProperty(key)
+        .containsAny(parseArrayValue(value).map(String));
+    }
+    case "all": {
+      return collection.filter
+        .byProperty(key)
+        .containsAll(parseArrayValue(value).map(String));
+    }
+    default: {
+      throw new Error(`Operator ${operator} is not supported.`);
+    }
+  }
+};
+
+const toWeaviateFilter = (
+  collection: Collection,
+  standardFilters?: MetadataFilters,
+): FilterValue | undefined => {
+  if (!standardFilters?.filters.length) return undefined;
+  const filtersList = standardFilters.filters.map((filter) =>
+    buildFilterItem(collection, filter),
+  );
+  if (filtersList.length === 1) return filtersList[0];
+  const condition = standardFilters.condition ?? "and";
+  return Filters[condition](...filtersList);
+};
+
+export class WeaviateVectorStore
+  extends VectorStoreBase
+  implements VectorStoreNoEmbedModel
+{
+  public storesText: boolean = true;
+  private flatMetadata: boolean = true;
+
+  private weaviateClient?: WeaviateClient;
+  private clusterURL!: string;
+  private apiKey!: string;
+  private indexName: string;
+
+  private idKey: string;
+  private contentKey: string;
+  private embeddingKey: string;
+  private metadataKey: string;
+
+  constructor(
+    init?: Partial<IEmbedModel> & {
+      weaviateClient?: WeaviateClient;
+      cloudOptions?: {
+        clusterURL?: string;
+        apiKey?: string;
+      };
+      indexName?: string;
+      idKey?: string;
+      contentKey?: string;
+      metadataKey?: string;
+      embeddingKey?: string;
+    },
+  ) {
+    super(init?.embedModel);
+
+    if (init?.weaviateClient) {
+      // Use the provided client
+      this.weaviateClient = init.weaviateClient;
+    } else {
+      // Load client cloud options from config or env
+      const clusterURL =
+        init?.cloudOptions?.clusterURL ?? process.env.WEAVIATE_CLUSTER_URL;
+      const apiKey = init?.cloudOptions?.apiKey ?? process.env.WEAVIATE_API_KEY;
+      if (!clusterURL || !apiKey) {
+        throw new Error(
+          "Must specify WEAVIATE_CLUSTER_URL and WEAVIATE_API_KEY via env variable.",
+        );
+      }
+      this.clusterURL = clusterURL;
+      this.apiKey = apiKey;
+    }
+
+    this.checkIndexName(init?.indexName);
+    this.indexName = init?.indexName ?? "LlamaIndex";
+    this.idKey = init?.idKey ?? "id";
+    this.contentKey = init?.contentKey ?? "text";
+    this.embeddingKey = init?.embeddingKey ?? "vectors";
+    this.metadataKey = init?.metadataKey ?? "node_info";
+  }
+
+  public client() {
+    return this.getClient();
+  }
+
+  public async add(nodes: BaseNode<Metadata>[]): Promise<string[]> {
+    const collection = await this.ensureCollection({ createIfNotExists: true });
+
+    const result = await collection.data.insertMany(
+      nodes.map((node) => {
+        const metadata = nodeToMetadata(
+          node,
+          true,
+          this.contentKey,
+          this.flatMetadata,
+        );
+        const body = {
+          [this.idKey]: node.id_,
+          [this.embeddingKey]: node.getEmbedding(),
+          properties: {
+            ...metadata,
+            [this.contentKey]: node.getContent(MetadataMode.NONE),
+            [this.metadataKey]: JSON.stringify(metadata),
+            relationships: JSON.stringify({ ref_doc_id: metadata.ref_doc_id }),
+          },
+        };
+        return body;
+      }),
+    );
+
+    return Object.values(result.uuids);
+  }
+
+  public async delete(
+    refDocId: string,
+    deleteOptions?: DeleteManyOptions<boolean>,
+  ): Promise<void> {
+    const collection = await this.ensureCollection();
+    await collection.data.deleteMany(
+      collection.filter.byProperty("ref_doc_id").like(refDocId),
+      deleteOptions,
+    );
+  }
+
+  public async query(query: VectorStoreQuery): Promise<VectorStoreQueryResult> {
+    const collection = await this.ensureCollection();
+    const allProperties = await this.getAllProperties();
+
+    let filters: FilterValue | undefined = undefined;
+
+    if (query.docIds) {
+      filters = collection.filter
+        .byProperty("doc_id")
+        .containsAny(query.docIds);
+    }
+
+    if (query.filters) {
+      filters = toWeaviateFilter(collection, query.filters);
+    }
+
+    const queryResult = await collection.query.hybrid(query.queryStr!, {
+      vector: query.queryEmbedding,
+      alpha: this.getQueryAlpha(query),
+      limit: query.similarityTopK,
+      returnMetadata: Object.values(SIMILARITY_KEYS),
+      returnProperties: allProperties,
+      includeVector: true,
+      filters,
+    });
+
+    const entries = queryResult.objects;
+
+    const similarityKey = SIMILARITY_KEYS[query.mode];
+    const nodes: BaseNode<Metadata>[] = [];
+    const similarities: number[] = [];
+    const ids: string[] = [];
+
+    entries.forEach((entry, index) => {
+      if (index < query.similarityTopK && entry.metadata) {
+        const node = metadataDictToNode(entry.properties);
+        node.setContent(entry.properties[this.contentKey]);
+        nodes.push(node);
+        ids.push(entry.uuid);
+        similarities.push(this.getNodeSimilarity(entry, similarityKey));
+      }
+    });
+
+    return {
+      nodes,
+      similarities,
+      ids,
+    };
+  }
+
+  private async getClient(): Promise<WeaviateClient> {
+    if (this.weaviateClient) return this.weaviateClient;
+    const client = await weaviate.connectToWeaviateCloud(this.clusterURL, {
+      authCredentials: new weaviate.ApiKey(this.apiKey),
+    });
+    this.weaviateClient = client;
+    return client;
+  }
+
+  private async ensureCollection({ createIfNotExists = false } = {}) {
+    const client = await this.getClient();
+    const exists = await this.doesCollectionExist();
+    if (!exists) {
+      if (createIfNotExists) {
+        await this.createCollection();
+      } else {
+        throw new Error(`Collection ${this.indexName} does not exist.`);
+      }
+    }
+    return client.collections.get(this.indexName);
+  }
+
+  private async doesCollectionExist() {
+    const client = await this.getClient();
+    return client.collections.exists(this.indexName);
+  }
+
+  private async createCollection() {
+    const client = await this.getClient();
+    return await client.collections.createFromSchema({
+      class: this.indexName,
+      description: `Collection for ${this.indexName}`,
+      properties: NODE_SCHEMA,
+    });
+  }
+
+  private getQueryAlpha(query: VectorStoreQuery): number | undefined {
+    if (!query.queryEmbedding) return undefined;
+    if (query.mode === VectorStoreQueryMode.DEFAULT) return 1;
+    if (query.mode === VectorStoreQueryMode.HYBRID && query.queryStr)
+      return query.alpha;
+    return undefined;
+  }
+
+  private async getAllProperties(): Promise<string[]> {
+    const collection = await this.ensureCollection();
+    const properties = (await collection.config.get()).properties;
+    return properties.map((p) => p.name);
+  }
+
+  private checkIndexName(indexName?: string) {
+    if (indexName && indexName[0] !== indexName[0].toUpperCase()) {
+      throw new Error(
+        "Index name must start with a capital letter, e.g. 'LlamaIndex'",
+      );
+    }
+  }
+
+  private getNodeSimilarity(
+    entry: WeaviateNonGenericObject,
+    similarityKey: "distance" | "score" = "distance",
+  ): number {
+    const distance = entry.metadata?.[similarityKey];
+    if (distance === undefined) return 1;
+    // convert distance https://forum.weaviate.io/t/distance-vs-certainty-scores/258
+    return 1 - distance;
+  }
+}

packages/llamaindex/src/storage/vectorStore/types.ts

@@ -33,6 +33,7 @@ export enum FilterOperator {
   ALL = "all", // Contains all (array of strings)
   TEXT_MATCH = "text_match", // full text match (allows you to search for a specific substring, token or phrase within the text field)
   CONTAINS = "contains", // metadata array contains value (string or number)
+  IS_EMPTY = "is_empty", // the field is not exist or empty (null or empty array)
 }
 
 export enum FilterCondition {
@@ -44,7 +45,7 @@ export type MetadataFilterValue = string | number | string[] | number[];
 
 export interface MetadataFilter {
   key: string;
-  value: MetadataFilterValue;
+  value?: MetadataFilterValue;
   operator: `${FilterOperator}`; // ==, any, all,...
 }
 

packages/llamaindex/src/storage/vectorStore/utils.ts

@@ -80,7 +80,7 @@ export function metadataDictToNode(
 }
 
 export const parsePrimitiveValue = (
-  value: MetadataFilterValue,
+  value?: MetadataFilterValue,
 ): string | number => {
   if (typeof value !== "number" && typeof value !== "string") {
     throw new Error("Value must be a string or number");
@@ -89,7 +89,7 @@ export const parsePrimitiveValue = (
 };
 
 export const parseArrayValue = (
-  value: MetadataFilterValue,
+  value?: MetadataFilterValue,
 ): string[] | number[] => {
   const isPrimitiveArray =
     Array.isArray(value) &&
@@ -99,3 +99,8 @@ export const parseArrayValue = (
   }
   return value;
 };
+
+export const parseNumberValue = (value?: MetadataFilterValue): number => {
+  if (typeof value !== "number") throw new Error("Value must be a number");
+  return value;
+};

packages/llamaindex/tests/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/core-test
 
+## 0.0.7
+
+### Patch Changes
+
+- 01c184c: Add is_empty operator for filtering vector store
+
 ## 0.0.6
 
 ### Patch Changes

packages/llamaindex/tests/import.test.ts

@@ -0,0 +1,11 @@
+import { expect, test } from "vitest";
+
+test("Node classes should be included in the top level", async () => {
+  const { Document, IndexNode, TextNode, BaseNode } = await import(
+    "llamaindex"
+  );
+  expect(Document).toBeDefined();
+  expect(IndexNode).toBeDefined();
+  expect(TextNode).toBeDefined();
+  expect(BaseNode).toBeDefined();
+});

packages/llamaindex/tests/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llamaindex-test",
   "private": true,
-  "version": "0.0.6",
+  "version": "0.0.7",
   "type": "module",
   "scripts": {
     "test": "vitest run"

packages/llamaindex/tests/vectorStores/SimpleVectorStore.test.ts

@@ -256,6 +256,18 @@ describe("SimpleVectorStore", () => {
         },
         expected: 1,
       },
+      {
+        title: "Filter IS_EMPTY",
+        filters: {
+          filters: [
+            {
+              key: "not-exist-key",
+              operator: "is_empty",
+            },
+          ],
+        },
+        expected: 3,
+      },
       {
         title: "Filter OR",
         filters: {

pnpm-lock.yaml

@@ -148,7 +148,7 @@ importers:
         version: 2.4.4
       chromadb:
         specifier: ^1.8.1
-        version: 1.8.1(@google/generative-ai@0.12.0)(cohere-ai@7.10.6(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.52.5(encoding@0.1.13))
+        version: 1.8.1(@google/generative-ai@0.12.0)(cohere-ai@7.10.6(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.55.3(encoding@0.1.13)(zod@3.23.8))
       commander:
         specifier: ^12.1.0
         version: 12.1.0
@@ -163,7 +163,7 @@ importers:
         version: link:../packages/llamaindex
       mongodb:
         specifier: ^6.7.0
-        version: 6.8.0(@aws-sdk/credential-providers@3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0)))
+        version: 6.8.0(@aws-sdk/credential-providers@3.613.0)
       pathe:
         specifier: ^1.1.2
         version: 1.1.2
@@ -272,7 +272,7 @@ importers:
         version: 1.1.0(@types/react@18.3.3)(react@18.3.1)
       ai:
         specifier: ^3.2.1
-        version: 3.2.19(openai@4.52.5)(react@18.3.1)(svelte@4.2.18)(vue@3.4.31(typescript@5.5.3))(zod@3.23.8)
+        version: 3.2.19(openai@4.55.3(zod@3.23.8))(react@18.3.1)(svelte@4.2.18)(vue@3.4.31(typescript@5.5.3))(zod@3.23.8)
       class-variance-authority:
         specifier: ^0.7.0
         version: 0.7.0
@@ -541,7 +541,7 @@ importers:
         version: 4.6.0
       chromadb:
         specifier: 1.8.1
-        version: 1.8.1(@google/generative-ai@0.12.0)(cohere-ai@7.10.6(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.52.5(encoding@0.1.13))
+        version: 1.8.1(@google/generative-ai@0.12.0)(cohere-ai@7.10.6(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.55.3(encoding@0.1.13)(zod@3.23.8))
       cohere-ai:
         specifier: 7.10.6
         version: 7.10.6(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))(encoding@0.1.13)
@@ -568,13 +568,13 @@ importers:
         version: 2.0.0
       mongodb:
         specifier: ^6.7.0
-        version: 6.8.0(@aws-sdk/credential-providers@3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0)))
+        version: 6.8.0(@aws-sdk/credential-providers@3.613.0)
       notion-md-crawler:
         specifier: ^1.0.0
         version: 1.0.0(encoding@0.1.13)
       openai:
-        specifier: ^4.52.5
-        version: 4.52.5(encoding@0.1.13)
+        specifier: ^4.55.3
+        version: 4.55.3(encoding@0.1.13)(zod@3.23.8)
       papaparse:
         specifier: ^5.4.1
         version: 5.4.1
@@ -602,6 +602,9 @@ importers:
       unpdf:
         specifier: ^0.11.0
         version: 0.11.0(encoding@0.1.13)
+      weaviate-client:
+        specifier: ^3.1.4
+        version: 3.1.4(encoding@0.1.13)
       wikipedia:
         specifier: ^2.1.2
         version: 2.1.2
@@ -684,7 +687,7 @@ importers:
     dependencies:
       ai:
         specifier: ^3.2.1
-        version: 3.2.19(openai@4.52.5)(react@18.3.1)(svelte@4.2.18)(vue@3.4.31(typescript@5.5.3))(zod@3.23.8)
+        version: 3.2.19(openai@4.55.3(zod@3.23.8))(react@18.3.1)(svelte@4.2.18)(vue@3.4.31(typescript@5.5.3))(zod@3.23.8)
       llamaindex:
         specifier: workspace:*
         version: link:../../..
@@ -2666,6 +2669,11 @@ packages:
     resolution: {integrity: sha512-krWjurjEUHSFhCX4lGHMOhbnpBfYZGU31mpHpPBQwcfWm0T+/+wxC4UCAJfkxxc3/HvGJVG8r4AqrffaeDHDlA==}
     engines: {node: '>=18.0.0'}
 
+  '@graphql-typed-document-node/core@3.2.0':
+    resolution: {integrity: sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==}
+    peerDependencies:
+      graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0
+
   '@grpc/grpc-js@1.10.11':
     resolution: {integrity: sha512-3RaoxOqkHHN2c05bwtBNVJmOf/UwMam0rZYtdl7dsRpsvDwcNpv6LkGgzltQ7xVf822LzBoKEPRvf4D7+xeIDw==}
     engines: {node: '>=12.10.0'}
@@ -4375,6 +4383,9 @@ packages:
   abbrev@1.1.1:
     resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==}
 
+  abort-controller-x@0.4.3:
+    resolution: {integrity: sha512-VtUwTNU8fpMwvWGn4xE93ywbogTYsuT+AUxAXOeelbXuQVIwNmC5YLeho9sH4vZ4ITW8414TTAOG1nW6uIVHCA==}
+
   abort-controller@3.0.0:
     resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
     engines: {node: '>=6.5'}
@@ -6568,6 +6579,15 @@ packages:
   graphemer@1.4.0:
     resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==}
 
+  graphql-request@6.1.0:
+    resolution: {integrity: sha512-p+XPfS4q7aIpKVcgmnZKhMNqhltk20hfXtkaIkTfjjmiKMJ5xrt5c743cL03y/K7y1rg3WrIC49xGiEQ4mxdNw==}
+    peerDependencies:
+      graphql: 14 - 16
+
+  graphql@16.9.0:
+    resolution: {integrity: sha512-GGTKBX4SD7Wdb8mqeDLni2oaRGYQWjWHGKPQ24ZMnUtKfcsVoiv4uX8+LJr1K6U5VW2Lu1BwJnj7uiori0YtRw==}
+    engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0}
+
   gray-matter@4.0.3:
     resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==}
     engines: {node: '>=6.0'}
@@ -8126,6 +8146,15 @@ packages:
       sass:
         optional: true
 
+  nice-grpc-client-middleware-deadline@2.0.12:
+    resolution: {integrity: sha512-drKxQJzTbh+Qkd6v+BcRhTmY2mw9zR8Qigu/jk0vIkDi90K6NOOJGgvBdbTxKXtv6QY1g07T1LvwaqW3Mlwdvw==}
+
+  nice-grpc-common@2.0.2:
+    resolution: {integrity: sha512-7RNWbls5kAL1QVUOXvBsv1uO0wPQK3lHv+cY1gwkTzirnG1Nop4cBJZubpgziNbaVc/bl9QJcyvsf/NQxa3rjQ==}
+
+  nice-grpc@2.1.9:
+    resolution: {integrity: sha512-shJlg1t4Wn3qTVE31gxofbTrgCX/p4tS1xRnk4bNskCYKvXNEUpJQZpjModsVk1aau69YZDViyC18K9nC7QHYA==}
+
   nice-napi@1.0.2:
     resolution: {integrity: sha512-px/KnJAJZf5RuBGcfD+Sp2pAKq0ytz8j+1NehvgIGFkvtvFrDM3T8E4x/JJODXK9WZow8RRGrbA9QQ3hs+pDhA==}
     os: ['!win32']
@@ -8330,6 +8359,15 @@ packages:
     resolution: {integrity: sha512-qqH8GsyPE3z06took/2uWOGqRcrZNlRoPAsihpg4jsl0+2Dfelnw6HDDMep0EI2Cfzw75nn3vHRZehep/IZzxg==}
     hasBin: true
 
+  openai@4.55.3:
+    resolution: {integrity: sha512-/IUDdK5w3aB1Kd88Ml7w5F+EkVM5aXlrY+lSpWXdIPL18CmGkC7lV9HFJ7beR0OUSFLFT0qmWvMynqtbMF06/Q==}
+    hasBin: true
+    peerDependencies:
+      zod: ^3.23.8
+    peerDependenciesMeta:
+      zod:
+        optional: true
+
   opener@1.5.2:
     resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==}
     hasBin: true
@@ -10304,6 +10342,9 @@ packages:
     peerDependencies:
       typescript: '>=4.2.0'
 
+  ts-error@1.0.6:
+    resolution: {integrity: sha512-tLJxacIQUM82IR7JO1UUkKlYuUTmoY9HBJAmNWFzheSlDS5SPMcNIepejHJa4BpPQLAcbRhRf3GDJzyj6rbKvA==}
+
   ts-graphviz@1.8.2:
     resolution: {integrity: sha512-5YhbFoHmjxa7pgQLkB07MtGnGJ/yhvjmc9uhsnDBEICME6gkPf83SBwLDQqGDoCa3XzUMWLk1AU2Wn1u1naDtA==}
     engines: {node: '>=14.16'}
@@ -10779,6 +10820,10 @@ packages:
   wcwidth@1.0.1:
     resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==}
 
+  weaviate-client@3.1.4:
+    resolution: {integrity: sha512-Bw9KV0wtFd4TdifhPAkmc2Lv7bKIX0L2oqObUNG8K8Mv0zoVixGcqlAS3xJdfQ2jSqz0vH3mfetsOBdlvogxfg==}
+    engines: {node: '>=18.0.0'}
+
   web-namespaces@2.0.1:
     resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==}
 
@@ -13978,6 +14023,10 @@ snapshots:
 
   '@google/generative-ai@0.12.0': {}
 
+  '@graphql-typed-document-node/core@3.2.0(graphql@16.9.0)':
+    dependencies:
+      graphql: 16.9.0
+
   '@grpc/grpc-js@1.10.11':
     dependencies:
       '@grpc/proto-loader': 0.7.13
@@ -15720,7 +15769,7 @@ snapshots:
       '@vue/shared': 3.4.31
       estree-walker: 2.0.2
       magic-string: 0.30.11
-      postcss: 8.4.40
+      postcss: 8.4.39
       source-map-js: 1.2.0
 
   '@vue/compiler-ssr@3.4.31':
@@ -15856,6 +15905,8 @@ snapshots:
   abbrev@1.1.1:
     optional: true
 
+  abort-controller-x@0.4.3: {}
+
   abort-controller@3.0.0:
     dependencies:
       event-target-shim: 5.0.1
@@ -15911,7 +15962,7 @@ snapshots:
       clean-stack: 2.2.0
       indent-string: 4.0.0
 
-  ai@3.2.19(openai@4.52.5)(react@18.3.1)(svelte@4.2.18)(vue@3.4.31(typescript@5.5.3))(zod@3.23.8):
+  ai@3.2.19(openai@4.55.3(zod@3.23.8))(react@18.3.1)(svelte@4.2.18)(vue@3.4.31(typescript@5.5.3))(zod@3.23.8):
     dependencies:
       '@ai-sdk/provider': 0.0.12
       '@ai-sdk/provider-utils': 1.0.2(zod@3.23.8)
@@ -15928,7 +15979,7 @@ snapshots:
       sswr: 2.1.0(svelte@4.2.18)
       zod-to-json-schema: 3.22.5(zod@3.23.8)
     optionalDependencies:
-      openai: 4.52.5(encoding@0.1.13)
+      openai: 4.55.3(zod@3.23.8)
       react: 18.3.1
       svelte: 4.2.18
       zod: 3.23.8
@@ -16623,14 +16674,14 @@ snapshots:
 
   chownr@2.0.0: {}
 
-  chromadb@1.8.1(@google/generative-ai@0.12.0)(cohere-ai@7.10.6(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.52.5(encoding@0.1.13)):
+  chromadb@1.8.1(@google/generative-ai@0.12.0)(cohere-ai@7.10.6(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.55.3(encoding@0.1.13)(zod@3.23.8)):
     dependencies:
       cliui: 8.0.1
       isomorphic-fetch: 3.0.0(encoding@0.1.13)
     optionalDependencies:
       '@google/generative-ai': 0.12.0
       cohere-ai: 7.10.6(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))(encoding@0.1.13)
-      openai: 4.52.5(encoding@0.1.13)
+      openai: 4.55.3(encoding@0.1.13)(zod@3.23.8)
     transitivePeerDependencies:
       - encoding
 
@@ -17691,16 +17742,6 @@ snapshots:
       - eslint-import-resolver-webpack
       - supports-color
 
-  eslint-module-utils@2.8.1(@typescript-eslint/parser@7.16.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0):
-    dependencies:
-      debug: 3.2.7
-    optionalDependencies:
-      '@typescript-eslint/parser': 7.16.0(eslint@8.57.0)(typescript@5.5.3)
-      eslint: 8.57.0
-      eslint-import-resolver-node: 0.3.9
-    transitivePeerDependencies:
-      - supports-color
-
   eslint-module-utils@2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
       debug: 3.2.7
@@ -17722,7 +17763,7 @@ snapshots:
       doctrine: 2.1.0
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.16.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0)
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
       hasown: 2.0.2
       is-core-module: 2.14.0
       is-glob: 4.0.3
@@ -18616,6 +18657,16 @@ snapshots:
 
   graphemer@1.4.0: {}
 
+  graphql-request@6.1.0(encoding@0.1.13)(graphql@16.9.0):
+    dependencies:
+      '@graphql-typed-document-node/core': 3.2.0(graphql@16.9.0)
+      cross-fetch: 3.1.8(encoding@0.1.13)
+      graphql: 16.9.0
+    transitivePeerDependencies:
+      - encoding
+
+  graphql@16.9.0: {}
+
   gray-matter@4.0.3:
     dependencies:
       js-yaml: 3.14.1
@@ -20412,7 +20463,7 @@ snapshots:
     optionalDependencies:
       '@aws-sdk/credential-providers': 3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))
 
-  mongodb@6.8.0(@aws-sdk/credential-providers@3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))):
+  mongodb@6.8.0(@aws-sdk/credential-providers@3.613.0):
     dependencies:
       '@mongodb-js/saslprep': 1.1.7
       bson: 6.8.0
@@ -20590,6 +20641,20 @@ snapshots:
       - '@babel/core'
       - babel-plugin-macros
 
+  nice-grpc-client-middleware-deadline@2.0.12:
+    dependencies:
+      nice-grpc-common: 2.0.2
+
+  nice-grpc-common@2.0.2:
+    dependencies:
+      ts-error: 1.0.6
+
+  nice-grpc@2.1.9:
+    dependencies:
+      '@grpc/grpc-js': 1.10.11
+      abort-controller-x: 0.4.3
+      nice-grpc-common: 2.0.2
+
   nice-napi@1.0.2:
     dependencies:
       node-addon-api: 3.2.1
@@ -20693,7 +20758,7 @@ snapshots:
       execa: 8.0.1
       pathe: 1.1.2
       pkg-types: 1.1.3
-      ufo: 1.5.3
+      ufo: 1.5.4
 
   object-assign@4.1.1: {}
 
@@ -20824,6 +20889,35 @@ snapshots:
     transitivePeerDependencies:
       - encoding
 
+  openai@4.55.3(encoding@0.1.13)(zod@3.23.8):
+    dependencies:
+      '@types/node': 18.19.39
+      '@types/node-fetch': 2.6.11
+      abort-controller: 3.0.0
+      agentkeepalive: 4.5.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.7.0(encoding@0.1.13)
+    optionalDependencies:
+      zod: 3.23.8
+    transitivePeerDependencies:
+      - encoding
+
+  openai@4.55.3(zod@3.23.8):
+    dependencies:
+      '@types/node': 18.19.39
+      '@types/node-fetch': 2.6.11
+      abort-controller: 3.0.0
+      agentkeepalive: 4.5.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.7.0(encoding@0.1.13)
+    optionalDependencies:
+      zod: 3.23.8
+    transitivePeerDependencies:
+      - encoding
+    optional: true
+
   opener@1.5.2: {}
 
   option@0.2.4: {}
@@ -23053,6 +23147,8 @@ snapshots:
     dependencies:
       typescript: 5.5.3
 
+  ts-error@1.0.6: {}
+
   ts-graphviz@1.8.2: {}
 
   ts-interface-checker@0.1.13: {}
@@ -23565,6 +23661,17 @@ snapshots:
     dependencies:
       defaults: 1.0.4
 
+  weaviate-client@3.1.4(encoding@0.1.13):
+    dependencies:
+      graphql: 16.9.0
+      graphql-request: 6.1.0(encoding@0.1.13)(graphql@16.9.0)
+      long: 5.2.3
+      nice-grpc: 2.1.9
+      nice-grpc-client-middleware-deadline: 2.0.12
+      uuid: 9.0.1
+    transitivePeerDependencies:
+      - encoding
+
   web-namespaces@2.0.1: {}
 
   web-streams-polyfill@3.3.3: {}