Release 0.5.18 (#1124)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/chilled-tomatoes-visit.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-feat: add splitByPage mode to LlamaParseReader

.eslintrc.cjs

@@ -31,7 +31,12 @@ module.exports = {
     "@typescript-eslint/ban-types": "off",
     "no-array-constructor": "off",
     "@typescript-eslint/no-array-constructor": "off",
-    "@typescript-eslint/no-base-to-string": "off",
+    "@typescript-eslint/no-base-to-string": [
+      "error",
+      {
+        ignoredTypeNames: ["Error", "RegExp", "URL", "URLSearchParams"],
+      },
+    ],
     "@typescript-eslint/no-duplicate-enum-values": "off",
     "@typescript-eslint/no-duplicate-type-constituents": "off",
     "@typescript-eslint/no-explicit-any": "off",

README.md

@@ -164,7 +164,7 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
 
 - [Node](/packages/llamaindex/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
 
-- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
+- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that question. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
 
 - [Indices](/packages/llamaindex/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
 

apps/docs/CHANGELOG.md

@@ -1,5 +1,56 @@
 # docs
 
+## 0.0.59
+
+### Patch Changes
+
+- Updated dependencies [8b66cf4]
+  - llamaindex@0.5.18
+
+## 0.0.58
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.57
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.56
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.55
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.54
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
 ## 0.0.52
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.52",
+  "version": "0.0.59",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/weaviate/README.md

@@ -0,0 +1,31 @@
+# Weaviate Vector Store
+
+Here are two sample scripts which work with loading and querying data from a Weaviate Vector Store.
+
+## Prerequisites
+
+- An Weaviate Vector Database
+  - Hosted https://weaviate.io/
+  - Self Hosted https://weaviate.io/developers/weaviate/installation/docker-compose#starter-docker-compose-file
+- An OpenAI API Key
+
+## Setup
+
+1. Set your env variables:
+
+- `WEAVIATE_CLUSTER_URL`: Address of your Weaviate Vector Store (like localhost:8080)
+- `WEAVIATE_API_KEY`: Your Weaviate API key
+- `OPENAI_API_KEY`: Your OpenAI key
+
+2. `cd` Into the `examples` directory
+3. run `npm i`
+
+## Load the data
+
+This sample loads the same dataset of movie reviews as sample dataset
+
+run `npx tsx weaviate/load`
+
+## Use RAG to Query the data
+
+run `npx tsx weaviate/query`

examples/weaviate/load.ts

@@ -0,0 +1,23 @@
+import {
+  PapaCSVReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+  WeaviateVectorStore,
+} from "llamaindex";
+
+const indexName = "MovieReviews";
+
+async function main() {
+  try {
+    const reader = new PapaCSVReader(false);
+    const docs = await reader.loadData("./data/movie_reviews.csv");
+    const vectorStore = new WeaviateVectorStore({ indexName });
+    const storageContext = await storageContextFromDefaults({ vectorStore });
+    await VectorStoreIndex.fromDocuments(docs, { storageContext });
+    console.log("Successfully loaded data into Weaviate");
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+void main();

examples/weaviate/query.ts

@@ -0,0 +1,46 @@
+import { VectorStoreIndex, WeaviateVectorStore } from "llamaindex";
+
+const indexName = "MovieReviews";
+
+async function main() {
+  try {
+    const query = "Get all movie titles.";
+    const vectorStore = new WeaviateVectorStore({ indexName });
+    const index = await VectorStoreIndex.fromVectorStore(vectorStore);
+    const retriever = index.asRetriever({ similarityTopK: 20 });
+
+    const queryEngine = index.asQueryEngine({ retriever });
+    const results = await queryEngine.query({ query });
+    console.log(`Query from ${results.sourceNodes?.length} nodes`);
+    console.log(results.response);
+
+    console.log("\n=====\nQuerying the index with filters");
+    const queryEngineWithFilters = index.asQueryEngine({
+      retriever,
+      preFilters: {
+        filters: [
+          {
+            key: "document_id",
+            value: "./data/movie_reviews.csv_37",
+            operator: "==",
+          },
+          {
+            key: "document_id",
+            value: "./data/movie_reviews.csv_21",
+            operator: "==",
+          },
+        ],
+        condition: "or",
+      },
+    });
+    const resultAfterFilter = await queryEngineWithFilters.query({
+      query: "Get all movie titles.",
+    });
+    console.log(`Query from ${resultAfterFilter.sourceNodes?.length} nodes`);
+    console.log(resultAfterFilter.response);
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+void main();

packages/autotool/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/autotool
 
+## 2.0.1
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
 ## 2.0.0
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,63 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.43
+
+### Patch Changes
+
+- Updated dependencies [8b66cf4]
+  - llamaindex@0.5.18
+  - @llamaindex/autotool@2.0.1
+
+## 0.1.42
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+  - @llamaindex/autotool@2.0.1
+
+## 0.1.41
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - @llamaindex/autotool@2.0.1
+  - llamaindex@0.5.16
+
+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.39
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.38
+
+### Patch Changes
+
+- llamaindex@0.5.13
+- @llamaindex/autotool@2.0.0
+
+## 0.1.37
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+  - @llamaindex/autotool@2.0.0
+
 ## 0.1.36
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.36",
+  "version": "0.1.43",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",
@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.5.11",
+    "llamaindex": "^0.5.18",
     "openai": "^4",
     "typescript": "^4"
   },
@@ -70,7 +70,7 @@
     "@swc/types": "^0.1.8",
     "@types/json-schema": "^7.0.15",
     "@types/node": "^20.12.11",
-    "bunchee": "5.3.0-beta.0",
+    "bunchee": "5.3.1",
     "llamaindex": "workspace:*",
     "next": "14.2.5",
     "rollup": "^4.18.0",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/cloud
 
+## 0.2.2
+
+### Patch Changes
+
+- 58abc57: fix: align version
+
 ## 0.2.1
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "type": "module",
   "license": "MIT",
   "scripts": {
@@ -35,6 +35,6 @@
   },
   "devDependencies": {
     "@hey-api/openapi-ts": "^0.48.0",
-    "bunchee": "5.3.0-beta.0"
+    "bunchee": "5.3.1"
   }
 }

packages/community/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/community
 
+## 0.0.30
+
+### Patch Changes
+
+- Updated dependencies [e27e7dd]
+  - @llamaindex/core@0.1.9
+
+## 0.0.29
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - @llamaindex/core@0.1.8
+  - @llamaindex/env@0.1.9
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [04b2f8e]
+  - @llamaindex/core@0.1.7
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [0452af9]
+  - @llamaindex/core@0.1.6
+
 ## 0.0.26
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.26",
+  "version": "0.0.30",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -43,7 +43,7 @@
   },
   "devDependencies": {
     "@types/node": "^20.14.2",
-    "bunchee": "5.3.0-beta.0"
+    "bunchee": "5.3.1"
   },
   "dependencies": {
     "@aws-sdk/client-bedrock-runtime": "^3.613.0",

packages/core/CHANGELOG.md

@@ -1,5 +1,31 @@
 # @llamaindex/core
 
+## 0.1.9
+
+### Patch Changes
+
+- e27e7dd: chore: bump `natural` to 8.0.1
+
+## 0.1.8
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - @llamaindex/env@0.1.9
+
+## 0.1.7
+
+### Patch Changes
+
+- 04b2f8e: Fix issue with metadata included after sentence splitter
+
+## 0.1.6
+
+### Patch Changes
+
+- 0452af9: fix: handling errors in splitBySentenceTokenizer
+
 ## 0.1.5
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.1.5",
+  "version": "0.1.9",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./node-parser": {
@@ -131,8 +131,8 @@
   },
   "devDependencies": {
     "ajv": "^8.16.0",
-    "bunchee": "5.3.0-beta.0",
-    "natural": "^7.1.0"
+    "bunchee": "5.3.1",
+    "natural": "^8.0.1"
   },
   "dependencies": {
     "@llamaindex/env": "workspace:*",

packages/core/src/node-parser/base.ts

@@ -140,7 +140,7 @@ export abstract class MetadataAwareTextSplitter extends TextSplitter {
     return nodes.reduce<TextNode[]>((allNodes, node) => {
       const metadataStr = this.getMetadataString(node);
       const splits = this.splitTextMetadataAware(
-        node.getContent(MetadataMode.ALL),
+        node.getContent(MetadataMode.NONE),
         metadataStr,
       );
       return allNodes.concat(buildNodeFromSplits(splits, node));

packages/core/src/node-parser/sentence_tokenizer.d.ts

@@ -1,4 +1,5 @@
 declare class SentenceTokenizer {
+  constructor(abbreviations?: string[]);
   tokenize(text: string): string[];
 }
 

packages/core/src/node-parser/sentence_tokenizer.js

@@ -0,0 +1,222 @@
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __commonJS = (cb, mod) =>
+  function __require() {
+    return (
+      mod ||
+        (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod),
+      mod.exports
+    );
+  };
+
+// lib/natural/tokenizers/tokenizer.js
+var require_tokenizer = __commonJS({
+  "lib/natural/tokenizers/tokenizer.js"(exports, module) {
+    "use strict";
+    var Tokenizer = class {
+      trim(array) {
+        while (array[array.length - 1] === "") {
+          array.pop();
+        }
+        while (array[0] === "") {
+          array.shift();
+        }
+        return array;
+      }
+    };
+    module.exports = Tokenizer;
+  },
+});
+
+// lib/natural/tokenizers/sentence_tokenizer.js
+var require_sentence_tokenizer = __commonJS({
+  "lib/natural/tokenizers/sentence_tokenizer.js"(exports, module) {
+    var Tokenizer = require_tokenizer();
+    var NUM = "NUMBER";
+    var DELIM = "DELIM";
+    var URI = "URI";
+    var ABBREV = "ABBREV";
+    var DEBUG = false;
+    function generateUniqueCode(base, index) {
+      return `{{${base}_${index}}}`;
+    }
+    function escapeRegExp(string) {
+      return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    }
+    var SentenceTokenizer = class extends Tokenizer {
+      constructor(abbreviations) {
+        super();
+        if (abbreviations) {
+          this.abbreviations = abbreviations;
+        } else {
+          this.abbreviations = [];
+        }
+        this.replacementMap = null;
+        this.replacementCounter = 0;
+      }
+      replaceUrisWithPlaceholders(text) {
+        const urlPattern =
+          /(https?:\/\/\S+|www\.\S+|ftp:\/\/\S+|(mailto:)?[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|file:\/\/\S+)/gi;
+        const modifiedText = text.replace(urlPattern, (match) => {
+          const placeholder = generateUniqueCode(
+            URI,
+            this.replacementCounter++,
+          );
+          this.replacementMap.set(placeholder, match);
+          return placeholder;
+        });
+        return modifiedText;
+      }
+      replaceAbbreviations(text) {
+        if (this.abbreviations.length === 0) {
+          return text;
+        }
+        const pattern = new RegExp(
+          `(${this.abbreviations.map((abbrev) => escapeRegExp(abbrev)).join("|")})`,
+          "gi",
+        );
+        const replacedText = text.replace(pattern, (match) => {
+          const code = generateUniqueCode(ABBREV, this.replacementCounter++);
+          this.replacementMap.set(code, match);
+          return code;
+        });
+        return replacedText;
+      }
+      replaceDelimitersWithPlaceholders(text) {
+        const delimiterPattern = /([.?!… ]*)([.?!…])(["'”’)}\]]?)/g;
+        const modifiedText = text.replace(
+          delimiterPattern,
+          (match, p1, p2, p3) => {
+            const placeholder = generateUniqueCode(
+              DELIM,
+              this.replacementCounter++,
+            );
+            this.delimiterMap.set(placeholder, p1 + p2 + p3);
+            return placeholder;
+          },
+        );
+        return modifiedText;
+      }
+      splitOnPlaceholders(text, placeholders) {
+        if (this.delimiterMap.size === 0) {
+          return [text];
+        }
+        const keys = Array.from(this.delimiterMap.keys());
+        const pattern = new RegExp(`(${keys.map(escapeRegExp).join("|")})`);
+        const parts = text.split(pattern);
+        const sentences = [];
+        for (let i = 0; i < parts.length; i += 2) {
+          const sentence = parts[i];
+          const placeholder = parts[i + 1] || "";
+          sentences.push(sentence + placeholder);
+        }
+        return sentences;
+      }
+      replaceNumbersWithCode(text) {
+        const numberPattern = /\b\d{1,3}(?:,\d{3})*(?:\.\d+)?\b/g;
+        const replacedText = text.replace(numberPattern, (match) => {
+          const code = generateUniqueCode(NUM, this.replacementCounter++);
+          this.replacementMap.set(code, match);
+          return code;
+        });
+        return replacedText;
+      }
+      revertReplacements(text) {
+        let originalText = text;
+        for (const [
+          placeholder,
+          replacement,
+        ] of this.replacementMap.entries()) {
+          const pattern = new RegExp(escapeRegExp(placeholder), "g");
+          originalText = originalText.replace(pattern, replacement);
+        }
+        return originalText;
+      }
+      revertDelimiters(text) {
+        let originalText = text;
+        for (const [placeholder, replacement] of this.delimiterMap.entries()) {
+          const pattern = new RegExp(escapeRegExp(placeholder), "g");
+          originalText = originalText.replace(pattern, replacement);
+        }
+        return originalText;
+      }
+      tokenize(text) {
+        this.replacementCounter = 0;
+        this.replacementMap = /* @__PURE__ */ new Map();
+        this.delimiterMap = /* @__PURE__ */ new Map();
+        DEBUG &&
+          console.log(
+            "---Start of sentence tokenization-----------------------",
+          );
+        DEBUG && console.log("Original input: >>>" + text + "<<<");
+        const result1 = this.replaceAbbreviations(text);
+        DEBUG &&
+          console.log(
+            "Phase 1: replacing abbreviations: " + JSON.stringify(result1),
+          );
+        const result2 = this.replaceUrisWithPlaceholders(result1);
+        DEBUG &&
+          console.log("Phase 2: replacing URIs: " + JSON.stringify(result2));
+        const result3 = this.replaceNumbersWithCode(result2);
+        DEBUG &&
+          console.log(
+            "Phase 3: replacing numbers with placeholders: " +
+              JSON.stringify(result3),
+          );
+        const result4 = this.replaceDelimitersWithPlaceholders(result3);
+        DEBUG &&
+          console.log(
+            "Phase 4: replacing delimiters with placeholders: " +
+              JSON.stringify(result4),
+          );
+        const sentences = this.splitOnPlaceholders(result4);
+        DEBUG &&
+          console.log(
+            "Phase 5: splitting into sentences on placeholders: " +
+              JSON.stringify(sentences),
+          );
+        const newSentences = sentences.map((s) => {
+          const s1 = this.revertReplacements(s);
+          return this.revertDelimiters(s1);
+        });
+        DEBUG &&
+          console.log(
+            "Phase 6: replacing back abbreviations, URIs, numbers and delimiters: " +
+              JSON.stringify(newSentences),
+          );
+        const trimmedSentences = this.trim(newSentences);
+        DEBUG &&
+          console.log(
+            "Phase 7: trimming array of empty sentences: " +
+              JSON.stringify(trimmedSentences),
+          );
+        const trimmedSentences2 = trimmedSentences.map((sent) => sent.trim());
+        DEBUG &&
+          console.log(
+            "Phase 8: trimming sentences from surrounding whitespace: " +
+              JSON.stringify(trimmedSentences2),
+          );
+        DEBUG &&
+          console.log(
+            "---End of sentence tokenization--------------------------",
+          );
+        DEBUG &&
+          console.log(
+            "---Replacement map---------------------------------------",
+          );
+        DEBUG && console.log([...this.replacementMap.entries()]);
+        DEBUG &&
+          console.log(
+            "---Delimiter map-----------------------------------------",
+          );
+        DEBUG && console.log([...this.delimiterMap.entries()]);
+        DEBUG &&
+          console.log(
+            "---------------------------------------------------------",
+          );
+        return trimmedSentences2;
+      }
+    };
+    module.exports = SentenceTokenizer;
+  },
+});
+export default require_sentence_tokenizer();

packages/core/src/node-parser/utils.ts

@@ -1,5 +1,5 @@
 import type { TextSplitter } from "./base";
-import SentenceTokenizerNew from "./sentence-tokenizer-parser.js";
+import SentenceTokenizer from "./sentence_tokenizer";
 
 export type TextSplitterFn = (text: string) => string[];
 
@@ -31,15 +31,25 @@ export const splitByChar = (): TextSplitterFn => {
   return (text: string) => text.split("");
 };
 
-let sentenceTokenizer: SentenceTokenizerNew | null = null;
+let sentenceTokenizer: SentenceTokenizer | null = null;
 
 export const splitBySentenceTokenizer = (): TextSplitterFn => {
   if (!sentenceTokenizer) {
-    sentenceTokenizer = new SentenceTokenizerNew();
+    sentenceTokenizer = new SentenceTokenizer([
+      "i.e.",
+      "etc.",
+      "vs.",
+      "Inc.",
+      "A.S.A.P.",
+    ]);
   }
   const tokenizer = sentenceTokenizer;
   return (text: string) => {
-    return tokenizer.tokenize(text);
+    try {
+      return tokenizer.tokenize(text);
+    } catch {
+      return [text];
+    }
   };
 };
 

packages/core/src/schema/index.ts

@@ -1,4 +1,4 @@
 export * from "./node";
-export { TransformComponent } from "./type";
+export { FileReader, TransformComponent, type BaseReader } from "./type";
 export { EngineResponse } from "./type/engine–response";
 export * from "./zod";

packages/core/src/schema/type.ts

@@ -1,5 +1,5 @@
-import { randomUUID } from "@llamaindex/env";
-import type { BaseNode } from "./node";
+import { fs, path, randomUUID } from "@llamaindex/env";
+import type { BaseNode, Document } from "./node";
 
 interface TransformComponentSignature {
   <Options extends Record<string, unknown>>(
@@ -28,3 +28,37 @@ export class TransformComponent {
     return transform;
   }
 }
+
+/**
+ * A reader takes imports data into Document objects.
+ */
+export interface BaseReader {
+  loadData(...args: unknown[]): Promise<Document[]>;
+}
+
+/**
+ * A FileReader takes file paths and imports data into Document objects.
+ */
+export abstract class FileReader implements BaseReader {
+  abstract loadDataAsContent(
+    fileContent: Uint8Array,
+    fileName?: string,
+  ): Promise<Document[]>;
+
+  async loadData(filePath: string): Promise<Document[]> {
+    const fileContent = await fs.readFile(filePath);
+    const fileName = path.basename(filePath);
+    const docs = await this.loadDataAsContent(fileContent, fileName);
+    docs.forEach(FileReader.addMetaData(filePath));
+    return docs;
+  }
+
+  static addMetaData(filePath: string) {
+    return (doc: Document, index: number) => {
+      // generate id as loadDataAsContent is only responsible for the content
+      doc.id_ = `${filePath}_${index + 1}`;
+      doc.metadata["file_path"] = path.resolve(filePath);
+      doc.metadata["file_name"] = path.basename(filePath);
+    };
+  }
+}

packages/core/tests/node-parser/text-splitter.test.ts

@@ -1,7 +1,10 @@
-import { SentenceSplitter } from "@llamaindex/core/node-parser";
+import {
+  SentenceSplitter,
+  splitBySentenceTokenizer,
+} from "@llamaindex/core/node-parser";
 import { describe, expect, test } from "vitest";
 
-describe("SentenceSplitter", () => {
+describe("sentence splitter", () => {
   test("initializes", () => {
     const sentenceSplitter = new SentenceSplitter();
     expect(sentenceSplitter).toBeDefined();
@@ -105,4 +108,11 @@ describe("SentenceSplitter", () => {
       "因为他照了人类，连我都在内。",
     ]);
   });
+
+  test("issue 1087 - edge case when input with brackets", () => {
+    const text =
+      "A card must be of uniform thickness and made of unfolded and uncreased paper or cardstock of approximately the quality and weight of a stamped card (i.e., a card available from USPS).";
+    const split = splitBySentenceTokenizer();
+    expect(split(text)).toEqual([text]);
+  });
 });

packages/env/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/env
 
+## 0.1.9
+
+### Patch Changes
+
+- 58abc57: fix: align version
+
 ## 0.1.8
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.8",
+  "version": "0.1.9",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/experimental/CHANGELOG.md

@@ -1,5 +1,57 @@
 # @llamaindex/experimental
 
+## 0.0.68
+
+### Patch Changes
+
+- Updated dependencies [8b66cf4]
+  - llamaindex@0.5.18
+
+## 0.0.67
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.66
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.65
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.64
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.63
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.62
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
 ## 0.0.61
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.61",
+  "version": "0.0.68",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/experimental/src/engines/query/JSONQueryEngine.ts

@@ -162,18 +162,18 @@ export class JSONQueryEngine implements QueryEngine {
 
     const schema = this.getSchemaContext();
 
-    const jsonPathResponseStr = await this.serviceContext.llm.complete({
+    const { text: jsonPathResponse } = await this.serviceContext.llm.complete({
       prompt: this.jsonPathPrompt({ query, schema }),
     });
 
     if (this.verbose) {
       console.log(
-        `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponseStr}\n\`\`\`\n`,
+        `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponse}\n\`\`\`\n`,
       );
     }
 
     const jsonPathOutput = await this.outputProcessor({
-      llmOutput: jsonPathResponseStr.text,
+      llmOutput: jsonPathResponse,
       jsonValue: this.jsonValue,
     });
 
@@ -188,7 +188,7 @@ export class JSONQueryEngine implements QueryEngine {
         prompt: this.responseSynthesisPrompt({
           query,
           jsonSchema: schema,
-          jsonPath: jsonPathResponseStr.text,
+          jsonPath: jsonPathResponse,
           jsonPathValue: JSON.stringify(jsonPathOutput),
         }),
       });
@@ -199,7 +199,7 @@ export class JSONQueryEngine implements QueryEngine {
     }
 
     const responseMetadata = {
-      jsonPathResponseStr,
+      jsonPathResponse,
     };
 
     const response = EngineResponse.fromResponse(responseStr, false);

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,59 @@
 # llamaindex
 
+## 0.5.18
+
+### Patch Changes
+
+- 8b66cf4: feat: support organization id in llamacloud index
+- Updated dependencies [e27e7dd]
+  - @llamaindex/core@0.1.9
+
+## 0.5.17
+
+### Patch Changes
+
+- c654398: Implement Weaviate Vector Store in TS
+
+## 0.5.16
+
+### Patch Changes
+
+- 58abc57: fix: align version
+- Updated dependencies [58abc57]
+  - @llamaindex/cloud@0.2.2
+  - @llamaindex/core@0.1.8
+  - @llamaindex/env@0.1.9
+
+## 0.5.15
+
+### Patch Changes
+
+- 01c184c: Add is_empty operator for filtering vector store
+- 07a275f: chore: bump openai
+
+## 0.5.14
+
+### Patch Changes
+
+- c825a2f: Add gpt-4o-mini to Azure. Add 2024-06-01 API version for Azure
+
+## 0.5.13
+
+### Patch Changes
+
+- Updated dependencies [04b2f8e]
+  - @llamaindex/core@0.1.7
+
+## 0.5.12
+
+### Patch Changes
+
+- 345300f: feat: add splitByPage mode to LlamaParseReader
+- da5cfc4: Add metadatafilter options to retriever constructors
+- da5cfc4: Fix system prompt not used in ContextChatEngine
+- Updated dependencies [0452af9]
+  - @llamaindex/core@0.1.6
+
 ## 0.5.11
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.52
+
+### Patch Changes
+
+- Updated dependencies [8b66cf4]
+  - llamaindex@0.5.18
+
+## 0.0.51
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.50
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.49
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.47
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
 ## 0.0.45
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.45",
+  "version": "0.0.52",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/next-agent-test
 
+## 0.1.52
+
+### Patch Changes
+
+- Updated dependencies [8b66cf4]
+  - llamaindex@0.5.18
+
+## 0.1.51
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.1.50
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.1.49
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.1.48
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.1.47
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.1.46
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
 ## 0.1.45
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.45",
+  "version": "0.1.52",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,56 @@
 # test-edge-runtime
 
+## 0.1.51
+
+### Patch Changes
+
+- Updated dependencies [8b66cf4]
+  - llamaindex@0.5.18
+
+## 0.1.50
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.1.49
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.1.48
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.1.47
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.1.46
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.1.45
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
 ## 0.1.44
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.44",
+  "version": "0.1.51",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.33
+
+### Patch Changes
+
+- Updated dependencies [8b66cf4]
+  - llamaindex@0.5.18
+
+## 0.0.32
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.31
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.30
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.28
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
 ## 0.0.26
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.26",
+  "version": "0.0.33",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.52
+
+### Patch Changes
+
+- Updated dependencies [8b66cf4]
+  - llamaindex@0.5.18
+
+## 0.0.51
+
+### Patch Changes
+
+- Updated dependencies [c654398]
+  - llamaindex@0.5.17
+
+## 0.0.50
+
+### Patch Changes
+
+- Updated dependencies [58abc57]
+  - llamaindex@0.5.16
+
+## 0.0.49
+
+### Patch Changes
+
+- Updated dependencies [01c184c]
+- Updated dependencies [07a275f]
+  - llamaindex@0.5.15
+
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [c825a2f]
+  - llamaindex@0.5.14
+
+## 0.0.47
+
+### Patch Changes
+
+- llamaindex@0.5.13
+
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [345300f]
+- Updated dependencies [da5cfc4]
+- Updated dependencies [da5cfc4]
+  - llamaindex@0.5.12
+
 ## 0.0.45
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.45",
+  "version": "0.0.52",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.5.11",
+  "version": "0.5.18",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -55,7 +55,7 @@
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.7.0",
     "notion-md-crawler": "^1.0.0",
-    "openai": "^4.52.5",
+    "openai": "^4.55.3",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
     "pg": "^8.12.0",
@@ -65,6 +65,7 @@
     "string-strip-html": "^13.4.8",
     "tiktoken": "^1.0.15",
     "unpdf": "^0.11.0",
+    "weaviate-client": "^3.1.4",
     "wikipedia": "^2.1.2",
     "wink-nlp": "^2.3.0",
     "zod": "^3.23.8"

packages/llamaindex/src/cloud/LlamaCloudIndex.ts

@@ -8,7 +8,7 @@ import type { CloudRetrieveParams } from "./LlamaCloudRetriever.js";
 import { LlamaCloudRetriever } from "./LlamaCloudRetriever.js";
 import { getPipelineCreate } from "./config.js";
 import type { CloudConstructorParams } from "./constants.js";
-import { getAppBaseUrl, initService } from "./utils.js";
+import { getAppBaseUrl, getProjectId, initService } from "./utils.js";
 
 import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api";
 import { SentenceSplitter } from "@llamaindex/core/node-parser";
@@ -132,18 +132,28 @@ export class LlamaCloudIndex {
     await this.waitForPipelineIngestion(verbose, raiseOnError);
   }
 
-  private async getPipelineId(
-    name: string,
-    projectName: string,
+  public async getPipelineId(
+    name?: string,
+    projectName?: string,
   ): Promise<string> {
     const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
-      projectName,
-      pipelineName: name,
+      projectId: await this.getProjectId(projectName),
+      pipelineName: name ?? this.params.name,
     });
 
     return pipelines[0].id;
   }
 
+  public async getProjectId(
+    projectName?: string,
+    organizationId?: string,
+  ): Promise<string> {
+    return await getProjectId(
+      projectName ?? this.params.projectName,
+      organizationId ?? this.params.organizationId,
+    );
+  }
+
   static async fromDocuments(
     params: {
       documents: Document[];
@@ -168,6 +178,7 @@ export class LlamaCloudIndex {
     });
 
     const project = await ProjectsService.upsertProjectApiV1ProjectsPut({
+      organizationId: params.organizationId,
       requestBody: {
         name: params.projectName ?? "default",
       },

packages/llamaindex/src/cloud/LlamaCloudRetriever.ts

@@ -11,16 +11,17 @@ import { extractText, wrapEventCaller } from "@llamaindex/core/utils";
 import type { BaseRetriever, RetrieveParams } from "../Retriever.js";
 import type { ClientParams, CloudConstructorParams } from "./constants.js";
 import { DEFAULT_PROJECT_NAME } from "./constants.js";
-import { initService } from "./utils.js";
+import { getProjectId, initService } from "./utils.js";
 
 export type CloudRetrieveParams = Omit<
   RetrievalParams,
-  "query" | "searchFilters" | "className" | "denseSimilarityTopK"
-> & { similarityTopK?: number };
+  "query" | "search_filters" | "dense_similarity_top_k"
+> & { similarityTopK?: number; filters?: MetadataFilters };
 
 export class LlamaCloudRetriever implements BaseRetriever {
   clientParams: ClientParams;
   retrieveParams: CloudRetrieveParams;
+  organizationId?: string;
   projectName: string = DEFAULT_PROJECT_NAME;
   pipelineName: string;
 
@@ -49,6 +50,9 @@ export class LlamaCloudRetriever implements BaseRetriever {
     if (params.projectName) {
       this.projectName = params.projectName;
     }
+    if (params.organizationId) {
+      this.organizationId = params.organizationId;
+    }
   }
 
   @wrapEventCaller
@@ -57,7 +61,7 @@ export class LlamaCloudRetriever implements BaseRetriever {
     preFilters,
   }: RetrieveParams): Promise<NodeWithScore[]> {
     const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
-      projectName: this.projectName,
+      projectId: await getProjectId(this.projectName, this.organizationId),
       pipelineName: this.pipelineName,
     });
 
@@ -84,7 +88,9 @@ export class LlamaCloudRetriever implements BaseRetriever {
         requestBody: {
           ...this.retrieveParams,
           query: extractText(query),
-          search_filters: preFilters as MetadataFilters,
+          search_filters:
+            this.retrieveParams.filters ?? (preFilters as MetadataFilters),
+          dense_similarity_top_k: this.retrieveParams.similarityTopK,
         },
       });
 

packages/llamaindex/src/cloud/constants.ts

@@ -8,5 +8,6 @@ export type ClientParams = { apiKey?: string; baseUrl?: string };
 export type CloudConstructorParams = {
   name: string;
   projectName: string;
+  organizationId?: string;
   serviceContext?: ServiceContext;
 } & ClientParams;

packages/llamaindex/src/cloud/utils.ts

@@ -1,4 +1,4 @@
-import { OpenAPI } from "@llamaindex/cloud/api";
+import { OpenAPI, ProjectsService } from "@llamaindex/cloud/api";
 import { getEnv } from "@llamaindex/env";
 import type { ClientParams } from "./constants.js";
 import { DEFAULT_BASE_URL } from "./constants.js";
@@ -20,3 +20,31 @@ export function initService({ apiKey, baseUrl }: ClientParams = {}) {
     );
   }
 }
+
+export async function getProjectId(
+  projectName: string,
+  organizationId?: string,
+): Promise<string> {
+  const projects = await ProjectsService.listProjectsApiV1ProjectsGet({
+    projectName: projectName,
+    organizationId: organizationId,
+  });
+
+  if (projects.length === 0) {
+    throw new Error(
+      `Unknown project name ${projectName}. Please confirm a managed project with this name exists.`,
+    );
+  } else if (projects.length > 1) {
+    throw new Error(
+      `Multiple projects found with name ${projectName}. Please specify organization_id.`,
+    );
+  }
+
+  const project = projects[0];
+
+  if (!project.id) {
+    throw new Error(`No project found with name ${projectName}`);
+  }
+
+  return project.id;
+}

packages/llamaindex/src/embeddings/JinaAIEmbedding.ts

@@ -79,7 +79,7 @@ export class JinaAIEmbedding extends MultiModalEmbedding {
   private async getImageInput(
     image: ImageType,
   ): Promise<{ bytes: string } | { url: string }> {
-    if (isLocal(image)) {
+    if (isLocal(image) || image instanceof Blob) {
       const base64 = await imageToDataUrl(image);
       const bytes = base64.split(",")[1];
       return { bytes };

packages/llamaindex/src/engines/chat/ContextChatEngine.ts

@@ -126,7 +126,7 @@ export class ContextChatEngine extends PromptMixin implements ChatEngine {
     if (!this.systemPrompt) return message;
     return {
       ...message,
-      content: this.systemPrompt.trim() + "\n" + message.content,
+      content: this.systemPrompt.trim() + "\n" + extractText(message.content),
     };
   }
 }

packages/llamaindex/src/engines/query/RouterQueryEngine.ts

@@ -133,7 +133,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
       const responses: EngineResponse[] = [];
       for (let i = 0; i < result.selections.length; i++) {
         const engineInd = result.selections[i];
-        const logStr = `Selecting query engine ${engineInd}: ${result.selections[i]}.`;
+        const logStr = `Selecting query engine ${engineInd.index}: ${result.selections[i].index}.`;
 
         if (this.verbose) {
           console.log(logStr + "\n");

packages/llamaindex/src/engines/query/SubQuestionQueryEngine.ts

@@ -119,15 +119,15 @@ export class SubQuestionQueryEngine
         return null;
       }
 
-      const responseText = await queryEngine?.call?.({
+      const responseValue = await queryEngine?.call?.({
         query: question,
       });
 
-      if (!responseText) {
+      if (responseValue == null) {
         return null;
       }
 
-      const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
+      const nodeText = `Sub question: ${question}\nResponse: ${typeof responseValue === "string" ? responseValue : JSON.stringify(responseValue)}`;
       const node = new TextNode({ text: nodeText });
       return { node, score: 0 };
     } catch (error) {

packages/llamaindex/src/evaluation/Relevancy.ts

@@ -78,7 +78,7 @@ export class RelevancyEvaluator extends PromptMixin implements BaseEvaluator {
       serviceContext: this.serviceContext,
     });
 
-    const queryResponse = `Question: ${query}\nResponse: ${response}`;
+    const queryResponse = `Question: ${extractText(query)}\nResponse: ${response}`;
 
     const queryEngine = index.asQueryEngine();
 

packages/llamaindex/src/index.edge.ts

@@ -1,6 +1,8 @@
 import type { AgentEndEvent, AgentStartEvent } from "./agent/types.js";
 import type { RetrievalEndEvent, RetrievalStartEvent } from "./llm/types.js";
 
+export * from "@llamaindex/core/schema";
+
 declare module "@llamaindex/core/global" {
   export interface LlamaIndexEventMaps {
     "retrieve-start": RetrievalStartEvent;

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -386,6 +386,7 @@ export type VectorIndexRetrieverOptions = {
   index: VectorStoreIndex;
   similarityTopK?: number;
   topK?: TopKMap;
+  filters?: MetadataFilters;
 };
 
 export class VectorIndexRetriever implements BaseRetriever {
@@ -393,14 +394,21 @@ export class VectorIndexRetriever implements BaseRetriever {
   topK: TopKMap;
 
   serviceContext?: ServiceContext;
+  filters?: MetadataFilters;
 
-  constructor({ index, similarityTopK, topK }: VectorIndexRetrieverOptions) {
+  constructor({
+    index,
+    similarityTopK,
+    topK,
+    filters,
+  }: VectorIndexRetrieverOptions) {
     this.index = index;
     this.serviceContext = this.index.serviceContext;
     this.topK = topK ?? {
       [ModalityType.TEXT]: similarityTopK ?? DEFAULT_SIMILARITY_TOP_K,
       [ModalityType.IMAGE]: DEFAULT_SIMILARITY_TOP_K,
     };
+    this.filters = filters;
   }
 
   /**
@@ -443,7 +451,7 @@ export class VectorIndexRetriever implements BaseRetriever {
     query: MessageContent,
     type: ModalityType,
     vectorStore: VectorStore,
-    preFilters?: MetadataFilters,
+    filters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
     // convert string message to multi-modal format
     if (typeof query === "string") {
@@ -460,7 +468,7 @@ export class VectorIndexRetriever implements BaseRetriever {
           queryEmbedding,
           mode: VectorStoreQueryMode.DEFAULT,
           similarityTopK: this.topK[type],
-          filters: preFilters ?? undefined,
+          filters: this.filters ?? filters ?? undefined,
         });
         nodes = nodes.concat(this.buildNodeListFromQueryResult(result));
       }

packages/llamaindex/src/ingestion/IngestionPipeline.ts

@@ -1,4 +1,4 @@
-import type { TransformComponent } from "@llamaindex/core/schema";
+import type { BaseReader, TransformComponent } from "@llamaindex/core/schema";
 import {
   ModalityType,
   splitNodesByType,
@@ -6,7 +6,6 @@ import {
   type Document,
   type Metadata,
 } from "@llamaindex/core/schema";
-import type { BaseReader } from "../readers/type.js";
 import type { BaseDocumentStore } from "../storage/docStore/types.js";
 import type {
   VectorStore,
@@ -107,6 +106,7 @@ export class IngestionPipeline {
       inputNodes.push(this.documents);
     }
     if (this.reader) {
+      // fixme: empty parameter might cause error
       inputNodes.push(await this.reader.loadData());
     }
     return inputNodes.flat();

packages/llamaindex/src/llm/azure.ts

@@ -18,6 +18,7 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     openAIModel: "gpt-3.5-turbo-16k",
   },
   "gpt-4o": { contextWindow: 128000, openAIModel: "gpt-4o" },
+  "gpt-4o-mini": { contextWindow: 128000, openAIModel: "gpt-4o-mini" },
   "gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
   "gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
   "gpt-4-turbo": {
@@ -40,6 +41,10 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     contextWindow: 128000,
     openAIModel: "gpt-4o-2024-05-13",
   },
+  "gpt-4o-mini-2024-07-18": {
+    contextWindow: 128000,
+    openAIModel: "gpt-4o-mini-2024-07-18",
+  },
 };
 
 const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
@@ -73,6 +78,7 @@ const ALL_AZURE_API_VERSIONS = [
   "2024-03-01-preview",
   "2024-04-01-preview",
   "2024-05-01-preview",
+  "2024-06-01",
 ];
 
 const DEFAULT_API_VERSION = "2023-05-15";

packages/llamaindex/src/llm/openai.ts

@@ -6,6 +6,7 @@ import type {
   ClientOptions as OpenAIClientOptions,
 } from "openai";
 import { AzureOpenAI, OpenAI as OrigOpenAI } from "openai";
+import type { ChatModel } from "openai/resources/chat/chat";
 
 import {
   type BaseTool,
@@ -108,16 +109,24 @@ export const GPT4_MODELS = {
   "gpt-4o-2024-05-13": { contextWindow: 128000 },
   "gpt-4o-mini": { contextWindow: 128000 },
   "gpt-4o-mini-2024-07-18": { contextWindow: 128000 },
+  "gpt-4o-2024-08-06": { contextWindow: 128000 },
+  "gpt-4o-2024-09-14": { contextWindow: 128000 },
+  "gpt-4o-2024-10-14": { contextWindow: 128000 },
+  "gpt-4-0613": { contextWindow: 128000 },
+  "gpt-4-turbo-2024-04-09": { contextWindow: 128000 },
+  "gpt-4-0314": { contextWindow: 128000 },
+  "gpt-4-32k-0314": { contextWindow: 32768 },
 };
 
 // NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
 export const GPT35_MODELS = {
-  "gpt-3.5-turbo": { contextWindow: 4096 },
+  "gpt-3.5-turbo": { contextWindow: 16385 },
   "gpt-3.5-turbo-0613": { contextWindow: 4096 },
-  "gpt-3.5-turbo-16k": { contextWindow: 16384 },
-  "gpt-3.5-turbo-16k-0613": { contextWindow: 16384 },
-  "gpt-3.5-turbo-1106": { contextWindow: 16384 },
-  "gpt-3.5-turbo-0125": { contextWindow: 16384 },
+  "gpt-3.5-turbo-16k": { contextWindow: 16385 },
+  "gpt-3.5-turbo-16k-0613": { contextWindow: 16385 },
+  "gpt-3.5-turbo-1106": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0125": { contextWindow: 16385 },
+  "gpt-3.5-turbo-0301": { contextWindow: 16385 },
 };
 
 /**
@@ -126,7 +135,7 @@ export const GPT35_MODELS = {
 export const ALL_AVAILABLE_OPENAI_MODELS = {
   ...GPT4_MODELS,
   ...GPT35_MODELS,
-};
+} satisfies Record<ChatModel, { contextWindow: number }>;
 
 export function isFunctionCallingModel(llm: LLM): llm is OpenAI {
   let model: string;
@@ -157,8 +166,10 @@ export type OpenAIAdditionalChatOptions = Omit<
 >;
 
 export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
-  // Per completion OpenAI params
-  model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
+  model:
+    | ChatModel
+    // string & {} is a hack to allow any string, but still give autocomplete
+    | (string & {});
   temperature: number;
   topP: number;
   maxTokens?: number;

packages/llamaindex/src/readers/AssemblyAIReader.ts

@@ -1,4 +1,4 @@
-import { Document } from "@llamaindex/core/schema";
+import { type BaseReader, Document } from "@llamaindex/core/schema";
 import { getEnv } from "@llamaindex/env";
 import type {
   BaseServiceParams,
@@ -8,7 +8,6 @@ import type {
   TranscriptSentence,
 } from "assemblyai";
 import { AssemblyAI } from "assemblyai";
-import type { BaseReader } from "./type.js";
 
 type AssemblyAIOptions = Partial<BaseServiceParams>;
 const defaultOptions = {

packages/llamaindex/src/readers/CSVReader.ts

@@ -1,7 +1,6 @@
-import { Document } from "@llamaindex/core/schema";
+import { type BaseReader, Document, FileReader } from "@llamaindex/core/schema";
 import type { ParseConfig } from "papaparse";
 import Papa from "papaparse";
-import { FileReader } from "./type.js";
 
 /**
  * papaparse-based csv parser

packages/llamaindex/src/readers/DiscordReader.ts

@@ -1,5 +1,5 @@
 import { REST, type RESTOptions } from "@discordjs/rest";
-import { Document } from "@llamaindex/core/schema";
+import { Document, type BaseReader } from "@llamaindex/core/schema";
 import { getEnv } from "@llamaindex/env";
 import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
 
@@ -7,7 +7,7 @@ import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
  * Represents a reader for Discord messages using @discordjs/rest
  * See https://github.com/discordjs/discord.js/tree/main/packages/rest
  */
-export class DiscordReader {
+export class DiscordReader implements BaseReader {
   private client: REST;
 
   constructor(

packages/llamaindex/src/readers/DocxReader.ts

@@ -1,6 +1,5 @@
-import { Document } from "@llamaindex/core/schema";
+import { Document, FileReader } from "@llamaindex/core/schema";
 import mammoth from "mammoth";
-import { FileReader } from "./type.js";
 
 export class DocxReader extends FileReader {
   /** DocxParser */

packages/llamaindex/src/readers/HTMLReader.ts

@@ -1,6 +1,4 @@
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
-
+import { Document, FileReader } from "@llamaindex/core/schema";
 /**
  * Extract the significant text from an arbitrary HTML document.
  * The contents of any head, script, style, and xml tags are removed completely.

packages/llamaindex/src/readers/ImageReader.ts

@@ -1,6 +1,5 @@
 import type { Document } from "@llamaindex/core/schema";
-import { ImageDocument } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
+import { FileReader, ImageDocument } from "@llamaindex/core/schema";
 
 /**
  * Reads the content of an image file into a Document object (which stores the image file as a Blob).

packages/llamaindex/src/readers/JSONReader.ts

@@ -1,7 +1,5 @@
 import type { JSONValue } from "@llamaindex/core/global";
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
-
+import { Document, FileReader } from "@llamaindex/core/schema";
 export interface JSONReaderOptions {
   /**
    * Whether to ensure only ASCII characters.
@@ -185,7 +183,7 @@ export class JSONReader<T extends JSONValue> extends FileReader {
       return jsonStr;
     } catch (e) {
       throw new JSONStringifyError(
-        `Error stringifying JSON: ${e} in "${data}"`,
+        `Error stringifying JSON: ${e} in "${JSON.stringify(data)}"`,
       );
     }
   }

packages/llamaindex/src/readers/LlamaParseReader.ts

@@ -1,7 +1,92 @@
-import { Document } from "@llamaindex/core/schema";
+import { Document, FileReader } from "@llamaindex/core/schema";
 import { fs, getEnv } from "@llamaindex/env";
 import { filetypeinfo } from "magic-bytes.js";
-import { FileReader, type Language, type ResultType } from "./type.js";
+
+export type ResultType = "text" | "markdown" | "json";
+export type Language =
+  | "abq"
+  | "ady"
+  | "af"
+  | "ang"
+  | "ar"
+  | "as"
+  | "ava"
+  | "az"
+  | "be"
+  | "bg"
+  | "bh"
+  | "bho"
+  | "bn"
+  | "bs"
+  | "ch_sim"
+  | "ch_tra"
+  | "che"
+  | "cs"
+  | "cy"
+  | "da"
+  | "dar"
+  | "de"
+  | "en"
+  | "es"
+  | "et"
+  | "fa"
+  | "fr"
+  | "ga"
+  | "gom"
+  | "hi"
+  | "hr"
+  | "hu"
+  | "id"
+  | "inh"
+  | "is"
+  | "it"
+  | "ja"
+  | "kbd"
+  | "kn"
+  | "ko"
+  | "ku"
+  | "la"
+  | "lbe"
+  | "lez"
+  | "lt"
+  | "lv"
+  | "mah"
+  | "mai"
+  | "mi"
+  | "mn"
+  | "mr"
+  | "ms"
+  | "mt"
+  | "ne"
+  | "new"
+  | "nl"
+  | "no"
+  | "oc"
+  | "pi"
+  | "pl"
+  | "pt"
+  | "ro"
+  | "ru"
+  | "rs_cyrillic"
+  | "rs_latin"
+  | "sck"
+  | "sk"
+  | "sl"
+  | "sq"
+  | "sv"
+  | "sw"
+  | "ta"
+  | "tab"
+  | "te"
+  | "th"
+  | "tjk"
+  | "tl"
+  | "tr"
+  | "ug"
+  | "uk"
+  | "ur"
+  | "uz"
+  | "vi";
 
 const SUPPORT_FILE_EXT: string[] = [
   ".pdf",

packages/llamaindex/src/readers/MarkdownReader.ts

@@ -1,5 +1,4 @@
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
+import { Document, FileReader } from "@llamaindex/core/schema";
 
 type MarkdownTuple = [string | null, string];
 

packages/llamaindex/src/readers/NotionReader.ts

@@ -1,7 +1,7 @@
+import type { BaseReader } from "@llamaindex/core/schema";
 import { Document } from "@llamaindex/core/schema";
 import type { Crawler, CrawlerOptions, Page } from "notion-md-crawler";
 import { crawler, pageToString } from "notion-md-crawler";
-import type { BaseReader } from "./type.js";
 
 type NotionReaderOptions = Pick<CrawlerOptions, "client" | "serializers">;
 

packages/llamaindex/src/readers/PDFReader.ts

@@ -1,11 +1,14 @@
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
+import { Document, FileReader } from "@llamaindex/core/schema";
 
 /**
  * Read the text of a PDF
  */
 export class PDFReader extends FileReader {
   async loadDataAsContent(content: Uint8Array): Promise<Document[]> {
+    // XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error if a Buffer passed
+    if (content instanceof Buffer) {
+      content = new Uint8Array(content);
+    }
     const { totalPages, text } = await readPDF(content);
     return text.map((text, page) => {
       const metadata = {

packages/llamaindex/src/readers/SimpleDirectoryReader.edge.ts

@@ -1,8 +1,8 @@
+import type { BaseReader, FileReader } from "@llamaindex/core/schema";
 import { Document } from "@llamaindex/core/schema";
 import { path } from "@llamaindex/env";
 import { walk } from "../storage/FileSystem.js";
 import { TextFileReader } from "./TextFileReader.js";
-import type { BaseReader, FileReader } from "./type.js";
 import pLimit from "./utils.js";
 
 type ReaderCallback = (

packages/llamaindex/src/readers/SimpleDirectoryReader.ts

@@ -1,3 +1,4 @@
+import type { FileReader } from "@llamaindex/core/schema";
 import { Document } from "@llamaindex/core/schema";
 import { PapaCSVReader } from "./CSVReader.js";
 import { DocxReader } from "./DocxReader.js";
@@ -10,7 +11,6 @@ import {
   type SimpleDirectoryReaderLoadDataParams,
 } from "./SimpleDirectoryReader.edge.js";
 import { TextFileReader } from "./TextFileReader.js";
-import type { FileReader } from "./type.js";
 
 export const FILE_EXT_TO_READER: Record<string, FileReader> = {
   txt: new TextFileReader(),

packages/llamaindex/src/readers/SimpleMongoReader.ts

@@ -1,7 +1,6 @@
 import type { Metadata } from "@llamaindex/core/schema";
-import { Document } from "@llamaindex/core/schema";
+import { type BaseReader, Document } from "@llamaindex/core/schema";
 import type { MongoClient } from "mongodb";
-import type { BaseReader } from "./type.js";
 
 /**
  * Read in from MongoDB

packages/llamaindex/src/readers/TextFileReader.ts

@@ -1,6 +1,4 @@
-import { Document } from "@llamaindex/core/schema";
-import { FileReader } from "./type.js";
-
+import { Document, FileReader } from "@llamaindex/core/schema";
 /**
  * Read a .txt file
  */

packages/llamaindex/src/readers/type.ts

@@ -1,125 +0,0 @@
-import type { Document } from "@llamaindex/core/schema";
-import { fs, path } from "@llamaindex/env";
-
-/**
- * A reader takes imports data into Document objects.
- */
-export interface BaseReader {
-  loadData(...args: unknown[]): Promise<Document[]>;
-}
-
-/**
- * A FileReader takes file paths and imports data into Document objects.
- */
-export abstract class FileReader implements BaseReader {
-  abstract loadDataAsContent(
-    fileContent: Uint8Array,
-    fileName?: string,
-  ): Promise<Document[]>;
-
-  async loadData(filePath: string): Promise<Document[]> {
-    // XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error in PDFReader
-    const fileContent = new Uint8Array(await fs.readFile(filePath));
-    const fileName = path.basename(filePath);
-    const docs = await this.loadDataAsContent(fileContent, fileName);
-    docs.forEach(FileReader.addMetaData(filePath));
-    return docs;
-  }
-
-  static addMetaData(filePath: string) {
-    return (doc: Document, index: number) => {
-      // generate id as loadDataAsContent is only responsible for the content
-      doc.id_ = `${filePath}_${index + 1}`;
-      doc.metadata["file_path"] = path.resolve(filePath);
-      doc.metadata["file_name"] = path.basename(filePath);
-    };
-  }
-}
-
-// For LlamaParseReader.ts
-
-export type ResultType = "text" | "markdown" | "json";
-export type Language =
-  | "abq"
-  | "ady"
-  | "af"
-  | "ang"
-  | "ar"
-  | "as"
-  | "ava"
-  | "az"
-  | "be"
-  | "bg"
-  | "bh"
-  | "bho"
-  | "bn"
-  | "bs"
-  | "ch_sim"
-  | "ch_tra"
-  | "che"
-  | "cs"
-  | "cy"
-  | "da"
-  | "dar"
-  | "de"
-  | "en"
-  | "es"
-  | "et"
-  | "fa"
-  | "fr"
-  | "ga"
-  | "gom"
-  | "hi"
-  | "hr"
-  | "hu"
-  | "id"
-  | "inh"
-  | "is"
-  | "it"
-  | "ja"
-  | "kbd"
-  | "kn"
-  | "ko"
-  | "ku"
-  | "la"
-  | "lbe"
-  | "lez"
-  | "lt"
-  | "lv"
-  | "mah"
-  | "mai"
-  | "mi"
-  | "mn"
-  | "mr"
-  | "ms"
-  | "mt"
-  | "ne"
-  | "new"
-  | "nl"
-  | "no"
-  | "oc"
-  | "pi"
-  | "pl"
-  | "pt"
-  | "ro"
-  | "ru"
-  | "rs_cyrillic"
-  | "rs_latin"
-  | "sck"
-  | "sk"
-  | "sl"
-  | "sq"
-  | "sv"
-  | "sw"
-  | "ta"
-  | "tab"
-  | "te"
-  | "th"
-  | "tjk"
-  | "tl"
-  | "tr"
-  | "ug"
-  | "uk"
-  | "ur"
-  | "uz"
-  | "vi";

packages/llamaindex/src/storage/index.ts

@@ -18,3 +18,4 @@ export { PineconeVectorStore } from "./vectorStore/PineconeVectorStore.js";
 export { QdrantVectorStore } from "./vectorStore/QdrantVectorStore.js";
 export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore.js";
 export * from "./vectorStore/types.js";
+export { WeaviateVectorStore } from "./vectorStore/WeaviateVectorStore.js";

packages/llamaindex/src/storage/vectorStore/PGVectorStore.ts

@@ -273,7 +273,7 @@ export class PGVectorStore
       const paramIndex = params.length + 1;
       whereClauses.push(`metadata->>'${filter.key}' = $${paramIndex}`);
       // TODO: support filter with other operators
-      if (!Array.isArray(filter.value)) {
+      if (!Array.isArray(filter.value) && filter.value) {
         params.push(filter.value);
       }
     });

packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts

@@ -36,7 +36,7 @@ type MetadataValue = Record<string, any>;
 
 // Mapping of filter operators to metadata filter functions
 const OPERATOR_TO_FILTER: {
-  [key in FilterOperator]: (
+  [key in FilterOperator]?: (
     { key, value }: MetadataFilter,
     metadata: MetadataValue,
   ) => boolean;
@@ -94,7 +94,20 @@ const buildFilterFn = (
   const queryCondition = condition || "and"; // default to and
 
   const itemFilterFn = (filter: MetadataFilter): boolean => {
-    if (metadata[filter.key] === undefined) return false; // always return false if the metadata key is not present
+    if (filter.operator === FilterOperator.IS_EMPTY) {
+      // for `is_empty` operator, return true if the metadata key is not present or the value is empty
+      const value = metadata[filter.key];
+      return (
+        value === undefined ||
+        value === null ||
+        value === "" ||
+        (Array.isArray(value) && value.length === 0)
+      );
+    }
+    if (metadata[filter.key] === undefined) {
+      // for other operators, always return false if the metadata key is not present
+      return false;
+    }
     const metadataLookupFn = OPERATOR_TO_FILTER[filter.operator];
     if (!metadataLookupFn)
       throw new Error(`Unsupported operator: ${filter.operator}`);

packages/llamaindex/src/storage/vectorStore/WeaviateVectorStore.ts

@@ -0,0 +1,339 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import { BaseNode, MetadataMode, type Metadata } from "@llamaindex/core/schema";
+import weaviate, {
+  Filters,
+  type Collection,
+  type DeleteManyOptions,
+  type FilterValue,
+  type WeaviateClient,
+  type WeaviateNonGenericObject,
+} from "weaviate-client";
+
+import {
+  VectorStoreBase,
+  VectorStoreQueryMode,
+  type IEmbedModel,
+  type MetadataFilter,
+  type MetadataFilters,
+  type VectorStoreNoEmbedModel,
+  type VectorStoreQuery,
+  type VectorStoreQueryResult,
+} from "./types.js";
+import {
+  metadataDictToNode,
+  nodeToMetadata,
+  parseArrayValue,
+  parseNumberValue,
+} from "./utils.js";
+
+const NODE_SCHEMA = [
+  {
+    dataType: ["text"],
+    description: "Text property",
+    name: "text",
+  },
+  {
+    dataType: ["text"],
+    description: "The ref_doc_id of the Node",
+    name: "ref_doc_id",
+  },
+  {
+    dataType: ["text"],
+    description: "node_info (in JSON)",
+    name: "node_info",
+  },
+  {
+    dataType: ["text"],
+    description: "The relationships of the node (in JSON)",
+    name: "relationships",
+  },
+];
+
+const SIMILARITY_KEYS: {
+  [key: string]: "distance" | "score";
+} = {
+  [VectorStoreQueryMode.DEFAULT]: "distance",
+  [VectorStoreQueryMode.HYBRID]: "score",
+};
+
+const buildFilterItem = (
+  collection: Collection,
+  filter: MetadataFilter,
+): FilterValue => {
+  const { key, operator, value } = filter;
+
+  switch (operator) {
+    case "==": {
+      return collection.filter.byProperty(key).equal(value);
+    }
+    case "!=": {
+      return collection.filter.byProperty(key).notEqual(value);
+    }
+    case ">": {
+      return collection.filter
+        .byProperty(key)
+        .greaterThan(parseNumberValue(value));
+    }
+    case "<": {
+      return collection.filter
+        .byProperty(key)
+        .lessThan(parseNumberValue(value));
+    }
+    case ">=": {
+      return collection.filter
+        .byProperty(key)
+        .greaterOrEqual(parseNumberValue(value));
+    }
+    case "<=": {
+      return collection.filter
+        .byProperty(key)
+        .lessOrEqual(parseNumberValue(value));
+    }
+    case "any": {
+      return collection.filter
+        .byProperty(key)
+        .containsAny(parseArrayValue(value).map(String));
+    }
+    case "all": {
+      return collection.filter
+        .byProperty(key)
+        .containsAll(parseArrayValue(value).map(String));
+    }
+    default: {
+      throw new Error(`Operator ${operator} is not supported.`);
+    }
+  }
+};
+
+const toWeaviateFilter = (
+  collection: Collection,
+  standardFilters?: MetadataFilters,
+): FilterValue | undefined => {
+  if (!standardFilters?.filters.length) return undefined;
+  const filtersList = standardFilters.filters.map((filter) =>
+    buildFilterItem(collection, filter),
+  );
+  if (filtersList.length === 1) return filtersList[0];
+  const condition = standardFilters.condition ?? "and";
+  return Filters[condition](...filtersList);
+};
+
+export class WeaviateVectorStore
+  extends VectorStoreBase
+  implements VectorStoreNoEmbedModel
+{
+  public storesText: boolean = true;
+  private flatMetadata: boolean = true;
+
+  private weaviateClient?: WeaviateClient;
+  private clusterURL!: string;
+  private apiKey!: string;
+  private indexName: string;
+
+  private idKey: string;
+  private contentKey: string;
+  private embeddingKey: string;
+  private metadataKey: string;
+
+  constructor(
+    init?: Partial<IEmbedModel> & {
+      weaviateClient?: WeaviateClient;
+      cloudOptions?: {
+        clusterURL?: string;
+        apiKey?: string;
+      };
+      indexName?: string;
+      idKey?: string;
+      contentKey?: string;
+      metadataKey?: string;
+      embeddingKey?: string;
+    },
+  ) {
+    super(init?.embedModel);
+
+    if (init?.weaviateClient) {
+      // Use the provided client
+      this.weaviateClient = init.weaviateClient;
+    } else {
+      // Load client cloud options from config or env
+      const clusterURL =
+        init?.cloudOptions?.clusterURL ?? process.env.WEAVIATE_CLUSTER_URL;
+      const apiKey = init?.cloudOptions?.apiKey ?? process.env.WEAVIATE_API_KEY;
+      if (!clusterURL || !apiKey) {
+        throw new Error(
+          "Must specify WEAVIATE_CLUSTER_URL and WEAVIATE_API_KEY via env variable.",
+        );
+      }
+      this.clusterURL = clusterURL;
+      this.apiKey = apiKey;
+    }
+
+    this.checkIndexName(init?.indexName);
+    this.indexName = init?.indexName ?? "LlamaIndex";
+    this.idKey = init?.idKey ?? "id";
+    this.contentKey = init?.contentKey ?? "text";
+    this.embeddingKey = init?.embeddingKey ?? "vectors";
+    this.metadataKey = init?.metadataKey ?? "node_info";
+  }
+
+  public client() {
+    return this.getClient();
+  }
+
+  public async add(nodes: BaseNode<Metadata>[]): Promise<string[]> {
+    const collection = await this.ensureCollection({ createIfNotExists: true });
+
+    const result = await collection.data.insertMany(
+      nodes.map((node) => {
+        const metadata = nodeToMetadata(
+          node,
+          true,
+          this.contentKey,
+          this.flatMetadata,
+        );
+        const body = {
+          [this.idKey]: node.id_,
+          [this.embeddingKey]: node.getEmbedding(),
+          properties: {
+            ...metadata,
+            [this.contentKey]: node.getContent(MetadataMode.NONE),
+            [this.metadataKey]: JSON.stringify(metadata),
+            relationships: JSON.stringify({ ref_doc_id: metadata.ref_doc_id }),
+          },
+        };
+        return body;
+      }),
+    );
+
+    return Object.values(result.uuids);
+  }
+
+  public async delete(
+    refDocId: string,
+    deleteOptions?: DeleteManyOptions<boolean>,
+  ): Promise<void> {
+    const collection = await this.ensureCollection();
+    await collection.data.deleteMany(
+      collection.filter.byProperty("ref_doc_id").like(refDocId),
+      deleteOptions,
+    );
+  }
+
+  public async query(query: VectorStoreQuery): Promise<VectorStoreQueryResult> {
+    const collection = await this.ensureCollection();
+    const allProperties = await this.getAllProperties();
+
+    let filters: FilterValue | undefined = undefined;
+
+    if (query.docIds) {
+      filters = collection.filter
+        .byProperty("doc_id")
+        .containsAny(query.docIds);
+    }
+
+    if (query.filters) {
+      filters = toWeaviateFilter(collection, query.filters);
+    }
+
+    const queryResult = await collection.query.hybrid(query.queryStr!, {
+      vector: query.queryEmbedding,
+      alpha: this.getQueryAlpha(query),
+      limit: query.similarityTopK,
+      returnMetadata: Object.values(SIMILARITY_KEYS),
+      returnProperties: allProperties,
+      includeVector: true,
+      filters,
+    });
+
+    const entries = queryResult.objects;
+
+    const similarityKey = SIMILARITY_KEYS[query.mode];
+    const nodes: BaseNode<Metadata>[] = [];
+    const similarities: number[] = [];
+    const ids: string[] = [];
+
+    entries.forEach((entry, index) => {
+      if (index < query.similarityTopK && entry.metadata) {
+        const node = metadataDictToNode(entry.properties);
+        node.setContent(entry.properties[this.contentKey]);
+        nodes.push(node);
+        ids.push(entry.uuid);
+        similarities.push(this.getNodeSimilarity(entry, similarityKey));
+      }
+    });
+
+    return {
+      nodes,
+      similarities,
+      ids,
+    };
+  }
+
+  private async getClient(): Promise<WeaviateClient> {
+    if (this.weaviateClient) return this.weaviateClient;
+    const client = await weaviate.connectToWeaviateCloud(this.clusterURL, {
+      authCredentials: new weaviate.ApiKey(this.apiKey),
+    });
+    this.weaviateClient = client;
+    return client;
+  }
+
+  private async ensureCollection({ createIfNotExists = false } = {}) {
+    const client = await this.getClient();
+    const exists = await this.doesCollectionExist();
+    if (!exists) {
+      if (createIfNotExists) {
+        await this.createCollection();
+      } else {
+        throw new Error(`Collection ${this.indexName} does not exist.`);
+      }
+    }
+    return client.collections.get(this.indexName);
+  }
+
+  private async doesCollectionExist() {
+    const client = await this.getClient();
+    return client.collections.exists(this.indexName);
+  }
+
+  private async createCollection() {
+    const client = await this.getClient();
+    return await client.collections.createFromSchema({
+      class: this.indexName,
+      description: `Collection for ${this.indexName}`,
+      properties: NODE_SCHEMA,
+    });
+  }
+
+  private getQueryAlpha(query: VectorStoreQuery): number | undefined {
+    if (!query.queryEmbedding) return undefined;
+    if (query.mode === VectorStoreQueryMode.DEFAULT) return 1;
+    if (query.mode === VectorStoreQueryMode.HYBRID && query.queryStr)
+      return query.alpha;
+    return undefined;
+  }
+
+  private async getAllProperties(): Promise<string[]> {
+    const collection = await this.ensureCollection();
+    const properties = (await collection.config.get()).properties;
+    return properties.map((p) => p.name);
+  }
+
+  private checkIndexName(indexName?: string) {
+    if (indexName && indexName[0] !== indexName[0].toUpperCase()) {
+      throw new Error(
+        "Index name must start with a capital letter, e.g. 'LlamaIndex'",
+      );
+    }
+  }
+
+  private getNodeSimilarity(
+    entry: WeaviateNonGenericObject,
+    similarityKey: "distance" | "score" = "distance",
+  ): number {
+    const distance = entry.metadata?.[similarityKey];
+    if (distance === undefined) return 1;
+    // convert distance https://forum.weaviate.io/t/distance-vs-certainty-scores/258
+    return 1 - distance;
+  }
+}

packages/llamaindex/src/storage/vectorStore/types.ts

@@ -33,6 +33,7 @@ export enum FilterOperator {
   ALL = "all", // Contains all (array of strings)
   TEXT_MATCH = "text_match", // full text match (allows you to search for a specific substring, token or phrase within the text field)
   CONTAINS = "contains", // metadata array contains value (string or number)
+  IS_EMPTY = "is_empty", // the field is not exist or empty (null or empty array)
 }
 
 export enum FilterCondition {
@@ -44,7 +45,7 @@ export type MetadataFilterValue = string | number | string[] | number[];
 
 export interface MetadataFilter {
   key: string;
-  value: MetadataFilterValue;
+  value?: MetadataFilterValue;
   operator: `${FilterOperator}`; // ==, any, all,...
 }
 

packages/llamaindex/src/storage/vectorStore/utils.ts

@@ -80,7 +80,7 @@ export function metadataDictToNode(
 }
 
 export const parsePrimitiveValue = (
-  value: MetadataFilterValue,
+  value?: MetadataFilterValue,
 ): string | number => {
   if (typeof value !== "number" && typeof value !== "string") {
     throw new Error("Value must be a string or number");
@@ -89,7 +89,7 @@ export const parsePrimitiveValue = (
 };
 
 export const parseArrayValue = (
-  value: MetadataFilterValue,
+  value?: MetadataFilterValue,
 ): string[] | number[] => {
   const isPrimitiveArray =
     Array.isArray(value) &&
@@ -99,3 +99,8 @@ export const parseArrayValue = (
   }
   return value;
 };
+
+export const parseNumberValue = (value?: MetadataFilterValue): number => {
+  if (typeof value !== "number") throw new Error("Value must be a number");
+  return value;
+};

packages/llamaindex/tests/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/core-test
 
+## 0.0.7
+
+### Patch Changes
+
+- 01c184c: Add is_empty operator for filtering vector store
+
 ## 0.0.6
 
 ### Patch Changes

packages/llamaindex/tests/import.test.ts

@@ -0,0 +1,11 @@
+import { expect, test } from "vitest";
+
+test("Node classes should be included in the top level", async () => {
+  const { Document, IndexNode, TextNode, BaseNode } = await import(
+    "llamaindex"
+  );
+  expect(Document).toBeDefined();
+  expect(IndexNode).toBeDefined();
+  expect(TextNode).toBeDefined();
+  expect(BaseNode).toBeDefined();
+});

packages/llamaindex/tests/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llamaindex-test",
   "private": true,
-  "version": "0.0.6",
+  "version": "0.0.7",
   "type": "module",
   "scripts": {
     "test": "vitest run"

packages/llamaindex/tests/vectorStores/SimpleVectorStore.test.ts

@@ -256,6 +256,18 @@ describe("SimpleVectorStore", () => {
         },
         expected: 1,
       },
+      {
+        title: "Filter IS_EMPTY",
+        filters: {
+          filters: [
+            {
+              key: "not-exist-key",
+              operator: "is_empty",
+            },
+          ],
+        },
+        expected: 3,
+      },
       {
         title: "Filter OR",
         filters: {