fix: ignore error

.changeset/chilled-tomatoes-visit.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat: add splitByPage mode to LlamaParseReader

.changeset/thin-pens-deliver.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/core": patch
+"@llamaindex/core-tests": patch
+---
+
+fix: handling errors in splitBySentenceTokenizer

.eslintrc.cjs

@@ -31,12 +31,7 @@ module.exports = {
     "@typescript-eslint/ban-types": "off",
     "no-array-constructor": "off",
     "@typescript-eslint/no-array-constructor": "off",
-    "@typescript-eslint/no-base-to-string": [
-      "error",
-      {
-        ignoredTypeNames: ["Error", "RegExp", "URL", "URLSearchParams"],
-      },
-    ],
+    "@typescript-eslint/no-base-to-string": "off",
     "@typescript-eslint/no-duplicate-enum-values": "off",
     "@typescript-eslint/no-duplicate-type-constituents": "off",
     "@typescript-eslint/no-explicit-any": "off",

README.md

@@ -164,7 +164,7 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
 
 - [Node](/packages/llamaindex/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
 
-- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that question. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
+- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
 
 - [Indices](/packages/llamaindex/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
 

apps/docs/CHANGELOG.md

@@ -1,35 +1,5 @@
 # docs
 
-## 0.0.56
-
-### Patch Changes
-
-- Updated dependencies [01c184c]
-- Updated dependencies [07a275f]
-  - llamaindex@0.5.15
-
-## 0.0.55
-
-### Patch Changes
-
-- Updated dependencies [c825a2f]
-  - llamaindex@0.5.14
-
-## 0.0.54
-
-### Patch Changes
-
-- llamaindex@0.5.13
-
-## 0.0.53
-
-### Patch Changes
-
-- Updated dependencies [345300f]
-- Updated dependencies [da5cfc4]
-- Updated dependencies [da5cfc4]
-  - llamaindex@0.5.12
-
 ## 0.0.52
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.56",
+  "version": "0.0.52",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,39 +1,5 @@
 # @llamaindex/autotool-02-next-example
 
-## 0.1.40
-
-### Patch Changes
-
-- Updated dependencies [01c184c]
-- Updated dependencies [07a275f]
-  - llamaindex@0.5.15
-  - @llamaindex/autotool@2.0.0
-
-## 0.1.39
-
-### Patch Changes
-
-- Updated dependencies [c825a2f]
-  - llamaindex@0.5.14
-  - @llamaindex/autotool@2.0.0
-
-## 0.1.38
-
-### Patch Changes
-
-- llamaindex@0.5.13
-- @llamaindex/autotool@2.0.0
-
-## 0.1.37
-
-### Patch Changes
-
-- Updated dependencies [345300f]
-- Updated dependencies [da5cfc4]
-- Updated dependencies [da5cfc4]
-  - llamaindex@0.5.12
-  - @llamaindex/autotool@2.0.0
-
 ## 0.1.36
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.40",
+  "version": "0.1.36",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.5.15",
+    "llamaindex": "^0.5.11",
     "openai": "^4",
     "typescript": "^4"
   },
@@ -70,7 +70,7 @@
     "@swc/types": "^0.1.8",
     "@types/json-schema": "^7.0.15",
     "@types/node": "^20.12.11",
-    "bunchee": "5.3.1",
+    "bunchee": "5.3.0-beta.0",
     "llamaindex": "workspace:*",
     "next": "14.2.5",
     "rollup": "^4.18.0",

packages/cloud/package.json

@@ -35,6 +35,6 @@
   },
   "devDependencies": {
     "@hey-api/openapi-ts": "^0.48.0",
-    "bunchee": "5.3.1"
+    "bunchee": "5.3.0-beta.0"
   }
 }

packages/community/CHANGELOG.md

@@ -1,19 +1,5 @@
 # @llamaindex/community
 
-## 0.0.28
-
-### Patch Changes
-
-- Updated dependencies [04b2f8e]
-  - @llamaindex/core@0.1.7
-
-## 0.0.27
-
-### Patch Changes
-
-- Updated dependencies [0452af9]
-  - @llamaindex/core@0.1.6
-
 ## 0.0.26
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.28",
+  "version": "0.0.26",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -43,7 +43,7 @@
   },
   "devDependencies": {
     "@types/node": "^20.14.2",
-    "bunchee": "5.3.1"
+    "bunchee": "5.3.0-beta.0"
   },
   "dependencies": {
     "@aws-sdk/client-bedrock-runtime": "^3.613.0",

packages/core/CHANGELOG.md

@@ -1,17 +1,5 @@
 # @llamaindex/core
 
-## 0.1.7
-
-### Patch Changes
-
-- 04b2f8e: Fix issue with metadata included after sentence splitter
-
-## 0.1.6
-
-### Patch Changes
-
-- 0452af9: fix: handling errors in splitBySentenceTokenizer
-
 ## 0.1.5
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.1.7",
+  "version": "0.1.5",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./node-parser": {
@@ -131,7 +131,7 @@
   },
   "devDependencies": {
     "ajv": "^8.16.0",
-    "bunchee": "5.3.1",
+    "bunchee": "5.3.0-beta.0",
     "natural": "^7.1.0"
   },
   "dependencies": {

packages/core/src/node-parser/base.ts

@@ -140,7 +140,7 @@ export abstract class MetadataAwareTextSplitter extends TextSplitter {
     return nodes.reduce<TextNode[]>((allNodes, node) => {
       const metadataStr = this.getMetadataString(node);
       const splits = this.splitTextMetadataAware(
-        node.getContent(MetadataMode.NONE),
+        node.getContent(MetadataMode.ALL),
         metadataStr,
       );
       return allNodes.concat(buildNodeFromSplits(splits, node));

packages/core/src/schema/index.ts

@@ -1,4 +1,4 @@
 export * from "./node";
-export { FileReader, TransformComponent, type BaseReader } from "./type";
+export { TransformComponent } from "./type";
 export { EngineResponse } from "./type/engine–response";
 export * from "./zod";

packages/core/src/schema/type.ts

@@ -1,5 +1,5 @@
-import { fs, path, randomUUID } from "@llamaindex/env";
-import type { BaseNode, Document } from "./node";
+import { randomUUID } from "@llamaindex/env";
+import type { BaseNode } from "./node";
 
 interface TransformComponentSignature {
   <Options extends Record<string, unknown>>(
@@ -28,37 +28,3 @@ export class TransformComponent {
     return transform;
   }
 }
-
-/**
- * A reader takes imports data into Document objects.
- */
-export interface BaseReader {
-  loadData(...args: unknown[]): Promise<Document[]>;
-}
-
-/**
- * A FileReader takes file paths and imports data into Document objects.
- */
-export abstract class FileReader implements BaseReader {
-  abstract loadDataAsContent(
-    fileContent: Uint8Array,
-    fileName?: string,
-  ): Promise<Document[]>;
-
-  async loadData(filePath: string): Promise<Document[]> {
-    const fileContent = await fs.readFile(filePath);
-    const fileName = path.basename(filePath);
-    const docs = await this.loadDataAsContent(fileContent, fileName);
-    docs.forEach(FileReader.addMetaData(filePath));
-    return docs;
-  }
-
-  static addMetaData(filePath: string) {
-    return (doc: Document, index: number) => {
-      // generate id as loadDataAsContent is only responsible for the content
-      doc.id_ = `${filePath}_${index + 1}`;
-      doc.metadata["file_path"] = path.resolve(filePath);
-      doc.metadata["file_name"] = path.basename(filePath);
-    };
-  }
-}

packages/experimental/CHANGELOG.md

@@ -1,35 +1,5 @@
 # @llamaindex/experimental
 
-## 0.0.65
-
-### Patch Changes
-
-- Updated dependencies [01c184c]
-- Updated dependencies [07a275f]
-  - llamaindex@0.5.15
-
-## 0.0.64
-
-### Patch Changes
-
-- Updated dependencies [c825a2f]
-  - llamaindex@0.5.14
-
-## 0.0.63
-
-### Patch Changes
-
-- llamaindex@0.5.13
-
-## 0.0.62
-
-### Patch Changes
-
-- Updated dependencies [345300f]
-- Updated dependencies [da5cfc4]
-- Updated dependencies [da5cfc4]
-  - llamaindex@0.5.12
-
 ## 0.0.61
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.65",
+  "version": "0.0.61",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/experimental/src/engines/query/JSONQueryEngine.ts

@@ -162,18 +162,18 @@ export class JSONQueryEngine implements QueryEngine {
 
     const schema = this.getSchemaContext();
 
-    const { text: jsonPathResponse } = await this.serviceContext.llm.complete({
+    const jsonPathResponseStr = await this.serviceContext.llm.complete({
       prompt: this.jsonPathPrompt({ query, schema }),
     });
 
     if (this.verbose) {
       console.log(
-        `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponse}\n\`\`\`\n`,
+        `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponseStr}\n\`\`\`\n`,
       );
     }
 
     const jsonPathOutput = await this.outputProcessor({
-      llmOutput: jsonPathResponse,
+      llmOutput: jsonPathResponseStr.text,
       jsonValue: this.jsonValue,
     });
 
@@ -188,7 +188,7 @@ export class JSONQueryEngine implements QueryEngine {
         prompt: this.responseSynthesisPrompt({
           query,
           jsonSchema: schema,
-          jsonPath: jsonPathResponse,
+          jsonPath: jsonPathResponseStr.text,
           jsonPathValue: JSON.stringify(jsonPathOutput),
         }),
       });
@@ -199,7 +199,7 @@ export class JSONQueryEngine implements QueryEngine {
     }
 
     const responseMetadata = {
-      jsonPathResponse,
+      jsonPathResponseStr,
     };
 
     const response = EngineResponse.fromResponse(responseStr, false);

packages/llamaindex/CHANGELOG.md

@@ -1,35 +1,5 @@
 # llamaindex
 
-## 0.5.15
-
-### Patch Changes
-
-- 01c184c: Add is_empty operator for filtering vector store
-- 07a275f: chore: bump openai
-
-## 0.5.14
-
-### Patch Changes
-
-- c825a2f: Add gpt-4o-mini to Azure. Add 2024-06-01 API version for Azure
-
-## 0.5.13
-
-### Patch Changes
-
-- Updated dependencies [04b2f8e]
-  - @llamaindex/core@0.1.7
-
-## 0.5.12
-
-### Patch Changes
-
-- 345300f: feat: add splitByPage mode to LlamaParseReader
-- da5cfc4: Add metadatafilter options to retriever constructors
-- da5cfc4: Fix system prompt not used in ContextChatEngine
-- Updated dependencies [0452af9]
-  - @llamaindex/core@0.1.6
-
 ## 0.5.11
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,35 +1,5 @@
 # @llamaindex/cloudflare-worker-agent-test
 
-## 0.0.49
-
-### Patch Changes
-
-- Updated dependencies [01c184c]
-- Updated dependencies [07a275f]
-  - llamaindex@0.5.15
-
-## 0.0.48
-
-### Patch Changes
-
-- Updated dependencies [c825a2f]
-  - llamaindex@0.5.14
-
-## 0.0.47
-
-### Patch Changes
-
-- llamaindex@0.5.13
-
-## 0.0.46
-
-### Patch Changes
-
-- Updated dependencies [345300f]
-- Updated dependencies [da5cfc4]
-- Updated dependencies [da5cfc4]
-  - llamaindex@0.5.12
-
 ## 0.0.45
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.49",
+  "version": "0.0.45",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,35 +1,5 @@
 # @llamaindex/next-agent-test
 
-## 0.1.49
-
-### Patch Changes
-
-- Updated dependencies [01c184c]
-- Updated dependencies [07a275f]
-  - llamaindex@0.5.15
-
-## 0.1.48
-
-### Patch Changes
-
-- Updated dependencies [c825a2f]
-  - llamaindex@0.5.14
-
-## 0.1.47
-
-### Patch Changes
-
-- llamaindex@0.5.13
-
-## 0.1.46
-
-### Patch Changes
-
-- Updated dependencies [345300f]
-- Updated dependencies [da5cfc4]
-- Updated dependencies [da5cfc4]
-  - llamaindex@0.5.12
-
 ## 0.1.45
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.49",
+  "version": "0.1.45",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,35 +1,5 @@
 # test-edge-runtime
 
-## 0.1.48
-
-### Patch Changes
-
-- Updated dependencies [01c184c]
-- Updated dependencies [07a275f]
-  - llamaindex@0.5.15
-
-## 0.1.47
-
-### Patch Changes
-
-- Updated dependencies [c825a2f]
-  - llamaindex@0.5.14
-
-## 0.1.46
-
-### Patch Changes
-
-- llamaindex@0.5.13
-
-## 0.1.45
-
-### Patch Changes
-
-- Updated dependencies [345300f]
-- Updated dependencies [da5cfc4]
-- Updated dependencies [da5cfc4]
-  - llamaindex@0.5.12
-
 ## 0.1.44
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.48",
+  "version": "0.1.44",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,35 +1,5 @@
 # @llamaindex/next-node-runtime
 
-## 0.0.30
-
-### Patch Changes
-
-- Updated dependencies [01c184c]
-- Updated dependencies [07a275f]
-  - llamaindex@0.5.15
-
-## 0.0.29
-
-### Patch Changes
-
-- Updated dependencies [c825a2f]
-  - llamaindex@0.5.14
-
-## 0.0.28
-
-### Patch Changes
-
-- llamaindex@0.5.13
-
-## 0.0.27
-
-### Patch Changes
-
-- Updated dependencies [345300f]
-- Updated dependencies [da5cfc4]
-- Updated dependencies [da5cfc4]
-  - llamaindex@0.5.12
-
 ## 0.0.26
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.30",
+  "version": "0.0.26",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,35 +1,5 @@
 # @llamaindex/waku-query-engine-test
 
-## 0.0.49
-
-### Patch Changes
-
-- Updated dependencies [01c184c]
-- Updated dependencies [07a275f]
-  - llamaindex@0.5.15
-
-## 0.0.48
-
-### Patch Changes
-
-- Updated dependencies [c825a2f]
-  - llamaindex@0.5.14
-
-## 0.0.47
-
-### Patch Changes
-
-- llamaindex@0.5.13
-
-## 0.0.46
-
-### Patch Changes
-
-- Updated dependencies [345300f]
-- Updated dependencies [da5cfc4]
-- Updated dependencies [da5cfc4]
-  - llamaindex@0.5.12
-
 ## 0.0.45
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.49",
+  "version": "0.0.45",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.5.15",
+  "version": "0.5.11",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -55,7 +55,7 @@
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.7.0",
     "notion-md-crawler": "^1.0.0",
-    "openai": "^4.55.3",
+    "openai": "^4.52.5",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
     "pg": "^8.12.0",

packages/llamaindex/src/cloud/LlamaCloudRetriever.ts

@@ -15,8 +15,8 @@ import { initService } from "./utils.js";
 
 export type CloudRetrieveParams = Omit<
   RetrievalParams,
-  "query" | "search_filters" | "dense_similarity_top_k"
-> & { similarityTopK?: number; filters?: MetadataFilters };
+  "query" | "searchFilters" | "className" | "denseSimilarityTopK"
+> & { similarityTopK?: number };
 
 export class LlamaCloudRetriever implements BaseRetriever {
   clientParams: ClientParams;
@@ -84,9 +84,7 @@ export class LlamaCloudRetriever implements BaseRetriever {
         requestBody: {
           ...this.retrieveParams,
           query: extractText(query),
-          search_filters:
-            this.retrieveParams.filters ?? (preFilters as MetadataFilters),
-          dense_similarity_top_k: this.retrieveParams.similarityTopK,
+          search_filters: preFilters as MetadataFilters,
         },
       });
 

packages/llamaindex/src/embeddings/JinaAIEmbedding.ts

@@ -79,7 +79,7 @@ export class JinaAIEmbedding extends MultiModalEmbedding {
   private async getImageInput(
     image: ImageType,
   ): Promise<{ bytes: string } | { url: string }> {
-    if (isLocal(image) || image instanceof Blob) {
+    if (isLocal(image)) {
       const base64 = await imageToDataUrl(image);
       const bytes = base64.split(",")[1];
       return { bytes };

packages/llamaindex/src/engines/chat/ContextChatEngine.ts

@@ -126,7 +126,7 @@ export class ContextChatEngine extends PromptMixin implements ChatEngine {
     if (!this.systemPrompt) return message;
     return {
       ...message,
-      content: this.systemPrompt.trim() + "\n" + extractText(message.content),
+      content: this.systemPrompt.trim() + "\n" + message.content,
     };
   }
 }

packages/llamaindex/src/engines/query/RouterQueryEngine.ts

@@ -133,7 +133,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
       const responses: EngineResponse[] = [];
       for (let i = 0; i < result.selections.length; i++) {
         const engineInd = result.selections[i];
-        const logStr = `Selecting query engine ${engineInd.index}: ${result.selections[i].index}.`;
+        const logStr = `Selecting query engine ${engineInd}: ${result.selections[i]}.`;
 
         if (this.verbose) {
           console.log(logStr + "\n");

packages/llamaindex/src/engines/query/SubQuestionQueryEngine.ts

@@ -119,15 +119,15 @@ export class SubQuestionQueryEngine
         return null;
       }
 
-      const responseValue = await queryEngine?.call?.({
+      const responseText = await queryEngine?.call?.({
         query: question,
       });
 
-      if (responseValue == null) {
+      if (!responseText) {
         return null;
       }
 
-      const nodeText = `Sub question: ${question}\nResponse: ${typeof responseValue === "string" ? responseValue : JSON.stringify(responseValue)}`;
+      const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
       const node = new TextNode({ text: nodeText });
       return { node, score: 0 };
     } catch (error) {

packages/llamaindex/src/evaluation/Relevancy.ts

@@ -78,7 +78,7 @@ export class RelevancyEvaluator extends PromptMixin implements BaseEvaluator {
       serviceContext: this.serviceContext,
     });
 
-    const queryResponse = `Question: ${extractText(query)}\nResponse: ${response}`;
+    const queryResponse = `Question: ${query}\nResponse: ${response}`;
 
     const queryEngine = index.asQueryEngine();
 

packages/llamaindex/src/index.edge.ts

@@ -1,8 +1,6 @@
 import type { AgentEndEvent, AgentStartEvent } from "./agent/types.js";
 import type { RetrievalEndEvent, RetrievalStartEvent } from "./llm/types.js";
 
-export * from "@llamaindex/core/schema";
-
 declare module "@llamaindex/core/global" {
   export interface LlamaIndexEventMaps {
     "retrieve-start": RetrievalStartEvent;

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -386,7 +386,6 @@ export type VectorIndexRetrieverOptions = {
   index: VectorStoreIndex;
   similarityTopK?: number;
   topK?: TopKMap;
-  filters?: MetadataFilters;
 };
 
 export class VectorIndexRetriever implements BaseRetriever {
@@ -394,21 +393,14 @@ export class VectorIndexRetriever implements BaseRetriever {
   topK: TopKMap;
 
   serviceContext?: ServiceContext;
-  filters?: MetadataFilters;
 
-  constructor({
-    index,
-    similarityTopK,
-    topK,
-    filters,
-  }: VectorIndexRetrieverOptions) {
+  constructor({ index, similarityTopK, topK }: VectorIndexRetrieverOptions) {
     this.index = index;
     this.serviceContext = this.index.serviceContext;
     this.topK = topK ?? {
       [ModalityType.TEXT]: similarityTopK ?? DEFAULT_SIMILARITY_TOP_K,
       [ModalityType.IMAGE]: DEFAULT_SIMILARITY_TOP_K,
     };
-    this.filters = filters;
   }
 
   /**
@@ -451,7 +443,7 @@ export class VectorIndexRetriever implements BaseRetriever {
     query: MessageContent,
     type: ModalityType,
     vectorStore: VectorStore,
-    filters?: MetadataFilters,
+    preFilters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
     // convert string message to multi-modal format
     if (typeof query === "string") {
@@ -468,7 +460,7 @@ export class VectorIndexRetriever implements BaseRetriever {
           queryEmbedding,
           mode: VectorStoreQueryMode.DEFAULT,
           similarityTopK: this.topK[type],
-          filters: this.filters ?? filters ?? undefined,
+          filters: preFilters ?? undefined,
         });
         nodes = nodes.concat(this.buildNodeListFromQueryResult(result));
       }

packages/llamaindex/src/ingestion/IngestionPipeline.ts

@@ -1,4 +1,4 @@
-import type { BaseReader, TransformComponent } from "@llamaindex/core/schema";
+import type { TransformComponent } from "@llamaindex/core/schema";
 import {
   ModalityType,
   splitNodesByType,
@@ -6,6 +6,7 @@ import {
   type Document,
   type Metadata,
 } from "@llamaindex/core/schema";
+import type { BaseReader } from "../readers/type.js";
 import type { BaseDocumentStore } from "../storage/docStore/types.js";
 import type {
   VectorStore,
@@ -106,7 +107,6 @@ export class IngestionPipeline {
       inputNodes.push(this.documents);
     }
     if (this.reader) {
-      // fixme: empty parameter might cause error
       inputNodes.push(await this.reader.loadData());
     }
     return inputNodes.flat();

packages/llamaindex/src/llm/azure.ts

@@ -18,7 +18,6 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     openAIModel: "gpt-3.5-turbo-16k",
   },
   "gpt-4o": { contextWindow: 128000, openAIModel: "gpt-4o" },
-  "gpt-4o-mini": { contextWindow: 128000, openAIModel: "gpt-4o-mini" },
   "gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
   "gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
   "gpt-4-turbo": {
@@ -41,10 +40,6 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     contextWindow: 128000,
     openAIModel: "gpt-4o-2024-05-13",
   },
-  "gpt-4o-mini-2024-07-18": {
-    contextWindow: 128000,
-    openAIModel: "gpt-4o-mini-2024-07-18",
-  },
 };
 
 const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
@@ -78,7 +73,6 @@ const ALL_AZURE_API_VERSIONS = [
   "2024-03-01-preview",
   "2024-04-01-preview",
   "2024-05-01-preview",
-  "2024-06-01",
 ];
 
 const DEFAULT_API_VERSION = "2023-05-15";

packages/llamaindex/src/llm/openai.ts

@@ -6,7 +6,6 @@ import type {
   ClientOptions as OpenAIClientOptions,
 } from "openai";
 import { AzureOpenAI, OpenAI as OrigOpenAI } from "openai";
-import type { ChatModel } from "openai/resources/chat/chat";
 
 import {
   type BaseTool,
@@ -109,24 +108,16 @@ export const GPT4_MODELS = {
   "gpt-4o-2024-05-13": { contextWindow: 128000 },
   "gpt-4o-mini": { contextWindow: 128000 },
   "gpt-4o-mini-2024-07-18": { contextWindow: 128000 },
-  "gpt-4o-2024-08-06": { contextWindow: 128000 },
-  "gpt-4o-2024-09-14": { contextWindow: 128000 },
-  "gpt-4o-2024-10-14": { contextWindow: 128000 },
-  "gpt-4-0613": { contextWindow: 128000 },
-  "gpt-4-turbo-2024-04-09": { contextWindow: 128000 },
-  "gpt-4-0314": { contextWindow: 128000 },
-  "gpt-4-32k-0314": { contextWindow: 32768 },
 };
 
 // NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
 export const GPT35_MODELS = {
-  "gpt-3.5-turbo": { contextWindow: 16385 },
+  "gpt-3.5-turbo": { contextWindow: 4096 },
   "gpt-3.5-turbo-0613": { contextWindow: 4096 },
-  "gpt-3.5-turbo-16k": { contextWindow: 16385 },
-  "gpt-3.5-turbo-16k-0613": { contextWindow: 16385 },
-  "gpt-3.5-turbo-1106": { contextWindow: 16385 },
-  "gpt-3.5-turbo-0125": { contextWindow: 16385 },
-  "gpt-3.5-turbo-0301": { contextWindow: 16385 },
+  "gpt-3.5-turbo-16k": { contextWindow: 16384 },
+  "gpt-3.5-turbo-16k-0613": { contextWindow: 16384 },
+  "gpt-3.5-turbo-1106": { contextWindow: 16384 },
+  "gpt-3.5-turbo-0125": { contextWindow: 16384 },
 };
 
 /**
@@ -135,7 +126,7 @@ export const GPT35_MODELS = {
 export const ALL_AVAILABLE_OPENAI_MODELS = {
   ...GPT4_MODELS,
   ...GPT35_MODELS,
-} satisfies Record<ChatModel, { contextWindow: number }>;
+};
 
 export function isFunctionCallingModel(llm: LLM): llm is OpenAI {
   let model: string;
@@ -166,10 +157,8 @@ export type OpenAIAdditionalChatOptions = Omit<
 >;
 
 export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
-  model:
-    | ChatModel
-    // string & {} is a hack to allow any string, but still give autocomplete
-    | (string & {});
+  // Per completion OpenAI params
+  model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
   temperature: number;
   topP: number;
   maxTokens?: number;

packages/llamaindex/src/readers/AssemblyAIReader.ts

@@ -1,4 +1,4 @@
-import { type BaseReader, Document } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
 import { getEnv } from "@llamaindex/env";
 import type {
   BaseServiceParams,
@@ -8,6 +8,7 @@ import type {
   TranscriptSentence,
 } from "assemblyai";
 import { AssemblyAI } from "assemblyai";
+import type { BaseReader } from "./type.js";
 
 type AssemblyAIOptions = Partial<BaseServiceParams>;
 const defaultOptions = {

packages/llamaindex/src/readers/CSVReader.ts

@@ -1,6 +1,7 @@
-import { type BaseReader, Document, FileReader } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
 import type { ParseConfig } from "papaparse";
 import Papa from "papaparse";
+import { FileReader } from "./type.js";
 
 /**
  * papaparse-based csv parser

packages/llamaindex/src/readers/DiscordReader.ts

@@ -1,5 +1,5 @@
 import { REST, type RESTOptions } from "@discordjs/rest";
-import { Document, type BaseReader } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
 import { getEnv } from "@llamaindex/env";
 import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
 
@@ -7,7 +7,7 @@ import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
  * Represents a reader for Discord messages using @discordjs/rest
  * See https://github.com/discordjs/discord.js/tree/main/packages/rest
  */
-export class DiscordReader implements BaseReader {
+export class DiscordReader {
   private client: REST;
 
   constructor(

packages/llamaindex/src/readers/DocxReader.ts

@@ -1,5 +1,6 @@
-import { Document, FileReader } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
 import mammoth from "mammoth";
+import { FileReader } from "./type.js";
 
 export class DocxReader extends FileReader {
   /** DocxParser */

packages/llamaindex/src/readers/HTMLReader.ts

@@ -1,4 +1,6 @@
-import { Document, FileReader } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
+import { FileReader } from "./type.js";
+
 /**
  * Extract the significant text from an arbitrary HTML document.
  * The contents of any head, script, style, and xml tags are removed completely.

packages/llamaindex/src/readers/ImageReader.ts

@@ -1,5 +1,6 @@
 import type { Document } from "@llamaindex/core/schema";
-import { FileReader, ImageDocument } from "@llamaindex/core/schema";
+import { ImageDocument } from "@llamaindex/core/schema";
+import { FileReader } from "./type.js";
 
 /**
  * Reads the content of an image file into a Document object (which stores the image file as a Blob).

packages/llamaindex/src/readers/JSONReader.ts

@@ -1,5 +1,7 @@
 import type { JSONValue } from "@llamaindex/core/global";
-import { Document, FileReader } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
+import { FileReader } from "./type.js";
+
 export interface JSONReaderOptions {
   /**
    * Whether to ensure only ASCII characters.
@@ -183,7 +185,7 @@ export class JSONReader<T extends JSONValue> extends FileReader {
       return jsonStr;
     } catch (e) {
       throw new JSONStringifyError(
-        `Error stringifying JSON: ${e} in "${JSON.stringify(data)}"`,
+        `Error stringifying JSON: ${e} in "${data}"`,
       );
     }
   }

packages/llamaindex/src/readers/LlamaParseReader.ts

@@ -1,92 +1,7 @@
-import { Document, FileReader } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
 import { fs, getEnv } from "@llamaindex/env";
 import { filetypeinfo } from "magic-bytes.js";
-
-export type ResultType = "text" | "markdown" | "json";
-export type Language =
-  | "abq"
-  | "ady"
-  | "af"
-  | "ang"
-  | "ar"
-  | "as"
-  | "ava"
-  | "az"
-  | "be"
-  | "bg"
-  | "bh"
-  | "bho"
-  | "bn"
-  | "bs"
-  | "ch_sim"
-  | "ch_tra"
-  | "che"
-  | "cs"
-  | "cy"
-  | "da"
-  | "dar"
-  | "de"
-  | "en"
-  | "es"
-  | "et"
-  | "fa"
-  | "fr"
-  | "ga"
-  | "gom"
-  | "hi"
-  | "hr"
-  | "hu"
-  | "id"
-  | "inh"
-  | "is"
-  | "it"
-  | "ja"
-  | "kbd"
-  | "kn"
-  | "ko"
-  | "ku"
-  | "la"
-  | "lbe"
-  | "lez"
-  | "lt"
-  | "lv"
-  | "mah"
-  | "mai"
-  | "mi"
-  | "mn"
-  | "mr"
-  | "ms"
-  | "mt"
-  | "ne"
-  | "new"
-  | "nl"
-  | "no"
-  | "oc"
-  | "pi"
-  | "pl"
-  | "pt"
-  | "ro"
-  | "ru"
-  | "rs_cyrillic"
-  | "rs_latin"
-  | "sck"
-  | "sk"
-  | "sl"
-  | "sq"
-  | "sv"
-  | "sw"
-  | "ta"
-  | "tab"
-  | "te"
-  | "th"
-  | "tjk"
-  | "tl"
-  | "tr"
-  | "ug"
-  | "uk"
-  | "ur"
-  | "uz"
-  | "vi";
+import { FileReader, type Language, type ResultType } from "./type.js";
 
 const SUPPORT_FILE_EXT: string[] = [
   ".pdf",

packages/llamaindex/src/readers/MarkdownReader.ts

@@ -1,4 +1,5 @@
-import { Document, FileReader } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
+import { FileReader } from "./type.js";
 
 type MarkdownTuple = [string | null, string];
 

packages/llamaindex/src/readers/NotionReader.ts

@@ -1,7 +1,7 @@
-import type { BaseReader } from "@llamaindex/core/schema";
 import { Document } from "@llamaindex/core/schema";
 import type { Crawler, CrawlerOptions, Page } from "notion-md-crawler";
 import { crawler, pageToString } from "notion-md-crawler";
+import type { BaseReader } from "./type.js";
 
 type NotionReaderOptions = Pick<CrawlerOptions, "client" | "serializers">;
 

packages/llamaindex/src/readers/PDFReader.ts

@@ -1,14 +1,11 @@
-import { Document, FileReader } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
+import { FileReader } from "./type.js";
 
 /**
  * Read the text of a PDF
  */
 export class PDFReader extends FileReader {
   async loadDataAsContent(content: Uint8Array): Promise<Document[]> {
-    // XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error if a Buffer passed
-    if (content instanceof Buffer) {
-      content = new Uint8Array(content);
-    }
     const { totalPages, text } = await readPDF(content);
     return text.map((text, page) => {
       const metadata = {

packages/llamaindex/src/readers/SimpleDirectoryReader.edge.ts

@@ -1,8 +1,8 @@
-import type { BaseReader, FileReader } from "@llamaindex/core/schema";
 import { Document } from "@llamaindex/core/schema";
 import { path } from "@llamaindex/env";
 import { walk } from "../storage/FileSystem.js";
 import { TextFileReader } from "./TextFileReader.js";
+import type { BaseReader, FileReader } from "./type.js";
 import pLimit from "./utils.js";
 
 type ReaderCallback = (

packages/llamaindex/src/readers/SimpleDirectoryReader.ts

@@ -1,4 +1,3 @@
-import type { FileReader } from "@llamaindex/core/schema";
 import { Document } from "@llamaindex/core/schema";
 import { PapaCSVReader } from "./CSVReader.js";
 import { DocxReader } from "./DocxReader.js";
@@ -11,6 +10,7 @@ import {
   type SimpleDirectoryReaderLoadDataParams,
 } from "./SimpleDirectoryReader.edge.js";
 import { TextFileReader } from "./TextFileReader.js";
+import type { FileReader } from "./type.js";
 
 export const FILE_EXT_TO_READER: Record<string, FileReader> = {
   txt: new TextFileReader(),

packages/llamaindex/src/readers/SimpleMongoReader.ts

@@ -1,6 +1,7 @@
 import type { Metadata } from "@llamaindex/core/schema";
-import { type BaseReader, Document } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
 import type { MongoClient } from "mongodb";
+import type { BaseReader } from "./type.js";
 
 /**
  * Read in from MongoDB

packages/llamaindex/src/readers/TextFileReader.ts

@@ -1,4 +1,6 @@
-import { Document, FileReader } from "@llamaindex/core/schema";
+import { Document } from "@llamaindex/core/schema";
+import { FileReader } from "./type.js";
+
 /**
  * Read a .txt file
  */

packages/llamaindex/src/readers/type.ts

@@ -0,0 +1,125 @@
+import type { Document } from "@llamaindex/core/schema";
+import { fs, path } from "@llamaindex/env";
+
+/**
+ * A reader takes imports data into Document objects.
+ */
+export interface BaseReader {
+  loadData(...args: unknown[]): Promise<Document[]>;
+}
+
+/**
+ * A FileReader takes file paths and imports data into Document objects.
+ */
+export abstract class FileReader implements BaseReader {
+  abstract loadDataAsContent(
+    fileContent: Uint8Array,
+    fileName?: string,
+  ): Promise<Document[]>;
+
+  async loadData(filePath: string): Promise<Document[]> {
+    // XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error in PDFReader
+    const fileContent = new Uint8Array(await fs.readFile(filePath));
+    const fileName = path.basename(filePath);
+    const docs = await this.loadDataAsContent(fileContent, fileName);
+    docs.forEach(FileReader.addMetaData(filePath));
+    return docs;
+  }
+
+  static addMetaData(filePath: string) {
+    return (doc: Document, index: number) => {
+      // generate id as loadDataAsContent is only responsible for the content
+      doc.id_ = `${filePath}_${index + 1}`;
+      doc.metadata["file_path"] = path.resolve(filePath);
+      doc.metadata["file_name"] = path.basename(filePath);
+    };
+  }
+}
+
+// For LlamaParseReader.ts
+
+export type ResultType = "text" | "markdown" | "json";
+export type Language =
+  | "abq"
+  | "ady"
+  | "af"
+  | "ang"
+  | "ar"
+  | "as"
+  | "ava"
+  | "az"
+  | "be"
+  | "bg"
+  | "bh"
+  | "bho"
+  | "bn"
+  | "bs"
+  | "ch_sim"
+  | "ch_tra"
+  | "che"
+  | "cs"
+  | "cy"
+  | "da"
+  | "dar"
+  | "de"
+  | "en"
+  | "es"
+  | "et"
+  | "fa"
+  | "fr"
+  | "ga"
+  | "gom"
+  | "hi"
+  | "hr"
+  | "hu"
+  | "id"
+  | "inh"
+  | "is"
+  | "it"
+  | "ja"
+  | "kbd"
+  | "kn"
+  | "ko"
+  | "ku"
+  | "la"
+  | "lbe"
+  | "lez"
+  | "lt"
+  | "lv"
+  | "mah"
+  | "mai"
+  | "mi"
+  | "mn"
+  | "mr"
+  | "ms"
+  | "mt"
+  | "ne"
+  | "new"
+  | "nl"
+  | "no"
+  | "oc"
+  | "pi"
+  | "pl"
+  | "pt"
+  | "ro"
+  | "ru"
+  | "rs_cyrillic"
+  | "rs_latin"
+  | "sck"
+  | "sk"
+  | "sl"
+  | "sq"
+  | "sv"
+  | "sw"
+  | "ta"
+  | "tab"
+  | "te"
+  | "th"
+  | "tjk"
+  | "tl"
+  | "tr"
+  | "ug"
+  | "uk"
+  | "ur"
+  | "uz"
+  | "vi";

packages/llamaindex/src/storage/vectorStore/PGVectorStore.ts

@@ -273,7 +273,7 @@ export class PGVectorStore
       const paramIndex = params.length + 1;
       whereClauses.push(`metadata->>'${filter.key}' = $${paramIndex}`);
       // TODO: support filter with other operators
-      if (!Array.isArray(filter.value) && filter.value) {
+      if (!Array.isArray(filter.value)) {
         params.push(filter.value);
       }
     });

packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts

@@ -36,7 +36,7 @@ type MetadataValue = Record<string, any>;
 
 // Mapping of filter operators to metadata filter functions
 const OPERATOR_TO_FILTER: {
-  [key in FilterOperator]?: (
+  [key in FilterOperator]: (
     { key, value }: MetadataFilter,
     metadata: MetadataValue,
   ) => boolean;
@@ -94,20 +94,7 @@ const buildFilterFn = (
   const queryCondition = condition || "and"; // default to and
 
   const itemFilterFn = (filter: MetadataFilter): boolean => {
-    if (filter.operator === FilterOperator.IS_EMPTY) {
-      // for `is_empty` operator, return true if the metadata key is not present or the value is empty
-      const value = metadata[filter.key];
-      return (
-        value === undefined ||
-        value === null ||
-        value === "" ||
-        (Array.isArray(value) && value.length === 0)
-      );
-    }
-    if (metadata[filter.key] === undefined) {
-      // for other operators, always return false if the metadata key is not present
-      return false;
-    }
+    if (metadata[filter.key] === undefined) return false; // always return false if the metadata key is not present
     const metadataLookupFn = OPERATOR_TO_FILTER[filter.operator];
     if (!metadataLookupFn)
       throw new Error(`Unsupported operator: ${filter.operator}`);

packages/llamaindex/src/storage/vectorStore/types.ts

@@ -33,7 +33,6 @@ export enum FilterOperator {
   ALL = "all", // Contains all (array of strings)
   TEXT_MATCH = "text_match", // full text match (allows you to search for a specific substring, token or phrase within the text field)
   CONTAINS = "contains", // metadata array contains value (string or number)
-  IS_EMPTY = "is_empty", // the field is not exist or empty (null or empty array)
 }
 
 export enum FilterCondition {
@@ -45,7 +44,7 @@ export type MetadataFilterValue = string | number | string[] | number[];
 
 export interface MetadataFilter {
   key: string;
-  value?: MetadataFilterValue;
+  value: MetadataFilterValue;
   operator: `${FilterOperator}`; // ==, any, all,...
 }
 

packages/llamaindex/src/storage/vectorStore/utils.ts

@@ -80,7 +80,7 @@ export function metadataDictToNode(
 }
 
 export const parsePrimitiveValue = (
-  value?: MetadataFilterValue,
+  value: MetadataFilterValue,
 ): string | number => {
   if (typeof value !== "number" && typeof value !== "string") {
     throw new Error("Value must be a string or number");
@@ -89,7 +89,7 @@ export const parsePrimitiveValue = (
 };
 
 export const parseArrayValue = (
-  value?: MetadataFilterValue,
+  value: MetadataFilterValue,
 ): string[] | number[] => {
   const isPrimitiveArray =
     Array.isArray(value) &&

packages/llamaindex/tests/CHANGELOG.md

@@ -1,11 +1,5 @@
 # @llamaindex/core-test
 
-## 0.0.7
-
-### Patch Changes
-
-- 01c184c: Add is_empty operator for filtering vector store
-
 ## 0.0.6
 
 ### Patch Changes

packages/llamaindex/tests/import.test.ts

@@ -1,11 +0,0 @@
-import { expect, test } from "vitest";
-
-test("Node classes should be included in the top level", async () => {
-  const { Document, IndexNode, TextNode, BaseNode } = await import(
-    "llamaindex"
-  );
-  expect(Document).toBeDefined();
-  expect(IndexNode).toBeDefined();
-  expect(TextNode).toBeDefined();
-  expect(BaseNode).toBeDefined();
-});

packages/llamaindex/tests/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llamaindex-test",
   "private": true,
-  "version": "0.0.7",
+  "version": "0.0.6",
   "type": "module",
   "scripts": {
     "test": "vitest run"

packages/llamaindex/tests/vectorStores/SimpleVectorStore.test.ts

@@ -256,18 +256,6 @@ describe("SimpleVectorStore", () => {
         },
         expected: 1,
       },
-      {
-        title: "Filter IS_EMPTY",
-        filters: {
-          filters: [
-            {
-              key: "not-exist-key",
-              operator: "is_empty",
-            },
-          ],
-        },
-        expected: 3,
-      },
       {
         title: "Filter OR",
         filters: {