Merge main into sweep/vercel-edge-runtime-support

.changeset/fluffy-spiders-turn.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Give HistoryChatEngine pluggable options (thanks @marcusschiesser)

.changeset/good-badgers-train.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Add HTMLReader (thanks @mtutty)

.changeset/pretty-points-kneel.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Added LLMMetadata (thanks @marcusschiesser)

.changeset/shiny-lions-clap.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Add observer/filter to the SimpleDirectoryReader (thanks @mtutty)

.gitignore

@@ -3,7 +3,6 @@
 # dependencies
 node_modules
 .pnp
-.pnpm-store
 .pnp.js
 
 # testing

README.md

@@ -25,14 +25,15 @@ pnpm install llamaindex
 pnpm install @types/node
 ```
 
-Create the file example.ts
-
-```ts
 // example.ts
 import fs from "fs/promises";
 import { Document, VectorStoreIndex } from "llamaindex";
+import { Tiktoken } from "@dqbd/tiktoken";
 
 async function main() {
+  // Initialize Tiktoken with the WASM binary file
+  const tiktoken = new Tiktoken({ wasmURL: '/path/to/tiktoken_bg.wasm' });
+
   // Load essay from abramov.txt in Node
   const essay = await fs.readFile(
     "node_modules/llamaindex/examples/abramov.txt",
@@ -56,7 +57,6 @@ async function main() {
 }
 
 main();
-```
 
 Then you can run it using
 

apps/docs/docs/starter.md

@@ -20,8 +20,12 @@ Create the file `example.ts`. This code will load some example data, create a do
 // example.ts
 import fs from "fs/promises";
 import { Document, VectorStoreIndex } from "llamaindex";
+import { Tiktoken } from "@dqbd/tiktoken";
 
 async function main() {
+  // Initialize Tiktoken with the WASM binary file
+  const tiktoken = new Tiktoken({ wasmURL: '/path/to/tiktoken_bg.wasm' });
+
   // Load essay from abramov.txt in Node
   const essay = await fs.readFile(
     "node_modules/llamaindex/examples/abramov.txt",

apps/simple/CHANGELOG.md

@@ -1,14 +1,5 @@
 # simple
 
-## 0.0.29
-
-### Patch Changes
-
-- Updated dependencies [6c55b2d]
-- Updated dependencies [8aa8c65]
-- Updated dependencies [6c55b2d]
-  - llamaindex@0.0.31
-
 ## 0.0.28
 
 ### Patch Changes

apps/simple/directory.ts

@@ -1,24 +0,0 @@
-import { SimpleDirectoryReader } from "llamaindex";
-
-function callback(
-  category: string,
-  name: string,
-  status: any,
-  message?: string,
-): boolean {
-  console.log(category, name, status, message);
-  if (name.endsWith(".pdf")) {
-    console.log("I DON'T WANT PDF FILES!");
-    return false;
-  }
-  return true;
-}
-
-async function main() {
-  // Load page
-  const reader = new SimpleDirectoryReader(callback);
-  const params = { directoryPath: "./data" };
-  await reader.loadData(params);
-}
-
-main().catch(console.error);

apps/simple/html.ts

@@ -1,21 +0,0 @@
-import { HTMLReader, VectorStoreIndex } from "llamaindex";
-
-async function main() {
-  // Load page
-  const reader = new HTMLReader();
-  const documents = await reader.loadData("data/18-1_Changelog.html");
-
-  // Split text and create embeddings. Store them in a VectorStoreIndex
-  const index = await VectorStoreIndex.fromDocuments(documents);
-
-  // Query the index
-  const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What were the notable changes in 18.1?",
-  );
-
-  // Output response
-  console.log(response.toString());
-}
-
-main().catch(console.error);

apps/simple/package.json

@@ -1,16 +1,15 @@
 {
-  "version": "0.0.29",
+  "version": "0.0.28",
   "private": true,
   "name": "simple",
   "dependencies": {
-    "@notionhq/client": "^2.2.13",
-    "@pinecone-database/pinecone": "^1.1.2",
-    "commander": "^11.1.0",
+    "@notionhq/client": "^2.2.12",
+    "@pinecone-database/pinecone": "^1.0.1",
+    "commander": "^11.0.0",
     "llamaindex": "workspace:*"
   },
   "devDependencies": {
-    "@types/node": "^18.18.6",
-    "ts-node": "^10.9.1"
+    "@types/node": "^18.17.12"
   },
   "scripts": {
     "lint": "eslint ."

apps/simple/vectorIndexCustomize.ts

@@ -3,7 +3,6 @@ import {
   OpenAI,
   RetrieverQueryEngine,
   serviceContextFromDefaults,
-  SimilarityPostprocessor,
   VectorStoreIndex,
 } from "llamaindex";
 import essay from "./essay";
@@ -22,16 +21,8 @@ async function main() {
 
   const retriever = index.asRetriever();
   retriever.similarityTopK = 5;
-  const nodePostprocessor = new SimilarityPostprocessor({
-    similarityCutoff: 0.7,
-  });
   // TODO: cannot pass responseSynthesizer into retriever query engine
-  const queryEngine = new RetrieverQueryEngine(
-    retriever,
-    undefined,
-    undefined,
-    [nodePostprocessor],
-  );
+  const queryEngine = new RetrieverQueryEngine(retriever);
 
   const response = await queryEngine.query(
     "What did the author do growing up?",

package.json

@@ -11,16 +11,16 @@
     "publish-snapshot": "turbo run build lint test && changeset version --snapshot && changeset publish"
   },
   "devDependencies": {
-    "@turbo/gen": "^1.10.16",
-    "@types/jest": "^29.5.6",
-    "eslint": "^8.52.0",
+    "@turbo/gen": "^1.10.15",
+    "@types/jest": "^29.5.5",
+    "eslint": "^7.32.0",
     "eslint-config-custom": "workspace:*",
     "husky": "^8.0.3",
     "jest": "^29.7.0",
     "prettier": "^3.0.3",
     "prettier-plugin-organize-imports": "^3.2.3",
     "ts-jest": "^29.1.1",
-    "turbo": "^1.10.16"
+    "turbo": "^1.10.15"
   },
   "packageManager": "pnpm@7.15.0",
   "dependencies": {
@@ -28,8 +28,7 @@
   },
   "pnpm": {
     "overrides": {
-      "trim": "1.0.1",
-      "@babel/traverse": "7.23.2"
+      "trim": "1.0.1"
     }
   }
 }

packages/core/CHANGELOG.md

@@ -1,13 +1,5 @@
 # llamaindex
 
-## 0.0.31
-
-### Patch Changes
-
-- 6c55b2d: Give HistoryChatEngine pluggable options (thanks @marcusschiesser)
-- 8aa8c65: Add SimilarityPostProcessor (thanks @TomPenguin)
-- 6c55b2d: Added LLMMetadata (thanks @marcusschiesser)
-
 ## 0.0.30
 
 ### Patch Changes

packages/core/package.json

@@ -1,35 +1,34 @@
 {
   "name": "llamaindex",
-  "version": "0.0.31",
+  "version": "0.0.30",
   "license": "MIT",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.8.1",
+    "@anthropic-ai/sdk": "^0.6.2",
     "@notionhq/client": "^2.2.13",
-    "js-tiktoken": "^1.0.7",
     "lodash": "^4.17.21",
     "mammoth": "^1.6.0",
     "md-utils-ts": "^2.0.0",
-    "mongodb": "^6.2.0",
+    "mongodb": "^6.1.0",
     "notion-md-crawler": "^0.0.2",
-    "openai": "^4.14.0",
+    "openai": "^4.11.1",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
     "portkey-ai": "^0.1.13",
     "rake-modified": "^1.0.8",
     "replicate": "^0.20.1",
-    "string-strip-html": "^13.4.3",
+    "tiktoken": "^1.0.10",
     "uuid": "^9.0.1",
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
-    "@types/lodash": "^4.14.200",
-    "@types/node": "^18.18.7",
-    "@types/papaparse": "^5.3.10",
-    "@types/pdf-parse": "^1.1.3",
-    "@types/uuid": "^9.0.6",
+    "@types/lodash": "^4.14.199",
+    "@types/node": "^18.18.4",
+    "@types/papaparse": "^5.3.9",
+    "@types/pdf-parse": "^1.1.2",
+    "@types/uuid": "^9.0.5",
     "node-stdlib-browser": "^1.2.0",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "^4.9.5"
   },
   "engines": {
     "node": ">=18.0.0"

packages/core/src/ChatEngine.ts

@@ -1,7 +1,6 @@
 import { v4 as uuidv4 } from "uuid";
 import { Event } from "./callbacks/CallbackManager";
 import { ChatHistory } from "./ChatHistory";
-import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
 import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 import { NodeWithScore, TextNode } from "./Node";
 import {
@@ -179,24 +178,14 @@ export interface ContextGenerator {
 export class DefaultContextGenerator implements ContextGenerator {
   retriever: BaseRetriever;
   contextSystemPrompt: ContextSystemPrompt;
-  nodePostprocessors: BaseNodePostprocessor[];
 
   constructor(init: {
     retriever: BaseRetriever;
     contextSystemPrompt?: ContextSystemPrompt;
-    nodePostprocessors?: BaseNodePostprocessor[];
   }) {
     this.retriever = init.retriever;
     this.contextSystemPrompt =
       init?.contextSystemPrompt ?? defaultContextSystemPrompt;
-    this.nodePostprocessors = init.nodePostprocessors || [];
-  }
-
-  private applyNodePostprocessors(nodes: NodeWithScore[]) {
-    return this.nodePostprocessors.reduce(
-      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
-      nodes,
-    );
   }
 
   async generate(message: string, parentEvent?: Event): Promise<Context> {
@@ -212,16 +201,16 @@ export class DefaultContextGenerator implements ContextGenerator {
       parentEvent,
     );
 
-    const nodes = this.applyNodePostprocessors(sourceNodesWithScore);
-
     return {
       message: {
         content: this.contextSystemPrompt({
-          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
+          context: sourceNodesWithScore
+            .map((r) => (r.node as TextNode).text)
+            .join("\n\n"),
         }),
         role: "system",
       },
-      nodes,
+      nodes: sourceNodesWithScore,
     };
   }
 }
@@ -241,7 +230,6 @@ export class ContextChatEngine implements ChatEngine {
     chatModel?: LLM;
     chatHistory?: ChatMessage[];
     contextSystemPrompt?: ContextSystemPrompt;
-    nodePostprocessors?: BaseNodePostprocessor[];
   }) {
     this.chatModel =
       init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });

packages/core/src/GlobalsHelper.ts

@@ -1,4 +1,5 @@
-import { encodingForModel, TiktokenModel } from "js-tiktoken";
+import cl100k_base from "tiktoken/encoders/cl100k_base.json";
+import { Tiktoken } from "tiktoken/lite";
 
 import { v4 as uuidv4 } from "uuid";
 import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
@@ -17,17 +18,18 @@ class GlobalsHelper {
   } | null = null;
 
   private initDefaultTokenizer() {
-    const encoding = encodingForModel("text-embedding-ada-002"); // cl100k_base
+    const encoding = new Tiktoken(
+      cl100k_base.bpe_ranks,
+      cl100k_base.special_tokens,
+      cl100k_base.pat_str,
+    );
 
     this.defaultTokenizer = {
       encode: (text: string) => {
-        return new Uint32Array(encoding.encode(text));
+        return encoding.encode(text);
       },
       decode: (tokens: Uint32Array) => {
-        const numberArray = Array.from(tokens);
-        const text = encoding.decode(numberArray);
-        const uint8Array = new TextEncoder().encode(text);
-        return new TextDecoder().decode(uint8Array);      
+        return new TextDecoder().decode(encoding.decode(tokens));
       },
     };
   }
@@ -39,10 +41,10 @@ class GlobalsHelper {
     if (!this.defaultTokenizer) {
       this.initDefaultTokenizer();
     }
-  
+
     return this.defaultTokenizer!.encode.bind(this.defaultTokenizer);
   }
-  
+
   tokenizerDecoder(encoding?: string) {
     if (encoding && encoding !== Tokenizers.CL100K_BASE) {
       throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
@@ -50,7 +52,7 @@ class GlobalsHelper {
     if (!this.defaultTokenizer) {
       this.initDefaultTokenizer();
     }
-  
+
     return this.defaultTokenizer!.decode.bind(this.defaultTokenizer);
   }
 

packages/core/src/QueryEngine.ts

@@ -1,6 +1,5 @@
 import { v4 as uuidv4 } from "uuid";
 import { Event } from "./callbacks/CallbackManager";
-import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   BaseQuestionGenerator,
@@ -31,14 +30,12 @@ export interface BaseQueryEngine {
 export class RetrieverQueryEngine implements BaseQueryEngine {
   retriever: BaseRetriever;
   responseSynthesizer: ResponseSynthesizer;
-  nodePostprocessors: BaseNodePostprocessor[];
   preFilters?: unknown;
 
   constructor(
     retriever: BaseRetriever,
     responseSynthesizer?: ResponseSynthesizer,
     preFilters?: unknown,
-    nodePostprocessors?: BaseNodePostprocessor[],
   ) {
     this.retriever = retriever;
     const serviceContext: ServiceContext | undefined =
@@ -46,24 +43,6 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
     this.responseSynthesizer =
       responseSynthesizer || new ResponseSynthesizer({ serviceContext });
     this.preFilters = preFilters;
-    this.nodePostprocessors = nodePostprocessors || [];
-  }
-
-  private applyNodePostprocessors(nodes: NodeWithScore[]) {
-    return this.nodePostprocessors.reduce(
-      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
-      nodes,
-    );
-  }
-
-  private async retrieve(query: string, parentEvent: Event) {
-    const nodes = await this.retriever.retrieve(
-      query,
-      parentEvent,
-      this.preFilters,
-    );
-
-    return this.applyNodePostprocessors(nodes);
   }
 
   async query(query: string, parentEvent?: Event) {
@@ -72,7 +51,11 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
       type: "wrapper",
       tags: ["final"],
     };
-    const nodes = await this.retrieve(query, _parentEvent);
+    const nodes = await this.retriever.retrieve(
+      query,
+      _parentEvent,
+      this.preFilters,
+    );
     return this.responseSynthesizer.synthesize(query, nodes, _parentEvent);
   }
 }

packages/core/src/index.ts

@@ -18,7 +18,6 @@ export * from "./readers/CSVReader";
 export * from "./readers/MarkdownReader";
 export * from "./readers/NotionReader";
 export * from "./readers/PDFReader";
-export * from "./readers/HTMLReader";
 export * from "./readers/SimpleDirectoryReader";
 export * from "./Response";
 export * from "./ResponseSynthesizer";

packages/core/src/indices/BaseNodePostprocessor.ts

@@ -1,20 +0,0 @@
-import { NodeWithScore } from "../Node";
-
-export interface BaseNodePostprocessor {
-  postprocessNodes: (nodes: NodeWithScore[]) => NodeWithScore[];
-}
-
-export class SimilarityPostprocessor implements BaseNodePostprocessor {
-  similarityCutoff?: number;
-
-  constructor(options?: { similarityCutoff?: number }) {
-    this.similarityCutoff = options?.similarityCutoff;
-  }
-
-  postprocessNodes(nodes: NodeWithScore[]) {
-    if (this.similarityCutoff === undefined) return nodes;
-
-    const cutoff = this.similarityCutoff || 0;
-    return nodes.filter((node) => node.score && node.score >= cutoff);
-  }
-}

packages/core/src/indices/index.ts

@@ -1,5 +1,4 @@
 export * from "./BaseIndex";
-export * from "./BaseNodePostprocessor";
 export * from "./keyword";
 export * from "./summary";
 export * from "./vectorStore";

packages/core/src/indices/keyword/KeywordTableIndex.ts

@@ -15,7 +15,6 @@ import {
   IndexStructType,
   KeywordTable,
 } from "../BaseIndex";
-import { BaseNodePostprocessor } from "../BaseNodePostprocessor";
 import {
   KeywordTableLLMRetriever,
   KeywordTableRAKERetriever,
@@ -130,15 +129,11 @@ export class KeywordTableIndex extends BaseIndex<KeywordTable> {
   asQueryEngine(options?: {
     retriever?: BaseRetriever;
     responseSynthesizer?: ResponseSynthesizer;
-    preFilters?: unknown;
-    nodePostprocessors?: BaseNodePostprocessor[];
   }): BaseQueryEngine {
     const { retriever, responseSynthesizer } = options ?? {};
     return new RetrieverQueryEngine(
       retriever ?? this.asRetriever(),
       responseSynthesizer,
-      options?.preFilters,
-      options?.nodePostprocessors,
     );
   }
 

packages/core/src/indices/summary/SummaryIndex.ts

@@ -10,18 +10,17 @@ import {
   ServiceContext,
   serviceContextFromDefaults,
 } from "../../ServiceContext";
-import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   StorageContext,
   storageContextFromDefaults,
 } from "../../storage/StorageContext";
+import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   BaseIndex,
   BaseIndexInit,
   IndexList,
   IndexStructType,
 } from "../BaseIndex";
-import { BaseNodePostprocessor } from "../BaseNodePostprocessor";
 import {
   SummaryIndexLLMRetriever,
   SummaryIndexRetriever,
@@ -156,8 +155,6 @@ export class SummaryIndex extends BaseIndex<IndexList> {
   asQueryEngine(options?: {
     retriever?: BaseRetriever;
     responseSynthesizer?: ResponseSynthesizer;
-    preFilters?: unknown;
-    nodePostprocessors?: BaseNodePostprocessor[];
   }): BaseQueryEngine {
     let { retriever, responseSynthesizer } = options ?? {};
 
@@ -173,12 +170,7 @@ export class SummaryIndex extends BaseIndex<IndexList> {
       });
     }
 
-    return new RetrieverQueryEngine(
-      retriever,
-      responseSynthesizer,
-      options?.preFilters,
-      options?.nodePostprocessors,
-    );
+    return new RetrieverQueryEngine(retriever, responseSynthesizer);
   }
 
   static async buildIndexFromNodes(

packages/core/src/indices/vectorStore/VectorStoreIndex.ts

@@ -18,7 +18,6 @@ import {
   IndexDict,
   IndexStructType,
 } from "../BaseIndex";
-import { BaseNodePostprocessor } from "../BaseNodePostprocessor";
 import { VectorIndexRetriever } from "./VectorIndexRetriever";
 
 export interface VectorIndexOptions {
@@ -247,15 +246,11 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
   asQueryEngine(options?: {
     retriever?: BaseRetriever;
     responseSynthesizer?: ResponseSynthesizer;
-    preFilters?: unknown;
-    nodePostprocessors?: BaseNodePostprocessor[];
   }): BaseQueryEngine {
     const { retriever, responseSynthesizer } = options ?? {};
     return new RetrieverQueryEngine(
       retriever ?? this.asRetriever(),
       responseSynthesizer,
-      options?.preFilters,
-      options?.nodePostprocessors,
     );
   }
 

packages/core/src/readers/HTMLReader.ts

@@ -1,77 +0,0 @@
-import { Document } from "../Node";
-import { DEFAULT_FS } from "../storage/constants";
-import { GenericFileSystem } from "../storage/FileSystem";
-import { BaseReader } from "./base";
-
-/**
- * Extract the significant text from an arbitrary HTML document.
- * The contents of any head, script, style, and xml tags are removed completely.
- * The URLs for a[href] tags are extracted, along with the inner text of the tag.
- * All other tags are removed, and the inner text is kept intact.
- * Html entities (e.g., &) are not decoded.
- */
-export class HTMLReader implements BaseReader {
-  /**
-   * Public method for this reader.
-   * Required by BaseReader interface.
-   * @param file Path/name of the file to be loaded.
-   * @param fs fs wrapper interface for getting the file content.
-   * @returns Promise<Document[]> A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
-   */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file, "utf-8");
-    const htmlOptions = this.getOptions();
-    const content = await this.parseContent(dataBuffer, htmlOptions);
-    return [new Document({ text: content, id_: file })];
-  }
-
-  /**
-   * Wrapper for string-strip-html usage.
-   * @param html Raw HTML content to be parsed.
-   * @param options An object of options for the underlying library
-   * @see getOptions
-   * @returns The HTML content, stripped of unwanted tags and attributes
-   */
-  async parseContent(html: string, options: any = {}): Promise<string> {
-    const { stripHtml } = await import("string-strip-html"); // ESM only
-    return stripHtml(html).result;
-  }
-
-  /**
-   * Wrapper for our configuration options passed to string-strip-html library
-   * @see https://codsen.com/os/string-strip-html/examples
-   * @returns An object of options for the underlying library
-   */
-  getOptions() {
-    return {
-      skipHtmlDecoding: true,
-      stripTogetherWithTheirContents: [
-        "script", // default
-        "style", // default
-        "xml", // default
-        "head", // <-- custom-added
-      ],
-      // Keep the URLs for embedded links
-      // cb: (tag: any, deleteFrom: number, deleteTo: number, insert: string, rangesArr: any, proposedReturn: string) => {
-      //   let temp;
-      //   if (
-      //     tag.name === "a" &&
-      //     tag.attributes &&
-      //     tag.attributes.some((attr: any) => {
-      //       if (attr.name === "href") {
-      //         temp = attr.value;
-      //         return true;
-      //       }
-      //     })
-      //   ) {
-      //     rangesArr.push([deleteFrom, deleteTo, `${temp} ${insert || ""}`]);
-      //   } else {
-      //     rangesArr.push(proposedReturn);
-      //   }
-      // },
-    };
-  }
-}

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -1,25 +1,12 @@
 import _ from "lodash";
 import { Document } from "../Node";
-import { CompleteFileSystem, walk } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
+import { CompleteFileSystem, walk } from "../storage/FileSystem";
+import { BaseReader } from "./base";
 import { PapaCSVReader } from "./CSVReader";
 import { DocxReader } from "./DocxReader";
-import { HTMLReader } from "./HTMLReader";
 import { MarkdownReader } from "./MarkdownReader";
 import { PDFReader } from "./PDFReader";
-import { BaseReader } from "./base";
-
-type ReaderCallback = (
-  category: "file" | "directory",
-  name: string,
-  status: ReaderStatus,
-  message?: string,
-) => boolean;
-enum ReaderStatus {
-  STARTED = 0,
-  COMPLETE,
-  ERROR,
-}
 
 /**
  * Read a .txt file
@@ -34,14 +21,12 @@ export class TextFileReader implements BaseReader {
   }
 }
 
-export const FILE_EXT_TO_READER: Record<string, BaseReader> = {
+const FILE_EXT_TO_READER: Record<string, BaseReader> = {
   txt: new TextFileReader(),
   pdf: new PDFReader(),
   csv: new PapaCSVReader(),
   md: new MarkdownReader(),
   docx: new DocxReader(),
-  htm: new HTMLReader(),
-  html: new HTMLReader(),
 };
 
 export type SimpleDirectoryReaderLoadDataProps = {
@@ -52,37 +37,20 @@ export type SimpleDirectoryReaderLoadDataProps = {
 };
 
 /**
- * Read all of the documents in a directory.
- * By default, supports the list of file types
- * in the FILE_EXIT_TO_READER map.
+ * Read all of the documents in a directory. Currently supports PDF and TXT files.
  */
 export class SimpleDirectoryReader implements BaseReader {
-  constructor(private observer?: ReaderCallback) {}
-
   async loadData({
     directoryPath,
     fs = DEFAULT_FS as CompleteFileSystem,
     defaultReader = new TextFileReader(),
     fileExtToReader = FILE_EXT_TO_READER,
   }: SimpleDirectoryReaderLoadDataProps): Promise<Document[]> {
-    // Observer can decide to skip the directory
-    if (
-      !this.doObserverCheck("directory", directoryPath, ReaderStatus.STARTED)
-    ) {
-      return [];
-    }
-
     let docs: Document[] = [];
     for await (const filePath of walk(fs, directoryPath)) {
       try {
         const fileExt = _.last(filePath.split(".")) || "";
 
-        // Observer can decide to skip each file
-        if (!this.doObserverCheck("file", filePath, ReaderStatus.STARTED)) {
-          // Skip this file
-          continue;
-        }
-
         let reader = null;
 
         if (fileExt in fileExtToReader) {
@@ -90,52 +58,16 @@ export class SimpleDirectoryReader implements BaseReader {
         } else if (!_.isNil(defaultReader)) {
           reader = defaultReader;
         } else {
-          const msg = `No reader for file extension of ${filePath}`;
-          console.warn(msg);
-
-          // In an error condition, observer's false cancels the whole process.
-          if (
-            !this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)
-          ) {
-            return [];
-          }
-
+          console.warn(`No reader for file extension of ${filePath}`);
           continue;
         }
 
         const fileDocs = await reader.loadData(filePath, fs);
-
-        // Observer can still cancel addition of the resulting docs from this file
-        if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) {
-          docs.push(...fileDocs);
-        }
+        docs.push(...fileDocs);
       } catch (e) {
-        const msg = `Error reading file ${filePath}: ${e}`;
-        console.error(msg);
-
-        // In an error condition, observer's false cancels the whole process.
-        if (!this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)) {
-          return [];
-        }
+        console.error(`Error reading file ${filePath}: ${e}`);
       }
     }
-
-    // After successful import of all files, directory completion
-    // is only a notification for observer, cannot be cancelled.
-    this.doObserverCheck("directory", directoryPath, ReaderStatus.COMPLETE);
-
     return docs;
   }
-
-  private doObserverCheck(
-    category: "file" | "directory",
-    name: string,
-    status: ReaderStatus,
-    message?: string,
-  ): boolean {
-    if (this.observer) {
-      return this.observer(category, name, status, message);
-    }
-    return true;
-  }
 }

packages/core/src/tests/TextSplitter.test.ts

@@ -73,7 +73,6 @@ describe("SentenceSplitter", () => {
     let splits = sentenceSplitter.splitText(
       "This is a sentence. This is another sentence. 1.0",
     );
-
     expect(splits).toEqual([
       "This is a sentence.",
       "This is another sentence.",

packages/core/tsconfig.json

@@ -3,7 +3,6 @@
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
     "isolatedModules": true,
-    "module": "esnext",
     "moduleResolution": "node",
     "preserveWatchOutput": true,
     "skipLibCheck": true,

packages/eslint-config-custom/package.json

@@ -8,7 +8,8 @@
     "eslint-config-next": "^13.4.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-config-turbo": "^1.9.3",
-    "eslint-plugin-react": "7.28.0"
+    "eslint-plugin-react": "7.28.0",
+    "@dqbd/tiktoken": "^1.0.7"
   },
   "publishConfig": {
     "access": "public"