remove extract text

Using `pnpm format:write` will cause two commits which is not expected

.changeset/new-shirts-reply.md

@@ -0,0 +1,10 @@
+---
+"llamaindex": patch
+"@llamaindex/env": patch
+---
+
+refactor: remove `defaultFS` from parameters
+
+We don't accept passing fs in the parameter since it's unnecessary for a determined JS environment.
+
+This was a polyfill way for the non-Node.js environment, but now we use another way to polyfill APIs.

.changeset/sharp-knives-ring.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Improve Gemini message and context preparation

.changeset/wet-boxes-compete.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Force ChromaDB version to 1.7.3 (to prevent NextJS issues)

.husky/pre-commit

@@ -1,3 +1,3 @@
-pnpm format:write
+pnpm format
 pnpm lint
 npx lint-staged

apps/docs/CHANGELOG.md

@@ -1,5 +1,12 @@
 # docs
 
+## 0.0.18
+
+### Patch Changes
+
+- Updated dependencies [4aba02e]
+  - llamaindex@0.3.10
+
 ## 0.0.17
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.17",
+  "version": "0.0.18",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/package.json

@@ -8,7 +8,7 @@
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^2.2.0",
     "@zilliz/milvus2-sdk-node": "^2.4.2",
-    "chromadb": "^1.8.1",
+    "chromadb": "^1.7.3",
     "commander": "^12.0.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.11",

package.json

@@ -2,8 +2,8 @@
   "name": "@llamaindex/monorepo",
   "private": true,
   "scripts": {
-    "build": "turbo run build --filter=\"!docs\" --filter=\"!*-test\"",
-    "build:release": "turbo run build lint test --filter=\"!docs\" --filter=\"!*-test\"",
+    "build": "turbo run build --filter=\"!docs\" --filter=\"!*-test\" --filter=\"!*-example\"",
+    "build:release": "turbo run build lint test --filter=\"!docs\" --filter=\"!*-test\" --filter=\"!*-example\"",
     "dev": "turbo run dev",
     "format": "prettier --ignore-unknown --cache --check .",
     "format:write": "prettier --ignore-unknown --write .",

packages/autotool/examples/01_node/CHANGELOG.md

@@ -4,6 +4,14 @@
 
 ### Patch Changes
 
+- Updated dependencies [4aba02e]
+  - llamaindex@0.3.10
+  - @llamaindex/autotool@0.0.1
+
+## null
+
+### Patch Changes
+
 - Updated dependencies [c3747d0]
   - llamaindex@0.3.9
   - @llamaindex/autotool@0.0.1

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [4aba02e]
+  - llamaindex@0.3.10
+  - @llamaindex/autotool@0.0.1
+
 ## 0.1.1
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.1",
+  "version": "0.1.2",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -2,7 +2,11 @@
   "name": "@llamaindex/autotool",
   "type": "module",
   "version": "0.0.1",
-  "description": "",
+  "description": "auto transpile your JS function to LLM Agent compatible",
+  "files": [
+    "dist",
+    "CHANGELOG.md"
+  ],
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
@@ -47,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.3.9",
+    "llamaindex": "^0.3.10",
     "openai": "^4",
     "typescript": "^4"
   },

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # llamaindex
 
+## 0.3.10
+
+### Patch Changes
+
+- 4aba02e: feat: support gpt4-o
+
 ## 0.3.9
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [4aba02e]
+  - llamaindex@0.3.10
+
 ## 0.0.10
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/next-agent-test
 
+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [4aba02e]
+  - llamaindex@0.3.10
+
 ## 0.1.10
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,12 @@
 # test-edge-runtime
 
+## 0.1.10
+
+### Patch Changes
+
+- Updated dependencies [4aba02e]
+  - llamaindex@0.3.10
+
 ## 0.1.9
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.9",
+  "version": "0.1.10",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [4aba02e]
+  - llamaindex@0.3.10
+
 ## 0.0.10
 
 ### Patch Changes

packages/core/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/core",
-  "version": "0.3.9",
+  "version": "0.3.10",
   "exports": "./src/index.ts",
   "imports": {
     "@llamaindex/env": "jsr:@llamaindex/env@0.1.2"

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.3.9",
+  "version": "0.3.10",
   "expectedMinorVersion": "3",
   "license": "MIT",
   "type": "module",
@@ -39,7 +39,7 @@
     "@zilliz/milvus2-sdk-node": "^2.4.2",
     "ajv": "^8.13.0",
     "assemblyai": "^4.4.2",
-    "chromadb": "~1.8.1",
+    "chromadb": "~1.7.3",
     "cohere-ai": "^7.9.5",
     "js-tiktoken": "^1.0.11",
     "lodash": "^4.17.21",
@@ -48,7 +48,7 @@
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.6.1",
     "notion-md-crawler": "^1.0.0",
-    "openai": "^4.43.0",
+    "openai": "^4.46.0",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
     "pdf2json": "3.0.5",

packages/core/src/Prompt.ts

@@ -371,6 +371,7 @@ export function messagesToHistoryStr(messages: ChatMessage[]) {
 }
 
 export const defaultContextSystemPrompt = ({ context = "" }) => {
+  if (!context) return "";
   return `Context information is below.
 ---------------------
 ${context}

packages/core/src/agent/base.ts

@@ -21,7 +21,6 @@ import type {
   LLM,
   MessageContent,
 } from "../llm/index.js";
-import { extractText } from "../llm/utils.js";
 import type { BaseToolWithCall, ToolOutput } from "../types.js";
 import type {
   AgentTaskContext,
@@ -169,7 +168,7 @@ export abstract class AgentWorker<
   abstract taskHandler: TaskHandler<AI, Store, AdditionalMessageOptions>;
 
   public createTask(
-    query: string,
+    query: MessageContent,
     context: AgentTaskContext<AI, Store, AdditionalMessageOptions>,
   ): ReadableStream<TaskStepOutput<AI, Store, AdditionalMessageOptions>> {
     context.store.messages.push({
@@ -305,7 +304,7 @@ export abstract class AgentRunner<
         });
       }
     }
-    return this.#runner.createTask(extractText(message), {
+    return this.#runner.createTask(message, {
       stream,
       toolCallCount: 0,
       llm: this.#llm,

packages/core/src/embeddings/utils.ts

@@ -1,4 +1,4 @@
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import _ from "lodash";
 import { filetypemime } from "magic-bytes.js";
 import type { ImageType } from "../Node.js";
@@ -243,7 +243,7 @@ export async function imageToDataUrl(input: ImageType): Promise<string> {
     _.isString(input)
   ) {
     // string or file URL
-    const dataBuffer = await defaultFS.readFile(
+    const dataBuffer = await fs.readFile(
       input instanceof URL ? input.pathname : input,
     );
     input = new Blob([dataBuffer]);

packages/core/src/llm/gemini.ts

@@ -139,7 +139,7 @@ class GeminiHelper {
   > = {
     user: "user",
     system: "user",
-    assistant: "user",
+    assistant: "model",
     memory: "user",
   };
 
@@ -152,38 +152,26 @@ class GeminiHelper {
   };
 
   public static mergeNeighboringSameRoleMessages(
-    messages: ChatMessage[],
-  ): ChatMessage[] {
-    // Gemini does not support multiple messages of the same role in a row, so we merge them
-    const mergedMessages: ChatMessage[] = [];
-    let i: number = 0;
-
-    while (i < messages.length) {
-      const currentMessage: ChatMessage = messages[i];
-      // Initialize merged content with current message content
-      const mergedContent: MessageContent[] = [currentMessage.content];
-
-      // Check if the next message exists and has the same role
-      while (
-        i + 1 < messages.length &&
-        this.ROLES_TO_GEMINI[messages[i + 1].role] ===
-          this.ROLES_TO_GEMINI[currentMessage.role]
-      ) {
-        i++;
-        const nextMessage: ChatMessage = messages[i];
-        mergedContent.push(nextMessage.content);
-      }
-
-      // Create a new ChatMessage object with merged content
-      const mergedMessage: ChatMessage = {
-        role: currentMessage.role,
-        content: mergedContent.join("\n"),
-      };
-      mergedMessages.push(mergedMessage);
-      i++;
-    }
-
-    return mergedMessages;
+    messages: GeminiMessageContent[],
+  ): GeminiMessageContent[] {
+    return messages.reduce(
+      (
+        result: GeminiMessageContent[],
+        current: GeminiMessageContent,
+        index: number,
+      ) => {
+        if (index > 0 && messages[index - 1].role === current.role) {
+          result[result.length - 1].parts = [
+            ...result[result.length - 1].parts,
+            ...current.parts,
+          ];
+        } else {
+          result.push(current);
+        }
+        return result;
+      },
+      [],
+    );
   }
 
   public static messageContentToGeminiParts(content: MessageContent): Part[] {
@@ -214,8 +202,8 @@ class GeminiHelper {
     message: ChatMessage,
   ): GeminiMessageContent {
     return {
-      role: this.ROLES_TO_GEMINI[message.role],
-      parts: this.messageContentToGeminiParts(message.content),
+      role: GeminiHelper.ROLES_TO_GEMINI[message.role],
+      parts: GeminiHelper.messageContentToGeminiParts(message.content),
     };
   }
 }
@@ -260,22 +248,20 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
     chat: ChatSession;
     messageContent: Part[];
   } {
-    const { messages } = params;
-    const mergedMessages =
-      GeminiHelper.mergeNeighboringSameRoleMessages(messages);
-    const history = mergedMessages.slice(0, -1);
-    const nextMessage = mergedMessages[mergedMessages.length - 1];
-    const messageContent = GeminiHelper.chatMessageToGemini(nextMessage).parts;
+    const messages = GeminiHelper.mergeNeighboringSameRoleMessages(
+      params.messages.map(GeminiHelper.chatMessageToGemini),
+    );
+
+    const history = messages.slice(0, -1);
 
     const client = this.session.gemini.getGenerativeModel(this.metadata);
 
     const chat = client.startChat({
-      history: history.map(GeminiHelper.chatMessageToGemini),
+      history,
     });
-
     return {
       chat,
-      messageContent,
+      messageContent: messages[messages.length - 1].parts,
     };
   }
 

packages/core/src/llm/index.ts

@@ -5,7 +5,7 @@ export {
   Anthropic,
 } from "./anthropic.js";
 export { FireworksLLM } from "./fireworks.js";
-export { GEMINI_MODEL, Gemini } from "./gemini.js";
+export { GEMINI_MODEL, Gemini, GeminiSession } from "./gemini.js";
 export { Groq } from "./groq.js";
 export { HuggingFaceInferenceAPI } from "./huggingface.js";
 export {

packages/core/src/llm/openai.ts

@@ -110,6 +110,9 @@ export const GPT4_MODELS = {
   "gpt-4-1106-preview": { contextWindow: 128000 },
   "gpt-4-0125-preview": { contextWindow: 128000 },
   "gpt-4-vision-preview": { contextWindow: 128000 },
+  // fixme: wait for openai documentation
+  "gpt-4o": { contextWindow: 128000 },
+  "gpt-4o-2024-05-13": { contextWindow: 128000 },
 };
 
 // NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future

packages/core/src/readers/CSVReader.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import type { ParseConfig } from "papaparse";
 import Papa from "papaparse";
 import { Document } from "../Node.js";
@@ -40,11 +39,8 @@ export class PapaCSVReader implements FileReader {
    * @param {GenericFileSystem} [fs=DEFAULT_FS] - The file system to use for reading the file.
    * @returns {Promise<Document[]>}
    */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const fileContent = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const fileContent = await fs.readFile(file, "utf-8");
     const result = Papa.parse(fileContent, this.papaConfig);
     const textList = result.data.map((row: any) => {
       // Compatible with header row mode

packages/core/src/readers/DocxReader.ts

@@ -1,16 +1,12 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import mammoth from "mammoth";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";
 
 export class DocxReader implements FileReader {
   /** DocxParser */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readRawFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file);
     const { value } = await mammoth.extractRawText({ buffer: dataBuffer });
     return [new Document({ text: value, id_: file })];
   }

packages/core/src/readers/HTMLReader.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";
 
@@ -15,14 +14,10 @@ export class HTMLReader implements FileReader {
    * Public method for this reader.
    * Required by BaseReader interface.
    * @param file Path/name of the file to be loaded.
-   * @param fs fs wrapper interface for getting the file content.
    * @returns Promise<Document[]> A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
    */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file, "utf-8");
     const htmlOptions = this.getOptions();
     const content = await this.parseContent(dataBuffer, htmlOptions);
     return [new Document({ text: content, id_: file })];

packages/core/src/readers/ImageReader.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import type { Document } from "../Node.js";
 import { ImageDocument } from "../Node.js";
 import type { FileReader } from "./type.js";
@@ -15,11 +14,8 @@ export class ImageReader implements FileReader {
    * @param fs fs wrapper interface for getting the file content.
    * @returns Promise<Document[]> A Promise object, eventually yielding zero or one ImageDocument of the specified file.
    */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readRawFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file);
     const blob = new Blob([dataBuffer]);
     return [new ImageDocument({ image: blob, id_: file })];
   }

packages/core/src/readers/LlamaParseReader.ts

@@ -1,4 +1,4 @@
-import { defaultFS, getEnv, type GenericFileSystem } from "@llamaindex/env";
+import { fs, getEnv } from "@llamaindex/env";
 import { filetypemime } from "magic-bytes.js";
 import { Document } from "../Node.js";
 import type { FileReader, Language, ResultType } from "./type.js";
@@ -79,14 +79,11 @@ export class LlamaParseReader implements FileReader {
     this.apiKey = params.apiKey;
   }
 
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
+  async loadData(file: string): Promise<Document[]> {
     const metadata = { file_path: file };
 
     // Load data, set the mime type
-    const data = await fs.readRawFile(file);
+    const data = await fs.readFile(file);
     const mimeType = await this.getMimeType(data);
 
     const body = new FormData();

packages/core/src/readers/MarkdownReader.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";
 
@@ -88,11 +87,8 @@ export class MarkdownReader implements FileReader {
     return this.markdownToTups(modifiedContent);
   }
 
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const content = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const content = await fs.readFile(file, "utf-8");
     const tups = this.parseTups(content);
     const results: Document[] = [];
     let counter = 0;

packages/core/src/readers/PDFReader.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { BaseReader } from "./type.js";
 
@@ -7,11 +6,8 @@ import type { BaseReader } from "./type.js";
  * Read the text of a PDF
  */
 export class PDFReader implements BaseReader {
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const content = await fs.readRawFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const content = await fs.readFile(file);
     const pages = await readPDF(content);
     return pages.map((text, page) => {
       const id_ = `${file}_${page + 1}`;

packages/core/src/readers/SimpleDirectoryReader.edge.ts

@@ -1,5 +1,4 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { fs, path } from "@llamaindex/env";
 import { Document, type Metadata } from "../Node.js";
 import { walk } from "../storage/FileSystem.js";
 import { TextFileReader } from "./TextFileReader.js";
@@ -19,7 +18,6 @@ enum ReaderStatus {
 
 export type SimpleDirectoryReaderLoadDataParams = {
   directoryPath: string;
-  fs?: CompleteFileSystem;
   defaultReader?: BaseReader | null;
   fileExtToReader?: Record<string, BaseReader>;
 };
@@ -45,7 +43,6 @@ export class SimpleDirectoryReader implements BaseReader {
 
     const {
       directoryPath,
-      fs = defaultFS,
       defaultReader = new TextFileReader(),
       fileExtToReader,
     } = params;
@@ -58,7 +55,7 @@ export class SimpleDirectoryReader implements BaseReader {
     }
 
     const docs: Document[] = [];
-    for await (const filePath of walk(fs, directoryPath)) {
+    for await (const filePath of walk(directoryPath)) {
       try {
         const fileExt = path.extname(filePath).slice(1).toLowerCase();
 

packages/core/src/readers/TextFileReader.ts

@@ -1,5 +1,4 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { BaseReader } from "./type.js";
 
@@ -8,11 +7,8 @@ import type { BaseReader } from "./type.js";
  */
 
 export class TextFileReader implements BaseReader {
-  async loadData(
-    file: string,
-    fs: CompleteFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file, "utf-8");
     return [new Document({ text: dataBuffer, id_: file })];
   }
 }

packages/core/src/readers/type.ts

@@ -1,4 +1,3 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
 import type { Document } from "../Node.js";
 
 /**
@@ -12,7 +11,7 @@ export interface BaseReader {
  * A reader takes file paths and imports data into Document objects.
  */
 export interface FileReader extends BaseReader {
-  loadData(filePath: string, fs?: CompleteFileSystem): Promise<Document[]>;
+  loadData(filePath: string): Promise<Document[]>;
 }
 
 // For LlamaParseReader.ts

packages/core/src/storage/FileSystem.ts

@@ -1,5 +1,6 @@
-import type { GenericFileSystem, WalkableFileSystem } from "@llamaindex/env";
-// FS utility functions
+// FS utility helpers
+
+import { fs } from "@llamaindex/env";
 
 /**
  * Checks if a file exists.
@@ -8,10 +9,7 @@ import type { GenericFileSystem, WalkableFileSystem } from "@llamaindex/env";
  * @param path The path to the file to check.
  * @returns A promise that resolves to true if the file exists, false otherwise.
  */
-export async function exists(
-  fs: GenericFileSystem,
-  path: string,
-): Promise<boolean> {
+export async function exists(path: string): Promise<boolean> {
   try {
     await fs.access(path);
     return true;
@@ -22,19 +20,15 @@ export async function exists(
 
 /**
  * Recursively traverses a directory and yields all the paths to the files in it.
- * @param fs The filesystem to use.
  * @param dirPath The path to the directory to traverse.
  */
-export async function* walk(
-  fs: WalkableFileSystem,
-  dirPath: string,
-): AsyncIterable<string> {
+export async function* walk(dirPath: string): AsyncIterable<string> {
   const entries = await fs.readdir(dirPath);
   for (const entry of entries) {
     const fullPath = `${dirPath}/${entry}`;
     const stats = await fs.stat(fullPath);
     if (stats.isDirectory()) {
-      yield* walk(fs, fullPath);
+      yield* walk(fullPath);
     } else {
       yield fullPath;
     }

packages/core/src/storage/StorageContext.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { path } from "@llamaindex/env";
 import {
   DEFAULT_IMAGE_VECTOR_NAMESPACE,
   DEFAULT_NAMESPACE,
@@ -25,7 +24,6 @@ export type BuilderParams = {
   imageVectorStore: VectorStore;
   storeImages: boolean;
   persistDir: string;
-  fs: GenericFileSystem;
 };
 
 export async function storageContextFromDefaults({
@@ -35,7 +33,6 @@ export async function storageContextFromDefaults({
   imageVectorStore,
   storeImages,
   persistDir,
-  fs,
 }: Partial<BuilderParams>): Promise<StorageContext> {
   if (!persistDir) {
     docStore = docStore || new SimpleDocumentStore();
@@ -43,22 +40,16 @@ export async function storageContextFromDefaults({
     vectorStore = vectorStore || new SimpleVectorStore();
     imageVectorStore = storeImages ? new SimpleVectorStore() : imageVectorStore;
   } else {
-    fs = fs || defaultFS;
     docStore =
       docStore ||
-      (await SimpleDocumentStore.fromPersistDir(
-        persistDir,
-        DEFAULT_NAMESPACE,
-        fs,
-      ));
+      (await SimpleDocumentStore.fromPersistDir(persistDir, DEFAULT_NAMESPACE));
     indexStore =
-      indexStore || (await SimpleIndexStore.fromPersistDir(persistDir, fs));
+      indexStore || (await SimpleIndexStore.fromPersistDir(persistDir));
     vectorStore =
-      vectorStore || (await SimpleVectorStore.fromPersistDir(persistDir, fs));
+      vectorStore || (await SimpleVectorStore.fromPersistDir(persistDir));
     imageVectorStore = storeImages
       ? await SimpleVectorStore.fromPersistDir(
           path.join(persistDir, DEFAULT_IMAGE_VECTOR_NAMESPACE),
-          fs,
         )
       : imageVectorStore;
   }

packages/core/src/storage/docStore/SimpleDocumentStore.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { path } from "@llamaindex/env";
 import _ from "lodash";
 import {
   DEFAULT_DOC_STORE_PERSIST_FILENAME,
@@ -25,26 +24,19 @@ export class SimpleDocumentStore extends KVDocumentStore {
   static async fromPersistDir(
     persistDir: string = DEFAULT_PERSIST_DIR,
     namespace?: string,
-    fsModule?: GenericFileSystem,
   ): Promise<SimpleDocumentStore> {
     const persistPath = path.join(
       persistDir,
       DEFAULT_DOC_STORE_PERSIST_FILENAME,
     );
-    return await SimpleDocumentStore.fromPersistPath(
-      persistPath,
-      namespace,
-      fsModule,
-    );
+    return await SimpleDocumentStore.fromPersistPath(persistPath, namespace);
   }
 
   static async fromPersistPath(
     persistPath: string,
     namespace?: string,
-    fs?: GenericFileSystem,
   ): Promise<SimpleDocumentStore> {
-    fs = fs || defaultFS;
-    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
+    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath);
     return new SimpleDocumentStore(simpleKVStore, namespace);
   }
 
@@ -53,14 +45,12 @@ export class SimpleDocumentStore extends KVDocumentStore {
       DEFAULT_PERSIST_DIR,
       DEFAULT_DOC_STORE_PERSIST_FILENAME,
     ),
-    fs?: GenericFileSystem,
   ): Promise<void> {
-    fs = fs || defaultFS;
     if (
       _.isObject(this.kvStore) &&
       this.kvStore instanceof BaseInMemoryKVStore
     ) {
-      await this.kvStore.persist(persistPath, fs);
+      await this.kvStore.persist(persistPath);
     }
   }
 

packages/core/src/storage/docStore/types.ts

@@ -1,4 +1,3 @@
-import type { GenericFileSystem } from "@llamaindex/env";
 import { BaseNode } from "../../Node.js";
 import {
   DEFAULT_DOC_STORE_PERSIST_FILENAME,
@@ -14,10 +13,7 @@ export interface RefDocInfo {
 
 export abstract class BaseDocumentStore {
   // Save/load
-  persist(
-    persistPath: string = defaultPersistPath,
-    fs?: GenericFileSystem,
-  ): void {
+  persist(persistPath: string = defaultPersistPath): void {
     // Persist the docstore to a file.
   }
 

packages/core/src/storage/indexStore/SimpleIndexStore.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { path } from "@llamaindex/env";
 import {
   DEFAULT_INDEX_STORE_PERSIST_FILENAME,
   DEFAULT_PERSIST_DIR,
@@ -20,28 +19,21 @@ export class SimpleIndexStore extends KVIndexStore {
 
   static async fromPersistDir(
     persistDir: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = defaultFS,
   ): Promise<SimpleIndexStore> {
     const persistPath = path.join(
       persistDir,
       DEFAULT_INDEX_STORE_PERSIST_FILENAME,
     );
-    return this.fromPersistPath(persistPath, fs);
+    return this.fromPersistPath(persistPath);
   }
 
-  static async fromPersistPath(
-    persistPath: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<SimpleIndexStore> {
-    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
+  static async fromPersistPath(persistPath: string): Promise<SimpleIndexStore> {
+    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath);
     return new SimpleIndexStore(simpleKVStore);
   }
 
-  async persist(
-    persistPath: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<void> {
-    await this.kvStore.persist(persistPath, fs);
+  async persist(persistPath: string = DEFAULT_PERSIST_DIR): Promise<void> {
+    this.kvStore.persist(persistPath);
   }
 
   static fromDict(saveDict: DataType): SimpleIndexStore {

packages/core/src/storage/indexStore/types.ts

@@ -1,4 +1,3 @@
-import type { GenericFileSystem } from "@llamaindex/env";
 import type { IndexStruct } from "../../indices/IndexStruct.js";
 import {
   DEFAULT_INDEX_STORE_PERSIST_FILENAME,
@@ -16,10 +15,7 @@ export abstract class BaseIndexStore {
 
   abstract getIndexStruct(structId?: string): Promise<IndexStruct | undefined>;
 
-  async persist(
-    persistPath: string = defaultPersistPath,
-    fs?: GenericFileSystem,
-  ): Promise<void> {
+  async persist(persistPath: string = defaultPersistPath): Promise<void> {
     // Persist the index store to disk.
   }
 }

packages/core/src/storage/kvStore/SimpleKVStore.ts

@@ -1,6 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
-import _ from "lodash";
+import { fs, path } from "@llamaindex/env";
 import { exists } from "../FileSystem.js";
 import { DEFAULT_COLLECTION } from "../constants.js";
 import { BaseKVStore } from "./types.js";
@@ -9,7 +7,6 @@ export type DataType = Record<string, Record<string, any>>;
 
 export class SimpleKVStore extends BaseKVStore {
   private persistPath: string | undefined;
-  private fs: GenericFileSystem | undefined;
 
   constructor(private data: DataType = {}) {
     super();
@@ -23,10 +20,10 @@ export class SimpleKVStore extends BaseKVStore {
     if (!(collection in this.data)) {
       this.data[collection] = {};
     }
-    this.data[collection][key] = _.clone(val); // Creating a shallow copy of the object
+    this.data[collection][key] = structuredClone(val); // Creating a shallow copy of the object
 
     if (this.persistPath) {
-      await this.persist(this.persistPath, this.fs);
+      await this.persist(this.persistPath);
     }
   }
 
@@ -35,17 +32,17 @@ export class SimpleKVStore extends BaseKVStore {
     collection: string = DEFAULT_COLLECTION,
   ): Promise<any> {
     const collectionData = this.data[collection];
-    if (_.isNil(collectionData)) {
+    if (collectionData == null) {
       return null;
     }
     if (!(key in collectionData)) {
       return null;
     }
-    return _.clone(collectionData[key]); // Creating a shallow copy of the object
+    return structuredClone(collectionData[key]); // Creating a shallow copy of the object
   }
 
   async getAll(collection: string = DEFAULT_COLLECTION): Promise<DataType> {
-    return _.clone(this.data[collection]); // Creating a shallow copy of the object
+    return structuredClone(this.data[collection]); // Creating a shallow copy of the object
   }
 
   async delete(
@@ -55,31 +52,25 @@ export class SimpleKVStore extends BaseKVStore {
     if (key in this.data[collection]) {
       delete this.data[collection][key];
       if (this.persistPath) {
-        await this.persist(this.persistPath, this.fs);
+        await this.persist(this.persistPath);
       }
       return true;
     }
     return false;
   }
 
-  async persist(
-    persistPath: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<void> {
+  async persist(persistPath: string): Promise<void> {
     // TODO: decide on a way to polyfill path
     const dirPath = path.dirname(persistPath);
-    if (!(await exists(fs, dirPath))) {
+    if (!(await exists(dirPath))) {
       await fs.mkdir(dirPath);
     }
     await fs.writeFile(persistPath, JSON.stringify(this.data));
   }
 
-  static async fromPersistPath(
-    persistPath: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<SimpleKVStore> {
+  static async fromPersistPath(persistPath: string): Promise<SimpleKVStore> {
     const dirPath = path.dirname(persistPath);
-    if (!(await exists(fs, dirPath))) {
+    if (!(await exists(dirPath))) {
       await fs.mkdir(dirPath);
     }
 
@@ -95,7 +86,6 @@ export class SimpleKVStore extends BaseKVStore {
 
     const store = new SimpleKVStore(data);
     store.persistPath = persistPath;
-    store.fs = fs;
     return store;
   }
 

packages/core/src/storage/kvStore/types.ts

@@ -1,4 +1,3 @@
-import type { GenericFileSystem } from "@llamaindex/env";
 const defaultCollection = "data";
 
 type StoredValue = Record<string, any> | null;
@@ -15,7 +14,7 @@ export abstract class BaseKVStore {
 }
 
 export abstract class BaseInMemoryKVStore extends BaseKVStore {
-  abstract persist(persistPath: string, fs?: GenericFileSystem): void;
+  abstract persist(persistPath: string): void;
   static fromPersistPath(persistPath: string): BaseInMemoryKVStore {
     throw new Error("Method not implemented.");
   }

packages/core/src/storage/vectorStore/PGVectorStore.ts

@@ -6,7 +6,6 @@ import type {
   VectorStoreQueryResult,
 } from "./types.js";
 
-import type { GenericFileSystem } from "@llamaindex/env";
 import type { BaseNode, Metadata } from "../../Node.js";
 import { Document, MetadataMode } from "../../Node.js";
 
@@ -307,10 +306,7 @@ export class PGVectorStore implements VectorStore {
    * @param fs
    * @returns Resolved Promise.
    */
-  persist(
-    persistPath: string,
-    fs?: GenericFileSystem | undefined,
-  ): Promise<void> {
+  persist(persistPath: string): Promise<void> {
     return Promise.resolve();
   }
 }

packages/core/src/storage/vectorStore/PineconeVectorStore.ts

@@ -6,7 +6,7 @@ import type {
   VectorStoreQueryResult,
 } from "./types.js";
 
-import { getEnv, type GenericFileSystem } from "@llamaindex/env";
+import { getEnv } from "@llamaindex/env";
 import type {
   FetchResponse,
   Index,
@@ -188,10 +188,7 @@ export class PineconeVectorStore implements VectorStore {
    * @param fs
    * @returns Resolved Promise.
    */
-  persist(
-    persistPath: string,
-    fs?: GenericFileSystem | undefined,
-  ): Promise<void> {
+  persist(persistPath: string): Promise<void> {
     return Promise.resolve();
   }
 

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -1,6 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
-import _ from "lodash";
+import { fs, path } from "@llamaindex/env";
 import type { BaseNode } from "../../Node.js";
 import {
   getTopKEmbeddings,
@@ -32,20 +30,17 @@ class SimpleVectorStoreData {
 export class SimpleVectorStore implements VectorStore {
   storesText: boolean = false;
   private data: SimpleVectorStoreData = new SimpleVectorStoreData();
-  private fs: GenericFileSystem = defaultFS;
   private persistPath: string | undefined;
 
-  constructor(data?: SimpleVectorStoreData, fs?: GenericFileSystem) {
+  constructor(data?: SimpleVectorStoreData) {
     this.data = data || new SimpleVectorStoreData();
-    this.fs = fs || defaultFS;
   }
 
   static async fromPersistDir(
     persistDir: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = defaultFS,
   ): Promise<SimpleVectorStore> {
-    const persistPath = `${persistDir}/vector_store.json`;
-    return await SimpleVectorStore.fromPersistPath(persistPath, fs);
+    const persistPath = path.join(persistDir, "vector_store.json");
+    return await SimpleVectorStore.fromPersistPath(persistPath);
   }
 
   get client(): any {
@@ -68,7 +63,7 @@ export class SimpleVectorStore implements VectorStore {
     }
 
     if (this.persistPath) {
-      await this.persist(this.persistPath, this.fs);
+      await this.persist(this.persistPath);
     }
 
     return embeddingResults.map((result) => result.id_);
@@ -83,13 +78,13 @@ export class SimpleVectorStore implements VectorStore {
       delete this.data.textIdToRefDocId[textId];
     }
     if (this.persistPath) {
-      await this.persist(this.persistPath, this.fs);
+      await this.persist(this.persistPath);
     }
     return Promise.resolve();
   }
 
   async query(query: VectorStoreQuery): Promise<VectorStoreQueryResult> {
-    if (!_.isNil(query.filters)) {
+    if (!(query.filters == null)) {
       throw new Error(
         "Metadata filters not implemented for SimpleVectorStore yet.",
       );
@@ -147,12 +142,10 @@ export class SimpleVectorStore implements VectorStore {
   }
 
   async persist(
-    persistPath: string = `${DEFAULT_PERSIST_DIR}/vector_store.json`,
-    fs?: GenericFileSystem,
+    persistPath: string = path.join(DEFAULT_PERSIST_DIR, "vector_store.json"),
   ): Promise<void> {
-    fs = fs || this.fs;
     const dirPath = path.dirname(persistPath);
-    if (!(await exists(fs, dirPath))) {
+    if (!(await exists(dirPath))) {
       await fs.mkdir(dirPath);
     }
 
@@ -161,10 +154,9 @@ export class SimpleVectorStore implements VectorStore {
 
   static async fromPersistPath(
     persistPath: string,
-    fs: GenericFileSystem = defaultFS,
   ): Promise<SimpleVectorStore> {
     const dirPath = path.dirname(persistPath);
-    if (!(await exists(fs, dirPath))) {
+    if (!(await exists(dirPath))) {
       await fs.mkdir(dirPath, { recursive: true });
     }
 
@@ -183,7 +175,6 @@ export class SimpleVectorStore implements VectorStore {
     data.textIdToRefDocId = dataDict.textIdToRefDocId ?? {};
     const store = new SimpleVectorStore(data);
     store.persistPath = persistPath;
-    store.fs = fs;
     return store;
   }
 

packages/core/tests/vitest.setup.ts

@@ -11,8 +11,8 @@ globalThis.fetch = function fetch(...args: Parameters<typeof originalFetch>) {
       url = url.toString();
     }
   }
-  const parsedUrl = new URL(url);
-  if (parsedUrl.hostname.includes("openai.com")) {
+  const { host } = new URL(url);
+  if (host.endsWith("openai.com")) {
     // todo: mock api using https://mswjs.io
     throw new Error(
       "Make sure to return a mock response for OpenAI API requests in your test.",

packages/env/jsr.json

@@ -2,8 +2,7 @@
   "name": "@llamaindex/env",
   "version": "0.1.2",
   "exports": {
-    ".": "./src/index.ts",
-    "./type": "./src/type.ts"
+    ".": "./src/index.ts"
   },
   "publish": {
     "include": ["LICENSE", "README.md", "src/**/*.ts", "jsr.json"]

packages/env/package.json

@@ -33,8 +33,8 @@
         "default": "./dist/index.workerd.js"
       },
       "edge-light": {
-        "types": "./dist/type/index.polyfill.d.ts",
-        "default": "./dist/index.polyfill.js"
+        "types": "./dist/type/index.edge-light.d.ts",
+        "default": "./dist/index.edge-light.js"
       },
       "import": {
         "types": "./dist/type/index.d.ts",
@@ -44,16 +44,6 @@
         "types": "./dist/type/index.d.ts",
         "default": "./dist/cjs/index.js"
       }
-    },
-    "./*": {
-      "import": {
-        "types": "./dist/type/*.d.ts",
-        "default": "./dist/*.js"
-      },
-      "require": {
-        "types": "./dist/type/*.d.ts",
-        "default": "./dist/cjs/*.js"
-      }
     }
   },
   "files": [
@@ -71,7 +61,8 @@
     "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file ../../.cjs.swcrc",
     "build:type": "tsc -p tsconfig.json",
     "postbuild": "node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"",
-    "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\""
+    "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\"",
+    "test": "vitest"
   },
   "devDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
@@ -79,7 +70,8 @@
     "@swc/core": "^1.5.5",
     "concurrently": "^8.2.2",
     "pathe": "^1.1.2",
-    "readable-stream": "^4.5.2"
+    "readable-stream": "^4.5.2",
+    "vitest": "^1.6.0"
   },
   "dependencies": {
     "@types/lodash": "^4.17.1",

packages/env/src/fs/memfs/README.md

@@ -0,0 +1 @@
+Copy from https://github.com/bundled-es-modules/memfs/blob/main/index-esm.js

packages/env/src/fs/memory.ts

@@ -0,0 +1,9 @@
+/**
+ * Memory file system, used by edge runtime, worker runtime which doesn't have access to the file system.
+ *
+ * @module
+ */
+// @ts-expect-error
+import memFS from "./memfs/index.js";
+
+export const fs = memFS.promises;

packages/env/src/fs/node.ts

@@ -0,0 +1,10 @@
+/**
+ * Node.js built-in file system module.
+ *
+ * Bun, Deno support node:fs/promises so we don't need to re-implement it.
+ *
+ * @module
+ */
+import fs from "node:fs/promises";
+
+export { fs };

packages/env/src/index.edge-light.ts

@@ -0,0 +1,6 @@
+/**
+ * Edge light environment polyfill.
+ *
+ * @module
+ */
+export * from "./polyfill.js";

packages/env/src/index.ts

@@ -13,7 +13,6 @@
  */
 import { ok } from "node:assert";
 import { createHash, randomUUID } from "node:crypto";
-import fs from "node:fs/promises";
 import { EOL } from "node:os";
 import path from "node:path";
 import { pipeline } from "node:stream/promises";
@@ -22,8 +21,8 @@ import {
   TransformStream,
   WritableStream,
 } from "node:stream/web";
-import type { SHA256 } from "./index.polyfill.js";
-import type { CompleteFileSystem } from "./type.js";
+import { fs } from "./fs/node.js";
+import type { SHA256 } from "./polyfill.js";
 
 export function createSHA256(): SHA256 {
   const hash = createHash("sha256");
@@ -37,29 +36,13 @@ export function createSHA256(): SHA256 {
   };
 }
 
-export const defaultFS: CompleteFileSystem = {
-  writeFile: function (path: string, content: string) {
-    return fs.writeFile(path, content, "utf-8");
-  },
-  readRawFile(path: string): Promise<Buffer> {
-    return fs.readFile(path);
-  },
-  readFile: function (path: string) {
-    return fs.readFile(path, "utf-8");
-  },
-  access: fs.access,
-  mkdir: fs.mkdir,
-  readdir: fs.readdir,
-  stat: fs.stat,
-};
-
-export type * from "./type.js";
 export { AsyncLocalStorage, CustomEvent, getEnv, setEnvs } from "./utils.js";
 export {
   EOL,
   ReadableStream,
   TransformStream,
   WritableStream,
+  fs,
   ok,
   path,
   pipeline,

packages/env/src/index.workerd.ts

@@ -7,7 +7,7 @@
  */
 import { INTERNAL_ENV } from "./utils.js";
 
-export * from "./index.polyfill.js";
+export * from "./polyfill.js";
 
 export function getEnv(name: string): string | undefined {
   return INTERNAL_ENV[name];

packages/env/src/polyfill.ts

@@ -1,19 +1,21 @@
 /**
- * Polyfill implementation some node.js APIs.
+ * Polyfill implementation for `@llamaindex/env`.
  *
  * The code should be compatible with any JS runtime.
  *
+ * Sometimes you should overwrite the polyfill with a native implementation.
+ *
  * @module
  */
 import { Sha256 } from "@aws-crypto/sha256-js";
 import pathe from "pathe";
-import { InMemoryFileSystem, type CompleteFileSystem } from "./type.js";
 // @ts-expect-error
 import { promises } from "readable-stream";
+import { fs } from "./fs/memory.js";
 
 const { pipeline } = promises;
 
-export { pathe as path, pipeline };
+export { fs, pathe as path, pipeline };
 
 export interface SHA256 {
   update(data: string | Uint8Array): void;
@@ -23,8 +25,6 @@ export interface SHA256 {
 
 export const EOL = "\n";
 
-export const defaultFS: CompleteFileSystem = new InMemoryFileSystem();
-
 export function ok(value: unknown, message?: string): asserts value {
   if (!value) {
     const error = Error(message);
@@ -49,7 +49,6 @@ export function createSHA256(): SHA256 {
 export function randomUUID(): string {
   return crypto.randomUUID();
 }
-export * from "./type.js";
 
 // @ts-expect-error
 const ReadableStream = globalThis.ReadableStream;

packages/env/src/type.ts

@@ -1,73 +0,0 @@
-/**
- * A filesystem interface that is meant to be compatible with
- * the 'fs' module from Node.js.
- * Allows for the use of similar interface implementation on different JS runtimes.
- */
-export type GenericFileSystem = {
-  writeFile(path: string, content: string): Promise<void>;
-  /**
-   * Reads a file and returns its content as a raw buffer.
-   */
-  readRawFile(path: string): Promise<Buffer>;
-  /**
-   * Reads a file and returns its content as an utf-8 string.
-   */
-  readFile(path: string): Promise<string>;
-  access(path: string): Promise<void>;
-  mkdir(
-    path: string,
-    options: {
-      recursive: boolean;
-    },
-  ): Promise<string | undefined>;
-  mkdir(path: string): Promise<void>;
-};
-export type WalkableFileSystem = {
-  readdir(path: string): Promise<string[]>;
-  stat(path: string): Promise<any>;
-};
-export type CompleteFileSystem = GenericFileSystem & WalkableFileSystem;
-
-/**
- * A filesystem implementation that stores files in memory.
- */
-export class InMemoryFileSystem implements CompleteFileSystem {
-  private files: Record<string, any> = {};
-
-  async writeFile(
-    path: string,
-    content: string,
-    options?: unknown,
-  ): Promise<void> {
-    this.files[path] = structuredClone(content);
-  }
-
-  async readFile(path: string): Promise<string> {
-    if (!(path in this.files)) {
-      throw new Error(`File ${path} does not exist`);
-    }
-    return structuredClone(this.files[path]);
-  }
-
-  async access(path: string): Promise<void> {
-    if (!(path in this.files)) {
-      throw new Error(`File ${path} does not exist`);
-    }
-  }
-
-  async mkdir(path: string): Promise<undefined> {
-    this.files[path] = this.files[path] ?? null;
-  }
-
-  async readdir(path: string): Promise<string[]> {
-    throw new Error("Not implemented");
-  }
-
-  async stat(path: string): Promise<any> {
-    throw new Error("Not implemented");
-  }
-
-  async readRawFile(path: string): Promise<Buffer> {
-    throw new Error("Not implemented");
-  }
-}

packages/env/tests/memfs.test.ts

@@ -0,0 +1,10 @@
+import { describe, expect, test } from "vitest";
+import { fs } from "../src/fs/memory.js";
+
+describe("memfs", () => {
+  test("should be able to read and write files", async () => {
+    await fs.writeFile("/foo", "bar");
+    const content = await fs.readFile("/foo", "utf8");
+    expect(content).toBe("bar");
+  });
+});

packages/env/tests/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./lib",
+    "module": "node16",
+    "moduleResolution": "node16",
+    "target": "ESNext"
+  },
+  "include": ["./**/*.ts"],
+  "references": [
+    {
+      "path": "../tsconfig.json"
+    }
+  ]
+}

packages/experimental/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/experimental
 
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [4aba02e]
+  - llamaindex@0.3.10
+
 ## 0.0.26
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.26",
+  "version": "0.0.27",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

pnpm-lock.yaml

@@ -138,8 +138,8 @@ importers:
         specifier: ^2.4.2
         version: 2.4.2
       chromadb:
-        specifier: ^1.8.1
-        version: 1.8.1(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.43.0(encoding@0.1.13))
+        specifier: ^1.7.3
+        version: 1.7.3(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.46.0(encoding@0.1.13))
       commander:
         specifier: ^12.0.0
         version: 12.0.0
@@ -386,8 +386,8 @@ importers:
         specifier: ^4.4.2
         version: 4.4.2
       chromadb:
-        specifier: ~1.8.1
-        version: 1.8.1(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.43.0(encoding@0.1.13))
+        specifier: ~1.7.3
+        version: 1.7.3(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.46.0(encoding@0.1.13))
       cohere-ai:
         specifier: ^7.9.5
         version: 7.9.5(encoding@0.1.13)
@@ -413,8 +413,8 @@ importers:
         specifier: ^1.0.0
         version: 1.0.0(encoding@0.1.13)
       openai:
-        specifier: ^4.43.0
-        version: 4.43.0(encoding@0.1.13)
+        specifier: ^4.46.0
+        version: 4.46.0(encoding@0.1.13)
       papaparse:
         specifier: ^5.4.1
         version: 5.4.1
@@ -655,6 +655,9 @@ importers:
       readable-stream:
         specifier: ^4.5.2
         version: 4.5.2
+      vitest:
+        specifier: ^1.6.0
+        version: 1.6.0(@types/node@20.12.11)(terser@5.31.0)
 
   packages/experimental:
     dependencies:
@@ -4075,8 +4078,8 @@ packages:
   chownr@1.1.4:
     resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
 
-  chromadb@1.8.1:
-    resolution: {integrity: sha512-NpbYydbg4Uqt/9BXKgkZXn0fqpsh2Z1yjhkhKH+rcHMoq0pwI18BFSU2QU7Fk/ZypwGefW2AvqyE/3ZJIgy4QA==}
+  chromadb@1.7.3:
+    resolution: {integrity: sha512-3GgvQjpqgk5C89x5EuTDaXKbfrdqYDJ5UVyLQ3ZmwxnpetNc+HhRDGjkvXa5KSvpQ3lmKoyDoqnN4tZepfFkbw==}
     engines: {node: '>=14.17.0'}
     peerDependencies:
       '@google/generative-ai': ^0.1.1
@@ -7191,6 +7194,10 @@ packages:
     resolution: {integrity: sha512-4SMUB/XiqnO5IrEcdzEGGTcHoeXq7D/k82v36zoqSitrMUjenZXGH5JysIH7aF7Wr+gjvq0dT2mV6wLVKA7Seg==}
     hasBin: true
 
+  openai@4.46.0:
+    resolution: {integrity: sha512-l0Betzsx3WIjdagqQiH14hWmwYouUzUCcB1ENvKyfG5XOqh6YC2XT7OukzEBTnweutMC91pW2ToddWn8uyD4SA==}
+    hasBin: true
+
   opener@1.5.2:
     resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==}
     hasBin: true
@@ -14162,14 +14169,14 @@ snapshots:
 
   chownr@1.1.4: {}
 
-  chromadb@1.8.1(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.43.0(encoding@0.1.13)):
+  chromadb@1.7.3(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.46.0(encoding@0.1.13)):
     dependencies:
       cliui: 8.0.1
       isomorphic-fetch: 3.0.0(encoding@0.1.13)
     optionalDependencies:
       '@google/generative-ai': 0.11.0
       cohere-ai: 7.9.5(encoding@0.1.13)
-      openai: 4.43.0(encoding@0.1.13)
+      openai: 4.46.0(encoding@0.1.13)
     transitivePeerDependencies:
       - encoding
 
@@ -18002,6 +18009,19 @@ snapshots:
     transitivePeerDependencies:
       - encoding
 
+  openai@4.46.0(encoding@0.1.13):
+    dependencies:
+      '@types/node': 18.19.33
+      '@types/node-fetch': 2.6.11
+      abort-controller: 3.0.0
+      agentkeepalive: 4.5.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.7.0(encoding@0.1.13)
+      web-streams-polyfill: 3.3.3
+    transitivePeerDependencies:
+      - encoding
+
   opener@1.5.2: {}
 
   option@0.2.4: {}

tsconfig.json

@@ -50,6 +50,9 @@
     {
       "path": "./packages/env/tsconfig.json"
     },
+    {
+      "path": "./packages/env/tests/tsconfig.json"
+    },
     {
       "path": "./examples"
     },