Release 0.3.11 (#835 )

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
fix: remove non-standard API pipeline (#850 )
2026-07-09 19:35:51 -04:00 · 2024-05-16 16:35:57 -07:00 · 2024-05-16 16:31:48 -07:00 · 2024-05-16 16:29:16 -07:00 · 2024-05-16 12:48:50 -07:00 · 2024-05-15 18:03:22 -07:00
80 changed files with 13334 additions and 563 deletions
@@ -76,8 +76,7 @@ jobs:
      - name: Run Type Check
        run: pnpm run type-check
      - name: Run Circular Dependency Check
-        run: pnpm run circular-check
-        working-directory: ./packages/core
+        run: pnpm dlx turbo run circular-check
      - uses: actions/upload-artifact@v3
        if: failure()
        with:
@@ -1,3 +1,3 @@
-pnpm format:write
+pnpm format
 pnpm lint
 npx lint-staged
@@ -1,5 +1,15 @@
 # docs

+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [e072c45]
+- Updated dependencies [9e133ac]
+- Updated dependencies [447105a]
+- Updated dependencies [320be3f]
+  - llamaindex@0.3.11
+
 ## 0.0.18

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "docs",
-  "version": "0.0.18",
+  "version": "0.0.19",
  "private": true,
  "scripts": {
    "docusaurus": "docusaurus",
@@ -8,7 +8,7 @@
    "@notionhq/client": "^2.2.15",
    "@pinecone-database/pinecone": "^2.2.0",
    "@zilliz/milvus2-sdk-node": "^2.4.2",
-    "chromadb": "^1.8.1",
+    "chromadb": "^1.7.3",
    "commander": "^12.0.0",
    "dotenv": "^16.4.5",
    "js-tiktoken": "^1.0.11",
@@ -2,8 +2,8 @@
  "name": "@llamaindex/monorepo",
  "private": true,
  "scripts": {
-    "build": "turbo run build --filter=\"!docs\" --filter=\"!*-test\"",
-    "build:release": "turbo run build lint test --filter=\"!docs\" --filter=\"!*-test\"",
+    "build": "turbo run build --filter=\"!docs\" --filter=\"!*-test\" --filter=\"!*-example\"",
+    "build:release": "turbo run build lint test --filter=\"!docs\" --filter=\"!*-test\" --filter=\"!*-example\"",
    "dev": "turbo run dev",
    "format": "prettier --ignore-unknown --cache --check .",
    "format:write": "prettier --ignore-unknown --write .",
@@ -28,6 +28,7 @@
    "eslint-plugin-react": "7.34.1",
    "husky": "^9.0.11",
    "lint-staged": "^15.2.2",
+    "madge": "^7.0.0",
    "prettier": "^3.2.5",
    "prettier-plugin-organize-imports": "^3.2.4",
    "turbo": "^1.13.3",
@@ -4,6 +4,17 @@

 ### Patch Changes

+- Updated dependencies [e072c45]
+- Updated dependencies [9e133ac]
+- Updated dependencies [447105a]
+- Updated dependencies [320be3f]
+  - llamaindex@0.3.11
+  - @llamaindex/autotool@0.0.1
+
+## null
+
+### Patch Changes
+
 - Updated dependencies [4aba02e]
  - llamaindex@0.3.10
  - @llamaindex/autotool@0.0.1
@@ -1,5 +1,16 @@
 # @llamaindex/autotool-02-next-example

+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies [e072c45]
+- Updated dependencies [9e133ac]
+- Updated dependencies [447105a]
+- Updated dependencies [320be3f]
+  - llamaindex@0.3.11
+  - @llamaindex/autotool@0.0.1
+
 ## 0.1.2

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/autotool-02-next-example",
  "private": true,
-  "version": "0.1.2",
+  "version": "0.1.3",
  "scripts": {
    "dev": "next dev",
    "build": "next build",
@@ -51,7 +51,7 @@
    "unplugin": "^1.10.1"
  },
  "peerDependencies": {
-    "llamaindex": "^0.3.10",
+    "llamaindex": "^0.3.11",
    "openai": "^4",
    "typescript": "^4"
  },
@@ -0,0 +1,7 @@
+{
+	"detectiveOptions": {
+		"ts": {
+			"skipTypeImports": true
+		}
+	}
+}
@@ -1,5 +1,22 @@
 # llamaindex

+## 0.3.11
+
+### Patch Changes
+
+- e072c45: fix: remove non-standard API `pipeline`
+- 9e133ac: refactor: remove `defaultFS` from parameters
+
+  We don't accept passing fs in the parameter since it's unnecessary for a determined JS environment.
+
+  This was a polyfill way for the non-Node.js environment, but now we use another way to polyfill APIs.
+
+- 447105a: Improve Gemini message and context preparation
+- 320be3f: Force ChromaDB version to 1.7.3 (to prevent NextJS issues)
+- Updated dependencies [e072c45]
+- Updated dependencies [9e133ac]
+  - @llamaindex/env@0.1.3
+
 ## 0.3.10

 ### Patch Changes
@@ -1,5 +1,15 @@
 # @llamaindex/cloudflare-worker-agent-test

+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [e072c45]
+- Updated dependencies [9e133ac]
+- Updated dependencies [447105a]
+- Updated dependencies [320be3f]
+  - llamaindex@0.3.11
+
 ## 0.0.11

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.11",
+  "version": "0.0.12",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,15 @@
 # @llamaindex/next-agent-test

+## 0.1.12
+
+### Patch Changes
+
+- Updated dependencies [e072c45]
+- Updated dependencies [9e133ac]
+- Updated dependencies [447105a]
+- Updated dependencies [320be3f]
+  - llamaindex@0.3.11
+
 ## 0.1.11

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-agent-test",
-  "version": "0.1.11",
+  "version": "0.1.12",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,15 @@
 # test-edge-runtime

+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [e072c45]
+- Updated dependencies [9e133ac]
+- Updated dependencies [447105a]
+- Updated dependencies [320be3f]
+  - llamaindex@0.3.11
+
 ## 0.1.10

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.10",
+  "version": "0.1.11",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,15 @@
 # @llamaindex/waku-query-engine-test

+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [e072c45]
+- Updated dependencies [9e133ac]
+- Updated dependencies [447105a]
+- Updated dependencies [320be3f]
+  - llamaindex@0.3.11
+
 ## 0.0.11

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.11",
+  "version": "0.0.12",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,9 +1,9 @@
 {
  "name": "@llamaindex/core",
-  "version": "0.3.10",
+  "version": "0.3.11",
  "exports": "./src/index.ts",
  "imports": {
-    "@llamaindex/env": "jsr:@llamaindex/env@0.1.2"
+    "@llamaindex/env": "jsr:@llamaindex/env@0.1.3"
  },
  "publish": {
    "include": ["LICENSE", "README.md", "src/**/*", "jsr.json"]
@@ -1,6 +1,6 @@
 {
  "name": "llamaindex",
-  "version": "0.3.10",
+  "version": "0.3.11",
  "expectedMinorVersion": "3",
  "license": "MIT",
  "type": "module",
@@ -39,7 +39,7 @@
    "@zilliz/milvus2-sdk-node": "^2.4.2",
    "ajv": "^8.13.0",
    "assemblyai": "^4.4.2",
-    "chromadb": "~1.8.1",
+    "chromadb": "~1.7.3",
    "cohere-ai": "^7.9.5",
    "js-tiktoken": "^1.0.11",
    "lodash": "^4.17.21",
@@ -48,7 +48,7 @@
    "md-utils-ts": "^2.0.0",
    "mongodb": "^6.6.1",
    "notion-md-crawler": "^1.0.0",
-    "openai": "^4.43.0",
+    "openai": "^4.46.0",
    "papaparse": "^5.4.1",
    "pathe": "^1.1.2",
    "pdf2json": "3.0.5",
@@ -69,7 +69,6 @@
    "@swc/core": "^1.5.5",
    "concurrently": "^8.2.2",
    "glob": "^10.3.12",
-    "madge": "^7.0.0",
    "typescript": "^5.4.5"
  },
  "engines": {
@@ -1,5 +1,5 @@
 import { createSHA256, path, randomUUID } from "@llamaindex/env";
-import _ from "lodash";
+import { chunkSizeCheck, lazyInitHash } from "./internal/decorator/node.js";

 export enum NodeRelationship {
  SOURCE = "SOURCE",
@@ -37,6 +37,16 @@ export type RelatedNodeType<T extends Metadata = Metadata> =
  | RelatedNodeInfo<T>
  | RelatedNodeInfo<T>[];

+export type BaseNodeParams<T extends Metadata = Metadata> = {
+  id_?: string;
+  metadata?: T;
+  excludedEmbedMetadataKeys?: string[];
+  excludedLlmMetadataKeys?: string[];
+  relationships?: Partial<Record<NodeRelationship, RelatedNodeType<T>>>;
+  hash?: string;
+  embedding?: number[];
+};
+
 /**
 * Generic abstract class for retrievable nodes
 */
@@ -47,21 +57,37 @@ export abstract class BaseNode<T extends Metadata = Metadata> {
   *
   * Set to a UUID by default.
   */
-  id_: string = randomUUID();
+  id_: string;
  embedding?: number[];

  // Metadata fields
-  metadata: T = {} as T;
-  excludedEmbedMetadataKeys: string[] = [];
-  excludedLlmMetadataKeys: string[] = [];
-  relationships: Partial<Record<NodeRelationship, RelatedNodeType<T>>> = {};
-  hash: string = "";
+  metadata: T;
+  excludedEmbedMetadataKeys: string[];
+  excludedLlmMetadataKeys: string[];
+  relationships: Partial<Record<NodeRelationship, RelatedNodeType<T>>>;

-  constructor(init?: Partial<BaseNode<T>>) {
-    Object.assign(this, init);
+  @lazyInitHash
+  accessor hash: string = "";
+
+  protected constructor(init?: BaseNodeParams<T>) {
+    const {
+      id_,
+      metadata,
+      excludedEmbedMetadataKeys,
+      excludedLlmMetadataKeys,
+      relationships,
+      hash,
+      embedding,
+    } = init || {};
+    this.id_ = id_ ?? randomUUID();
+    this.metadata = metadata ?? ({} as T);
+    this.excludedEmbedMetadataKeys = excludedEmbedMetadataKeys ?? [];
+    this.excludedLlmMetadataKeys = excludedLlmMetadataKeys ?? [];
+    this.relationships = relationships ?? {};
+    this.embedding = embedding;
  }

-  abstract getType(): ObjectType;
+  abstract get type(): ObjectType;

  abstract getContent(metadataMode: MetadataMode): string;
  abstract getMetadataStr(metadataMode: MetadataMode): string;
@@ -146,7 +172,12 @@ export abstract class BaseNode<T extends Metadata = Metadata> {
   * @see toMutableJSON - use to return a mutable JSON instead
   */
  toJSON(): Record<string, any> {
-    return { ...this, type: this.getType() };
+    return {
+      ...this,
+      type: this.type,
+      // hash is an accessor property, so it's not included in the rest operator
+      hash: this.hash,
+    };
  }

  clone(): BaseNode {
@@ -159,32 +190,43 @@ export abstract class BaseNode<T extends Metadata = Metadata> {
   * @return {Record<string, any>} - The JSON representation of the object.
   */
  toMutableJSON(): Record<string, any> {
-    return _.cloneDeep(this.toJSON());
+    return structuredClone(this.toJSON());
  }
 }

+export type TextNodeParams<T extends Metadata = Metadata> =
+  BaseNodeParams<T> & {
+    text?: string;
+    textTemplate?: string;
+    startCharIdx?: number;
+    endCharIdx?: number;
+    metadataSeparator?: string;
+  };
+
 /**
 * TextNode is the default node type for text. Most common node type in LlamaIndex.TS
 */
 export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
-  text: string = "";
-  textTemplate: string = "";
+  text: string;
+  textTemplate: string;

  startCharIdx?: number;
  endCharIdx?: number;
  // textTemplate: NOTE write your own formatter if needed
  // metadataTemplate: NOTE write your own formatter if needed
-  metadataSeparator: string = "\n";
+  metadataSeparator: string;

-  constructor(init?: Partial<TextNode<T>>) {
+  constructor(init: TextNodeParams<T> = {}) {
    super(init);
-    Object.assign(this, init);
-
-    if (new.target === TextNode) {
-      // Don't generate the hash repeatedly so only do it if this is
-      // constructing the derived class
-      this.hash = init?.hash ?? this.generateHash();
+    const { text, textTemplate, startCharIdx, endCharIdx, metadataSeparator } =
+      init;
+    this.text = text ?? "";
+    this.textTemplate = textTemplate ?? "";
+    if (startCharIdx) {
+      this.startCharIdx = startCharIdx;
    }
+    this.endCharIdx = endCharIdx;
+    this.metadataSeparator = metadataSeparator ?? "\n";
  }

  /**
@@ -194,7 +236,7 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
   */
  generateHash() {
    const hashFunction = createSHA256();
-    hashFunction.update(`type=${this.getType()}`);
+    hashFunction.update(`type=${this.type}`);
    hashFunction.update(
      `startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`,
    );
@@ -202,10 +244,11 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
    return hashFunction.digest();
  }

-  getType(): ObjectType {
+  get type() {
    return ObjectType.TEXT;
  }

+  @chunkSizeCheck
  getContent(metadataMode: MetadataMode = MetadataMode.NONE): string {
    const metadataStr = this.getMetadataStr(metadataMode).trim();
    return `${metadataStr}\n\n${this.text}`.trim();
@@ -246,19 +289,21 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
  }
 }

+export type IndexNodeParams<T extends Metadata = Metadata> =
+  TextNodeParams<T> & {
+    indexId: string;
+  };
+
 export class IndexNode<T extends Metadata = Metadata> extends TextNode<T> {
-  indexId: string = "";
+  indexId: string;

-  constructor(init?: Partial<IndexNode<T>>) {
+  constructor(init?: IndexNodeParams<T>) {
    super(init);
-    Object.assign(this, init);
-
-    if (new.target === IndexNode) {
-      this.hash = init?.hash ?? this.generateHash();
-    }
+    const { indexId } = init || {};
+    this.indexId = indexId ?? "";
  }

-  getType(): ObjectType {
+  get type() {
    return ObjectType.INDEX;
  }
 }
@@ -267,16 +312,11 @@ export class IndexNode<T extends Metadata = Metadata> extends TextNode<T> {
 * A document is just a special text node with a docId.
 */
 export class Document<T extends Metadata = Metadata> extends TextNode<T> {
-  constructor(init?: Partial<Document<T>>) {
+  constructor(init?: TextNodeParams<T>) {
    super(init);
-    Object.assign(this, init);
-
-    if (new.target === Document) {
-      this.hash = init?.hash ?? this.generateHash();
-    }
  }

-  getType() {
+  get type() {
    return ObjectType.DOCUMENT;
  }
 }
@@ -303,21 +343,21 @@ export function jsonToNode(json: any, type?: ObjectType) {

 export type ImageType = string | Blob | URL;

-export type ImageNodeConstructorProps<T extends Metadata> = Pick<
-  ImageNode<T>,
-  "image" | "id_"
-> &
-  Partial<ImageNode<T>>;
+export type ImageNodeParams<T extends Metadata = Metadata> =
+  TextNodeParams<T> & {
+    image: ImageType;
+  };

 export class ImageNode<T extends Metadata = Metadata> extends TextNode<T> {
  image: ImageType; // image as blob

-  constructor(init: ImageNodeConstructorProps<T>) {
+  constructor(init: ImageNodeParams<T>) {
    super(init);
-    this.image = init.image;
+    const { image } = init;
+    this.image = image;
  }

-  getType(): ObjectType {
+  get type() {
    return ObjectType.IMAGE;
  }

@@ -360,15 +400,11 @@ export class ImageNode<T extends Metadata = Metadata> extends TextNode<T> {
 }

 export class ImageDocument<T extends Metadata = Metadata> extends ImageNode<T> {
-  constructor(init: ImageNodeConstructorProps<T>) {
+  constructor(init: ImageNodeParams<T>) {
    super(init);
-
-    if (new.target === ImageDocument) {
-      this.hash = init?.hash ?? this.generateHash();
-    }
  }

-  getType() {
+  get type() {
    return ObjectType.IMAGE_DOCUMENT;
  }
 }
@@ -371,6 +371,7 @@ export function messagesToHistoryStr(messages: ChatMessage[]) {
 }

 export const defaultContextSystemPrompt = ({ context = "" }) => {
+  if (!context) return "";
  return `Context information is below.
 ---------------------
 ${context}
@@ -13,6 +13,11 @@ import {
  setCallbackManager,
  withCallbackManager,
 } from "./internal/settings/CallbackManager.js";
+import {
+  getChunkSize,
+  setChunkSize,
+  withChunkSize,
+} from "./internal/settings/chunk-size.js";
 import type { LLM } from "./llm/types.js";
 import type { NodeParser } from "./nodeParsers/types.js";

@@ -41,14 +46,12 @@ class GlobalSettings implements Config {
  #promptHelper: PromptHelper | null = null;
  #embedModel: BaseEmbedding | null = null;
  #nodeParser: NodeParser | null = null;
-  #chunkSize?: number;
  #chunkOverlap?: number;

  #llmAsyncLocalStorage = new AsyncLocalStorage<LLM>();
  #promptHelperAsyncLocalStorage = new AsyncLocalStorage<PromptHelper>();
  #embedModelAsyncLocalStorage = new AsyncLocalStorage<BaseEmbedding>();
  #nodeParserAsyncLocalStorage = new AsyncLocalStorage<NodeParser>();
-  #chunkSizeAsyncLocalStorage = new AsyncLocalStorage<number>();
  #chunkOverlapAsyncLocalStorage = new AsyncLocalStorage<number>();
  #promptAsyncLocalStorage = new AsyncLocalStorage<PromptConfig>();

@@ -115,8 +118,8 @@ class GlobalSettings implements Config {
  get nodeParser(): NodeParser {
    if (this.#nodeParser === null) {
      this.#nodeParser = new SimpleNodeParser({
-        chunkSize: this.#chunkSize,
-        chunkOverlap: this.#chunkOverlap,
+        chunkSize: this.chunkSize,
+        chunkOverlap: this.chunkOverlap,
      });
    }

@@ -147,15 +150,15 @@ class GlobalSettings implements Config {
  }

  set chunkSize(chunkSize: number | undefined) {
-    this.#chunkSize = chunkSize;
+    setChunkSize(chunkSize);
  }

  get chunkSize(): number | undefined {
-    return this.#chunkSizeAsyncLocalStorage.getStore() ?? this.#chunkSize;
+    return getChunkSize();
  }

  withChunkSize<Result>(chunkSize: number, fn: () => Result): Result {
-    return this.#chunkSizeAsyncLocalStorage.run(chunkSize, fn);
+    return withChunkSize(chunkSize, fn);
  }

  get chunkOverlap(): number | undefined {
@@ -1,9 +1,4 @@
-import {
-  ReadableStream,
-  TransformStream,
-  pipeline,
-  randomUUID,
-} from "@llamaindex/env";
+import { ReadableStream, TransformStream, randomUUID } from "@llamaindex/env";
 import { Settings } from "../Settings.js";
 import {
  type ChatEngine,
@@ -21,7 +16,6 @@ import type {
  LLM,
  MessageContent,
 } from "../llm/index.js";
-import { extractText } from "../llm/utils.js";
 import type { BaseToolWithCall, ToolOutput } from "../types.js";
 import type {
  AgentTaskContext,
@@ -169,7 +163,7 @@ export abstract class AgentWorker<
  abstract taskHandler: TaskHandler<AI, Store, AdditionalMessageOptions>;

  public createTask(
-    query: string,
+    query: MessageContent,
    context: AgentTaskContext<AI, Store, AdditionalMessageOptions>,
  ): ReadableStream<TaskStepOutput<AI, Store, AdditionalMessageOptions>> {
    context.store.messages.push({
@@ -305,7 +299,7 @@ export abstract class AgentRunner<
        });
      }
    }
-    return this.#runner.createTask(extractText(message), {
+    return this.#runner.createTask(message, {
      stream,
      toolCallCount: 0,
      llm: this.#llm,
@@ -340,46 +334,36 @@ export abstract class AgentRunner<
    | ReadableStream<AgentStreamChatResponse<AdditionalMessageOptions>>
  > {
    const task = this.createTask(params.message, !!params.stream);
-    const stepOutput = await pipeline(
-      task,
-      async (
-        iter: AsyncIterable<
-          TaskStepOutput<AI, Store, AdditionalMessageOptions>
-        >,
-      ) => {
-        for await (const stepOutput of iter) {
-          // update chat history for each round
-          this.#chatHistory = [...stepOutput.taskStep.context.store.messages];
-          if (stepOutput.isLast) {
-            return stepOutput;
-          }
-        }
-        throw new Error("Task did not complete");
-      },
-    );
-    const { output, taskStep } = stepOutput;
-    if (isAsyncIterable(output)) {
-      return output.pipeThrough<
-        AgentStreamChatResponse<AdditionalMessageOptions>
-      >(
-        new TransformStream({
-          transform(chunk, controller) {
-            controller.enqueue({
-              response: chunk,
-              get sources() {
-                return [...taskStep.context.store.toolOutputs];
+    for await (const stepOutput of task) {
+      // update chat history for each round
+      this.#chatHistory = [...stepOutput.taskStep.context.store.messages];
+      if (stepOutput.isLast) {
+        const { output, taskStep } = stepOutput;
+        if (isAsyncIterable(output)) {
+          return output.pipeThrough<
+            AgentStreamChatResponse<AdditionalMessageOptions>
+          >(
+            new TransformStream({
+              transform(chunk, controller) {
+                controller.enqueue({
+                  response: chunk,
+                  get sources() {
+                    return [...taskStep.context.store.toolOutputs];
+                  },
+                });
              },
-            });
-          },
-        }),
-      );
-    } else {
-      return {
-        response: output,
-        get sources() {
-          return [...taskStep.context.store.toolOutputs];
-        },
-      } satisfies AgentChatResponse<AdditionalMessageOptions>;
+            }),
+          );
+        } else {
+          return {
+            response: output,
+            get sources() {
+              return [...taskStep.context.store.toolOutputs];
+            },
+          } satisfies AgentChatResponse<AdditionalMessageOptions>;
+        }
+      }
    }
+    throw new Error("Task ended without a last step.");
  }
 }
@@ -1,4 +1,5 @@
-import { pipeline, ReadableStream } from "@llamaindex/env";
+import { ReadableStream } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
 import { stringifyJSONToMessageContent } from "../internal/utils.js";
 import type {
  ChatResponseChunk,
@@ -8,7 +9,6 @@ import type {
 } from "../llm/index.js";
 import { OpenAI } from "../llm/openai.js";
 import { ObjectRetriever } from "../objects/index.js";
-import { Settings } from "../Settings.js";
 import type { BaseToolWithCall } from "../types.js";
 import { AgentRunner, AgentWorker, type AgentParamsBase } from "./base.js";
 import type { TaskHandler } from "./types.js";
@@ -130,22 +130,14 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {

      if (hasToolCall) {
        // you need to consume the response to get the full toolCalls
-        const toolCalls = await pipeline(
-          pipStream,
-          async (
-            iter: AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>,
-          ) => {
-            const toolCalls = new Map<string, ToolCall | PartialToolCall>();
-            for await (const chunk of iter) {
-              if (chunk.options && "toolCall" in chunk.options) {
-                const toolCall = chunk.options.toolCall;
-                toolCalls.set(toolCall.id, toolCall);
-              }
-            }
-            return [...toolCalls.values()];
-          },
-        );
-        for (const toolCall of toolCalls) {
+        const toolCalls = new Map<string, ToolCall | PartialToolCall>();
+        for await (const chunk of pipStream) {
+          if (chunk.options && "toolCall" in chunk.options) {
+            const toolCall = chunk.options.toolCall;
+            toolCalls.set(toolCall.id, toolCall);
+          }
+        }
+        for (const toolCall of toolCalls.values()) {
          const targetTool = tools.find(
            (tool) => tool.metadata.name === toolCall.name,
          );
@@ -1,4 +1,4 @@
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import _ from "lodash";
 import { filetypemime } from "magic-bytes.js";
 import type { ImageType } from "../Node.js";
@@ -243,7 +243,7 @@ export async function imageToDataUrl(input: ImageType): Promise<string> {
    _.isString(input)
  ) {
    // string or file URL
-    const dataBuffer = await defaultFS.readFile(
+    const dataBuffer = await fs.readFile(
      input instanceof URL ? input.pathname : input,
    );
    input = new Blob([dataBuffer]);
@@ -311,7 +311,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
    // NOTE: if the vector store keeps text,
    // we only need to add image and index nodes
    for (let i = 0; i < nodes.length; ++i) {
-      const type = nodes[i].getType();
+      const { type } = nodes[i];
      if (
        !vectorStore.storesText ||
        type === ObjectType.INDEX ||
@@ -0,0 +1,60 @@
+import { getEnv } from "@llamaindex/env";
+import type { BaseNode } from "../../Node.js";
+import { getChunkSize } from "../settings/chunk-size.js";
+
+const emitOnce = false;
+
+export function chunkSizeCheck(
+  contentGetter: () => string,
+  _context: ClassMethodDecoratorContext | ClassGetterDecoratorContext,
+) {
+  return function <Node extends BaseNode>(this: Node) {
+    const content = contentGetter.call(this);
+    const chunkSize = getChunkSize();
+    const enableChunkSizeCheck = getEnv("ENABLE_CHUNK_SIZE_CHECK") === "true";
+    if (
+      enableChunkSizeCheck &&
+      chunkSize !== undefined &&
+      content.length > chunkSize
+    ) {
+      console.warn(
+        `Node (${this.id_}) is larger than chunk size: ${content.length}`,
+      );
+      if (!emitOnce) {
+        console.warn(
+          "Will truncate the content if it is larger than chunk size",
+        );
+        console.warn("If you want to disable this behavior:");
+        console.warn("  1. Set Settings.chunkSize = undefined");
+        console.warn("  2. Set Settings.chunkSize to a larger value");
+        console.warn(
+          "  3. Change the way of splitting content into smaller chunks",
+        );
+      }
+      return content.slice(0, chunkSize);
+    }
+    return content;
+  };
+}
+
+export function lazyInitHash(
+  value: ClassAccessorDecoratorTarget<BaseNode, string>,
+  _context: ClassAccessorDecoratorContext,
+): ClassAccessorDecoratorResult<BaseNode, string> {
+  return {
+    get() {
+      const oldValue = value.get.call(this);
+      if (oldValue === "") {
+        const hash = this.generateHash();
+        value.set.call(this, hash);
+      }
+      return value.get.call(this);
+    },
+    set(newValue: string) {
+      value.set.call(this, newValue);
+    },
+    init(value: string): string {
+      return value;
+    },
+  };
+}
@@ -0,0 +1,19 @@
+import { AsyncLocalStorage } from "@llamaindex/env";
+
+const chunkSizeAsyncLocalStorage = new AsyncLocalStorage<number | undefined>();
+const globalChunkSize: number | null = null;
+
+export function getChunkSize(): number | undefined {
+  return globalChunkSize ?? chunkSizeAsyncLocalStorage.getStore();
+}
+
+export function setChunkSize(chunkSize: number | undefined) {
+  chunkSizeAsyncLocalStorage.enterWith(chunkSize);
+}
+
+export function withChunkSize<Result>(
+  embeddedModel: number,
+  fn: () => Result,
+): Result {
+  return chunkSizeAsyncLocalStorage.run(embeddedModel, fn);
+}
@@ -139,7 +139,7 @@ class GeminiHelper {
  > = {
    user: "user",
    system: "user",
-    assistant: "user",
+    assistant: "model",
    memory: "user",
  };

@@ -152,38 +152,26 @@ class GeminiHelper {
  };

  public static mergeNeighboringSameRoleMessages(
-    messages: ChatMessage[],
-  ): ChatMessage[] {
-    // Gemini does not support multiple messages of the same role in a row, so we merge them
-    const mergedMessages: ChatMessage[] = [];
-    let i: number = 0;
-
-    while (i < messages.length) {
-      const currentMessage: ChatMessage = messages[i];
-      // Initialize merged content with current message content
-      const mergedContent: MessageContent[] = [currentMessage.content];
-
-      // Check if the next message exists and has the same role
-      while (
-        i + 1 < messages.length &&
-        this.ROLES_TO_GEMINI[messages[i + 1].role] ===
-          this.ROLES_TO_GEMINI[currentMessage.role]
-      ) {
-        i++;
-        const nextMessage: ChatMessage = messages[i];
-        mergedContent.push(nextMessage.content);
-      }
-
-      // Create a new ChatMessage object with merged content
-      const mergedMessage: ChatMessage = {
-        role: currentMessage.role,
-        content: mergedContent.join("\n"),
-      };
-      mergedMessages.push(mergedMessage);
-      i++;
-    }
-
-    return mergedMessages;
+    messages: GeminiMessageContent[],
+  ): GeminiMessageContent[] {
+    return messages.reduce(
+      (
+        result: GeminiMessageContent[],
+        current: GeminiMessageContent,
+        index: number,
+      ) => {
+        if (index > 0 && messages[index - 1].role === current.role) {
+          result[result.length - 1].parts = [
+            ...result[result.length - 1].parts,
+            ...current.parts,
+          ];
+        } else {
+          result.push(current);
+        }
+        return result;
+      },
+      [],
+    );
  }

  public static messageContentToGeminiParts(content: MessageContent): Part[] {
@@ -214,8 +202,8 @@ class GeminiHelper {
    message: ChatMessage,
  ): GeminiMessageContent {
    return {
-      role: this.ROLES_TO_GEMINI[message.role],
-      parts: this.messageContentToGeminiParts(message.content),
+      role: GeminiHelper.ROLES_TO_GEMINI[message.role],
+      parts: GeminiHelper.messageContentToGeminiParts(message.content),
    };
  }
 }
@@ -260,22 +248,20 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
    chat: ChatSession;
    messageContent: Part[];
  } {
-    const { messages } = params;
-    const mergedMessages =
-      GeminiHelper.mergeNeighboringSameRoleMessages(messages);
-    const history = mergedMessages.slice(0, -1);
-    const nextMessage = mergedMessages[mergedMessages.length - 1];
-    const messageContent = GeminiHelper.chatMessageToGemini(nextMessage).parts;
+    const messages = GeminiHelper.mergeNeighboringSameRoleMessages(
+      params.messages.map(GeminiHelper.chatMessageToGemini),
+    );
+
+    const history = messages.slice(0, -1);

    const client = this.session.gemini.getGenerativeModel(this.metadata);

    const chat = client.startChat({
-      history: history.map(GeminiHelper.chatMessageToGemini),
+      history,
    });
-
    return {
      chat,
-      messageContent,
+      messageContent: messages[messages.length - 1].parts,
    };
  }

@@ -5,7 +5,7 @@ export {
  Anthropic,
 } from "./anthropic.js";
 export { FireworksLLM } from "./fireworks.js";
-export { GEMINI_MODEL, Gemini } from "./gemini.js";
+export { GEMINI_MODEL, Gemini, GeminiSession } from "./gemini.js";
 export { Groq } from "./groq.js";
 export { HuggingFaceInferenceAPI } from "./huggingface.js";
 export {
@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import type { ParseConfig } from "papaparse";
 import Papa from "papaparse";
 import { Document } from "../Node.js";
@@ -40,11 +39,8 @@ export class PapaCSVReader implements FileReader {
   * @param {GenericFileSystem} [fs=DEFAULT_FS] - The file system to use for reading the file.
   * @returns {Promise<Document[]>}
   */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const fileContent = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const fileContent = await fs.readFile(file, "utf-8");
    const result = Papa.parse(fileContent, this.papaConfig);
    const textList = result.data.map((row: any) => {
      // Compatible with header row mode
@@ -1,16 +1,12 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import mammoth from "mammoth";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";

 export class DocxReader implements FileReader {
  /** DocxParser */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readRawFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file);
    const { value } = await mammoth.extractRawText({ buffer: dataBuffer });
    return [new Document({ text: value, id_: file })];
  }
@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";

@@ -15,14 +14,10 @@ export class HTMLReader implements FileReader {
   * Public method for this reader.
   * Required by BaseReader interface.
   * @param file Path/name of the file to be loaded.
-   * @param fs fs wrapper interface for getting the file content.
   * @returns Promise<Document[]> A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
   */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file, "utf-8");
    const htmlOptions = this.getOptions();
    const content = await this.parseContent(dataBuffer, htmlOptions);
    return [new Document({ text: content, id_: file })];
@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import type { Document } from "../Node.js";
 import { ImageDocument } from "../Node.js";
 import type { FileReader } from "./type.js";
@@ -15,11 +14,8 @@ export class ImageReader implements FileReader {
   * @param fs fs wrapper interface for getting the file content.
   * @returns Promise<Document[]> A Promise object, eventually yielding zero or one ImageDocument of the specified file.
   */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readRawFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file);
    const blob = new Blob([dataBuffer]);
    return [new ImageDocument({ image: blob, id_: file })];
  }
@@ -1,4 +1,4 @@
-import { defaultFS, getEnv, type GenericFileSystem } from "@llamaindex/env";
+import { fs, getEnv } from "@llamaindex/env";
 import { filetypemime } from "magic-bytes.js";
 import { Document } from "../Node.js";
 import type { FileReader, Language, ResultType } from "./type.js";
@@ -79,14 +79,11 @@ export class LlamaParseReader implements FileReader {
    this.apiKey = params.apiKey;
  }

-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
+  async loadData(file: string): Promise<Document[]> {
    const metadata = { file_path: file };

    // Load data, set the mime type
-    const data = await fs.readRawFile(file);
+    const data = await fs.readFile(file);
    const mimeType = await this.getMimeType(data);

    const body = new FormData();
@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";

@@ -88,11 +87,8 @@ export class MarkdownReader implements FileReader {
    return this.markdownToTups(modifiedContent);
  }

-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const content = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const content = await fs.readFile(file, "utf-8");
    const tups = this.parseTups(content);
    const results: Document[] = [];
    let counter = 0;
@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { BaseReader } from "./type.js";

@@ -7,11 +6,8 @@ import type { BaseReader } from "./type.js";
 * Read the text of a PDF
 */
 export class PDFReader implements BaseReader {
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const content = await fs.readRawFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const content = await fs.readFile(file);
    const pages = await readPDF(content);
    return pages.map((text, page) => {
      const id_ = `${file}_${page + 1}`;
@@ -1,5 +1,4 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { fs, path } from "@llamaindex/env";
 import { Document, type Metadata } from "../Node.js";
 import { walk } from "../storage/FileSystem.js";
 import { TextFileReader } from "./TextFileReader.js";
@@ -19,7 +18,6 @@ enum ReaderStatus {

 export type SimpleDirectoryReaderLoadDataParams = {
  directoryPath: string;
-  fs?: CompleteFileSystem;
  defaultReader?: BaseReader | null;
  fileExtToReader?: Record<string, BaseReader>;
 };
@@ -45,7 +43,6 @@ export class SimpleDirectoryReader implements BaseReader {

    const {
      directoryPath,
-      fs = defaultFS,
      defaultReader = new TextFileReader(),
      fileExtToReader,
    } = params;
@@ -58,7 +55,7 @@ export class SimpleDirectoryReader implements BaseReader {
    }

    const docs: Document[] = [];
-    for await (const filePath of walk(fs, directoryPath)) {
+    for await (const filePath of walk(directoryPath)) {
      try {
        const fileExt = path.extname(filePath).slice(1).toLowerCase();

@@ -1,5 +1,4 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { BaseReader } from "./type.js";

@@ -8,11 +7,8 @@ import type { BaseReader } from "./type.js";
 */

 export class TextFileReader implements BaseReader {
-  async loadData(
-    file: string,
-    fs: CompleteFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file, "utf-8");
    return [new Document({ text: dataBuffer, id_: file })];
  }
 }
@@ -1,4 +1,3 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
 import type { Document } from "../Node.js";

 /**
@@ -12,7 +11,7 @@ export interface BaseReader {
 * A reader takes file paths and imports data into Document objects.
 */
 export interface FileReader extends BaseReader {
-  loadData(filePath: string, fs?: CompleteFileSystem): Promise<Document[]>;
+  loadData(filePath: string): Promise<Document[]>;
 }

 // For LlamaParseReader.ts
@@ -1,5 +1,6 @@
-import type { GenericFileSystem, WalkableFileSystem } from "@llamaindex/env";
-// FS utility functions
+// FS utility helpers
+
+import { fs } from "@llamaindex/env";

 /**
 * Checks if a file exists.
@@ -8,10 +9,7 @@ import type { GenericFileSystem, WalkableFileSystem } from "@llamaindex/env";
 * @param path The path to the file to check.
 * @returns A promise that resolves to true if the file exists, false otherwise.
 */
-export async function exists(
-  fs: GenericFileSystem,
-  path: string,
-): Promise<boolean> {
+export async function exists(path: string): Promise<boolean> {
  try {
    await fs.access(path);
    return true;
@@ -22,19 +20,15 @@ export async function exists(

 /**
 * Recursively traverses a directory and yields all the paths to the files in it.
- * @param fs The filesystem to use.
 * @param dirPath The path to the directory to traverse.
 */
-export async function* walk(
-  fs: WalkableFileSystem,
-  dirPath: string,
-): AsyncIterable<string> {
+export async function* walk(dirPath: string): AsyncIterable<string> {
  const entries = await fs.readdir(dirPath);
  for (const entry of entries) {
    const fullPath = `${dirPath}/${entry}`;
    const stats = await fs.stat(fullPath);
    if (stats.isDirectory()) {
-      yield* walk(fs, fullPath);
+      yield* walk(fullPath);
    } else {
      yield fullPath;
    }
@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { path } from "@llamaindex/env";
 import {
  DEFAULT_IMAGE_VECTOR_NAMESPACE,
  DEFAULT_NAMESPACE,
@@ -25,7 +24,6 @@ export type BuilderParams = {
  imageVectorStore: VectorStore;
  storeImages: boolean;
  persistDir: string;
-  fs: GenericFileSystem;
 };

 export async function storageContextFromDefaults({
@@ -35,7 +33,6 @@ export async function storageContextFromDefaults({
  imageVectorStore,
  storeImages,
  persistDir,
-  fs,
 }: Partial<BuilderParams>): Promise<StorageContext> {
  if (!persistDir) {
    docStore = docStore || new SimpleDocumentStore();
@@ -43,22 +40,16 @@ export async function storageContextFromDefaults({
    vectorStore = vectorStore || new SimpleVectorStore();
    imageVectorStore = storeImages ? new SimpleVectorStore() : imageVectorStore;
  } else {
-    fs = fs || defaultFS;
    docStore =
      docStore ||
-      (await SimpleDocumentStore.fromPersistDir(
-        persistDir,
-        DEFAULT_NAMESPACE,
-        fs,
-      ));
+      (await SimpleDocumentStore.fromPersistDir(persistDir, DEFAULT_NAMESPACE));
    indexStore =
-      indexStore || (await SimpleIndexStore.fromPersistDir(persistDir, fs));
+      indexStore || (await SimpleIndexStore.fromPersistDir(persistDir));
    vectorStore =
-      vectorStore || (await SimpleVectorStore.fromPersistDir(persistDir, fs));
+      vectorStore || (await SimpleVectorStore.fromPersistDir(persistDir));
    imageVectorStore = storeImages
      ? await SimpleVectorStore.fromPersistDir(
          path.join(persistDir, DEFAULT_IMAGE_VECTOR_NAMESPACE),
-          fs,
        )
      : imageVectorStore;
  }
@@ -5,7 +5,7 @@ import { DEFAULT_NAMESPACE } from "../constants.js";
 import type { BaseKVStore } from "../kvStore/types.js";
 import type { RefDocInfo } from "./types.js";
 import { BaseDocumentStore } from "./types.js";
-import { docToJson, jsonToDoc } from "./utils.js";
+import { docToJson, isValidDocJson, jsonToDoc } from "./utils.js";

 type DocMetaData = { docHash: string; refDocId?: string };

@@ -27,7 +27,12 @@ export class KVDocumentStore extends BaseDocumentStore {
    const jsonDict = await this.kvstore.getAll(this.nodeCollection);
    const docs: Record<string, BaseNode> = {};
    for (const key in jsonDict) {
-      docs[key] = jsonToDoc(jsonDict[key] as Record<string, any>);
+      const value = jsonDict[key];
+      if (isValidDocJson(value)) {
+        docs[key] = jsonToDoc(value);
+      } else {
+        console.warn(`Invalid JSON for docId ${key}`);
+      }
    }
    return docs;
  }
@@ -51,7 +56,7 @@ export class KVDocumentStore extends BaseDocumentStore {
      await this.kvstore.put(nodeKey, data, this.nodeCollection);
      const metadata: DocMetaData = { docHash: doc.hash };

-      if (doc.getType() === ObjectType.TEXT && doc.sourceNode !== undefined) {
+      if (doc.type === ObjectType.TEXT && doc.sourceNode !== undefined) {
        const refDocInfo = (await this.getRefDocInfo(
          doc.sourceNode.nodeId,
        )) || {
@@ -86,6 +91,9 @@ export class KVDocumentStore extends BaseDocumentStore {
        return;
      }
    }
+    if (!isValidDocJson(json)) {
+      throw new Error(`Invalid JSON for docId ${docId}`);
+    }
    return jsonToDoc(json);
  }

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { path } from "@llamaindex/env";
 import _ from "lodash";
 import {
  DEFAULT_DOC_STORE_PERSIST_FILENAME,
@@ -25,26 +24,19 @@ export class SimpleDocumentStore extends KVDocumentStore {
  static async fromPersistDir(
    persistDir: string = DEFAULT_PERSIST_DIR,
    namespace?: string,
-    fsModule?: GenericFileSystem,
  ): Promise<SimpleDocumentStore> {
    const persistPath = path.join(
      persistDir,
      DEFAULT_DOC_STORE_PERSIST_FILENAME,
    );
-    return await SimpleDocumentStore.fromPersistPath(
-      persistPath,
-      namespace,
-      fsModule,
-    );
+    return await SimpleDocumentStore.fromPersistPath(persistPath, namespace);
  }

  static async fromPersistPath(
    persistPath: string,
    namespace?: string,
-    fs?: GenericFileSystem,
  ): Promise<SimpleDocumentStore> {
-    fs = fs || defaultFS;
-    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
+    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath);
    return new SimpleDocumentStore(simpleKVStore, namespace);
  }

@@ -53,14 +45,12 @@ export class SimpleDocumentStore extends KVDocumentStore {
      DEFAULT_PERSIST_DIR,
      DEFAULT_DOC_STORE_PERSIST_FILENAME,
    ),
-    fs?: GenericFileSystem,
  ): Promise<void> {
-    fs = fs || defaultFS;
    if (
      _.isObject(this.kvStore) &&
      this.kvStore instanceof BaseInMemoryKVStore
    ) {
-      await this.kvStore.persist(persistPath, fs);
+      await this.kvStore.persist(persistPath);
    }
  }

@@ -1,4 +1,3 @@
-import type { GenericFileSystem } from "@llamaindex/env";
 import { BaseNode } from "../../Node.js";
 import {
  DEFAULT_DOC_STORE_PERSIST_FILENAME,
@@ -14,10 +13,7 @@ export interface RefDocInfo {

 export abstract class BaseDocumentStore {
  // Save/load
-  persist(
-    persistPath: string = defaultPersistPath,
-    fs?: GenericFileSystem,
-  ): void {
+  persist(persistPath: string = defaultPersistPath): void {
    // Persist the docstore to a file.
  }

@@ -4,14 +4,28 @@ import { Document, ObjectType, TextNode } from "../../Node.js";
 const TYPE_KEY = "__type__";
 const DATA_KEY = "__data__";

-export function docToJson(doc: BaseNode): Record<string, any> {
+type DocJson = {
+  [TYPE_KEY]: ObjectType;
+  [DATA_KEY]: string;
+};
+
+export function isValidDocJson(docJson: any): docJson is DocJson {
+  return (
+    typeof docJson === "object" &&
+    docJson !== null &&
+    docJson[TYPE_KEY] !== undefined &&
+    docJson[DATA_KEY] !== undefined
+  );
+}
+
+export function docToJson(doc: BaseNode): DocJson {
  return {
-    [DATA_KEY]: JSON.stringify(doc),
-    [TYPE_KEY]: doc.getType(),
+    [DATA_KEY]: JSON.stringify(doc.toJSON()),
+    [TYPE_KEY]: doc.type,
  };
 }

-export function jsonToDoc(docDict: Record<string, any>): BaseNode {
+export function jsonToDoc(docDict: DocJson): BaseNode {
  const docType = docDict[TYPE_KEY];
  const dataDict = JSON.parse(docDict[DATA_KEY]);
  let doc: BaseNode;
@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { path } from "@llamaindex/env";
 import {
  DEFAULT_INDEX_STORE_PERSIST_FILENAME,
  DEFAULT_PERSIST_DIR,
@@ -20,28 +19,21 @@ export class SimpleIndexStore extends KVIndexStore {

  static async fromPersistDir(
    persistDir: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = defaultFS,
  ): Promise<SimpleIndexStore> {
    const persistPath = path.join(
      persistDir,
      DEFAULT_INDEX_STORE_PERSIST_FILENAME,
    );
-    return this.fromPersistPath(persistPath, fs);
+    return this.fromPersistPath(persistPath);
  }

-  static async fromPersistPath(
-    persistPath: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<SimpleIndexStore> {
-    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
+  static async fromPersistPath(persistPath: string): Promise<SimpleIndexStore> {
+    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath);
    return new SimpleIndexStore(simpleKVStore);
  }

-  async persist(
-    persistPath: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<void> {
-    await this.kvStore.persist(persistPath, fs);
+  async persist(persistPath: string = DEFAULT_PERSIST_DIR): Promise<void> {
+    this.kvStore.persist(persistPath);
  }

  static fromDict(saveDict: DataType): SimpleIndexStore {
@@ -1,4 +1,3 @@
-import type { GenericFileSystem } from "@llamaindex/env";
 import type { IndexStruct } from "../../indices/IndexStruct.js";
 import {
  DEFAULT_INDEX_STORE_PERSIST_FILENAME,
@@ -16,10 +15,7 @@ export abstract class BaseIndexStore {

  abstract getIndexStruct(structId?: string): Promise<IndexStruct | undefined>;

-  async persist(
-    persistPath: string = defaultPersistPath,
-    fs?: GenericFileSystem,
-  ): Promise<void> {
+  async persist(persistPath: string = defaultPersistPath): Promise<void> {
    // Persist the index store to disk.
  }
 }
@@ -1,6 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
-import _ from "lodash";
+import { fs, path } from "@llamaindex/env";
 import { exists } from "../FileSystem.js";
 import { DEFAULT_COLLECTION } from "../constants.js";
 import { BaseKVStore } from "./types.js";
@@ -9,7 +7,6 @@ export type DataType = Record<string, Record<string, any>>;

 export class SimpleKVStore extends BaseKVStore {
  private persistPath: string | undefined;
-  private fs: GenericFileSystem | undefined;

  constructor(private data: DataType = {}) {
    super();
@@ -23,10 +20,10 @@ export class SimpleKVStore extends BaseKVStore {
    if (!(collection in this.data)) {
      this.data[collection] = {};
    }
-    this.data[collection][key] = _.clone(val); // Creating a shallow copy of the object
+    this.data[collection][key] = structuredClone(val); // Creating a shallow copy of the object

    if (this.persistPath) {
-      await this.persist(this.persistPath, this.fs);
+      await this.persist(this.persistPath);
    }
  }

@@ -35,17 +32,17 @@ export class SimpleKVStore extends BaseKVStore {
    collection: string = DEFAULT_COLLECTION,
  ): Promise<any> {
    const collectionData = this.data[collection];
-    if (_.isNil(collectionData)) {
+    if (collectionData == null) {
      return null;
    }
    if (!(key in collectionData)) {
      return null;
    }
-    return _.clone(collectionData[key]); // Creating a shallow copy of the object
+    return structuredClone(collectionData[key]); // Creating a shallow copy of the object
  }

  async getAll(collection: string = DEFAULT_COLLECTION): Promise<DataType> {
-    return _.clone(this.data[collection]); // Creating a shallow copy of the object
+    return structuredClone(this.data[collection]); // Creating a shallow copy of the object
  }

  async delete(
@@ -55,31 +52,25 @@ export class SimpleKVStore extends BaseKVStore {
    if (key in this.data[collection]) {
      delete this.data[collection][key];
      if (this.persistPath) {
-        await this.persist(this.persistPath, this.fs);
+        await this.persist(this.persistPath);
      }
      return true;
    }
    return false;
  }

-  async persist(
-    persistPath: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<void> {
+  async persist(persistPath: string): Promise<void> {
    // TODO: decide on a way to polyfill path
    const dirPath = path.dirname(persistPath);
-    if (!(await exists(fs, dirPath))) {
+    if (!(await exists(dirPath))) {
      await fs.mkdir(dirPath);
    }
    await fs.writeFile(persistPath, JSON.stringify(this.data));
  }

-  static async fromPersistPath(
-    persistPath: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<SimpleKVStore> {
+  static async fromPersistPath(persistPath: string): Promise<SimpleKVStore> {
    const dirPath = path.dirname(persistPath);
-    if (!(await exists(fs, dirPath))) {
+    if (!(await exists(dirPath))) {
      await fs.mkdir(dirPath);
    }

@@ -95,7 +86,6 @@ export class SimpleKVStore extends BaseKVStore {

    const store = new SimpleKVStore(data);
    store.persistPath = persistPath;
-    store.fs = fs;
    return store;
  }

@@ -1,4 +1,3 @@
-import type { GenericFileSystem } from "@llamaindex/env";
 const defaultCollection = "data";

 type StoredValue = Record<string, any> | null;
@@ -15,7 +14,7 @@ export abstract class BaseKVStore {
 }

 export abstract class BaseInMemoryKVStore extends BaseKVStore {
-  abstract persist(persistPath: string, fs?: GenericFileSystem): void;
+  abstract persist(persistPath: string): void;
  static fromPersistPath(persistPath: string): BaseInMemoryKVStore {
    throw new Error("Method not implemented.");
  }
@@ -6,7 +6,6 @@ import type {
  VectorStoreQueryResult,
 } from "./types.js";

-import type { GenericFileSystem } from "@llamaindex/env";
 import type { BaseNode, Metadata } from "../../Node.js";
 import { Document, MetadataMode } from "../../Node.js";

@@ -307,10 +306,7 @@ export class PGVectorStore implements VectorStore {
   * @param fs
   * @returns Resolved Promise.
   */
-  persist(
-    persistPath: string,
-    fs?: GenericFileSystem | undefined,
-  ): Promise<void> {
+  persist(persistPath: string): Promise<void> {
    return Promise.resolve();
  }
 }
@@ -6,7 +6,7 @@ import type {
  VectorStoreQueryResult,
 } from "./types.js";

-import { getEnv, type GenericFileSystem } from "@llamaindex/env";
+import { getEnv } from "@llamaindex/env";
 import type {
  FetchResponse,
  Index,
@@ -188,10 +188,7 @@ export class PineconeVectorStore implements VectorStore {
   * @param fs
   * @returns Resolved Promise.
   */
-  persist(
-    persistPath: string,
-    fs?: GenericFileSystem | undefined,
-  ): Promise<void> {
+  persist(persistPath: string): Promise<void> {
    return Promise.resolve();
  }

@@ -1,6 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
-import _ from "lodash";
+import { fs, path } from "@llamaindex/env";
 import type { BaseNode } from "../../Node.js";
 import {
  getTopKEmbeddings,
@@ -32,20 +30,17 @@ class SimpleVectorStoreData {
 export class SimpleVectorStore implements VectorStore {
  storesText: boolean = false;
  private data: SimpleVectorStoreData = new SimpleVectorStoreData();
-  private fs: GenericFileSystem = defaultFS;
  private persistPath: string | undefined;

-  constructor(data?: SimpleVectorStoreData, fs?: GenericFileSystem) {
+  constructor(data?: SimpleVectorStoreData) {
    this.data = data || new SimpleVectorStoreData();
-    this.fs = fs || defaultFS;
  }

  static async fromPersistDir(
    persistDir: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = defaultFS,
  ): Promise<SimpleVectorStore> {
-    const persistPath = `${persistDir}/vector_store.json`;
-    return await SimpleVectorStore.fromPersistPath(persistPath, fs);
+    const persistPath = path.join(persistDir, "vector_store.json");
+    return await SimpleVectorStore.fromPersistPath(persistPath);
  }

  get client(): any {
@@ -68,7 +63,7 @@ export class SimpleVectorStore implements VectorStore {
    }

    if (this.persistPath) {
-      await this.persist(this.persistPath, this.fs);
+      await this.persist(this.persistPath);
    }

    return embeddingResults.map((result) => result.id_);
@@ -83,13 +78,13 @@ export class SimpleVectorStore implements VectorStore {
      delete this.data.textIdToRefDocId[textId];
    }
    if (this.persistPath) {
-      await this.persist(this.persistPath, this.fs);
+      await this.persist(this.persistPath);
    }
    return Promise.resolve();
  }

  async query(query: VectorStoreQuery): Promise<VectorStoreQueryResult> {
-    if (!_.isNil(query.filters)) {
+    if (!(query.filters == null)) {
      throw new Error(
        "Metadata filters not implemented for SimpleVectorStore yet.",
      );
@@ -147,12 +142,10 @@ export class SimpleVectorStore implements VectorStore {
  }

  async persist(
-    persistPath: string = `${DEFAULT_PERSIST_DIR}/vector_store.json`,
-    fs?: GenericFileSystem,
+    persistPath: string = path.join(DEFAULT_PERSIST_DIR, "vector_store.json"),
  ): Promise<void> {
-    fs = fs || this.fs;
    const dirPath = path.dirname(persistPath);
-    if (!(await exists(fs, dirPath))) {
+    if (!(await exists(dirPath))) {
      await fs.mkdir(dirPath);
    }

@@ -161,10 +154,9 @@ export class SimpleVectorStore implements VectorStore {

  static async fromPersistPath(
    persistPath: string,
-    fs: GenericFileSystem = defaultFS,
  ): Promise<SimpleVectorStore> {
    const dirPath = path.dirname(persistPath);
-    if (!(await exists(fs, dirPath))) {
+    if (!(await exists(dirPath))) {
      await fs.mkdir(dirPath, { recursive: true });
    }

@@ -183,7 +175,6 @@ export class SimpleVectorStore implements VectorStore {
    data.textIdToRefDocId = dataDict.textIdToRefDocId ?? {};
    const store = new SimpleVectorStore(data);
    store.persistPath = persistPath;
-    store.fs = fs;
    return store;
  }

@@ -1,8 +1,4 @@
-import {
-  OpenAIEmbedding,
-  SimilarityType,
-  similarity,
-} from "llamaindex/embeddings/index";
+import { OpenAIEmbedding, SimilarityType, similarity } from "llamaindex";
 import { beforeAll, describe, expect, test } from "vitest";
 import { mockEmbeddingModel } from "./utility/mockOpenAI.js";

@@ -1,6 +1,26 @@
-import { TextNode } from "llamaindex/Node";
+import { Document, TextNode } from "llamaindex/Node";
 import { beforeEach, describe, expect, test } from "vitest";

+describe("Document", () => {
+  let document: Document;
+
+  beforeEach(() => {
+    document = new Document({ text: "Hello World" });
+  });
+
+  test("should generate a hash", () => {
+    expect(document.hash).toMatchInlineSnapshot(
+      `"1mkNkQC30mZlBBG48DNuG2WSKcTQ32DImC+4JUoVijg="`,
+    );
+  });
+
+  test("clone should have the same hash", () => {
+    const hash = document.hash;
+    const clone = document.clone();
+    expect(clone.hash).toBe(hash);
+  });
+});
+
 describe("TextNode", () => {
  let node: TextNode;

@@ -9,7 +29,9 @@ describe("TextNode", () => {
  });

  test("should generate a hash", () => {
-    expect(node.hash).toBe("nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do=");
+    expect(node.hash).toMatchInlineSnapshot(
+      `"nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do="`,
+    );
  });

  test("clone should have the same hash", () => {
@@ -17,4 +39,35 @@ describe("TextNode", () => {
    const clone = node.clone();
    expect(clone.hash).toBe(hash);
  });
+
+  test("node toJSON should keep the same", () => {
+    node.metadata.something = 1;
+    node.metadata.somethingElse = "2";
+    expect(node.toJSON()).toMatchInlineSnapshot(
+      {
+        id_: expect.any(String),
+      },
+      `
+      {
+        "embedding": undefined,
+        "endCharIdx": undefined,
+        "excludedEmbedMetadataKeys": [],
+        "excludedLlmMetadataKeys": [],
+        "hash": "nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do=",
+        "id_": Any<String>,
+        "metadata": {
+          "something": 1,
+          "somethingElse": "2",
+        },
+        "metadataSeparator": "
+      ",
+        "relationships": {},
+        "startCharIdx": undefined,
+        "text": "Hello World",
+        "textTemplate": "",
+        "type": "TEXT",
+      }
+    `,
+    );
+  });
 });
@@ -5,8 +5,7 @@ import {
  storageContextFromDefaults,
 } from "llamaindex";
 import { DocStoreStrategy } from "llamaindex/ingestion/strategies/index";
-import { rmSync } from "node:fs";
-import { mkdtemp } from "node:fs/promises";
+import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll, beforeAll, describe, expect, test } from "vitest";
@@ -15,7 +14,7 @@ const testDir = await mkdtemp(join(tmpdir(), "test-"));

 import { mockServiceContext } from "../utility/mockServiceContext.js";

-describe.sequential("VectorStoreIndex", () => {
+describe("VectorStoreIndex", () => {
  let serviceContext: ServiceContext;
  let storageContext: StorageContext;
  let testStrategy: (
@@ -57,7 +56,7 @@ describe.sequential("VectorStoreIndex", () => {
    expect(entries[0]).toBe(entries[1]);
  });

-  afterAll(() => {
-    rmSync(testDir, { recursive: true });
+  afterAll(async () => {
+    await rm(testDir, { recursive: true });
  });
 });
@@ -0,0 +1,7 @@
+{
+	"detectiveOptions": {
+		"ts": {
+			"skipTypeImports": true
+		}
+	}
+}
@@ -1,5 +1,16 @@
 # @llamaindex/env

+## 0.1.3
+
+### Patch Changes
+
+- e072c45: fix: remove non-standard API `pipeline`
+- 9e133ac: refactor: remove `defaultFS` from parameters
+
+  We don't accept passing fs in the parameter since it's unnecessary for a determined JS environment.
+
+  This was a polyfill way for the non-Node.js environment, but now we use another way to polyfill APIs.
+
 ## 0.1.2

 ### Patch Changes
@@ -1,11 +1,10 @@
 {
  "name": "@llamaindex/env",
-  "version": "0.1.2",
+  "version": "0.1.3",
  "exports": {
-    ".": "./src/index.ts",
-    "./type": "./src/type.ts"
+    ".": "./src/index.ts"
  },
  "publish": {
-    "include": ["LICENSE", "README.md", "src/**/*.ts", "jsr.json"]
+    "include": ["LICENSE", "README.md", "src/**/*", "jsr.json"]
  }
 }
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/env",
  "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.2",
+  "version": "0.1.3",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -33,8 +33,8 @@
        "default": "./dist/index.workerd.js"
      },
      "edge-light": {
-        "types": "./dist/type/index.polyfill.d.ts",
-        "default": "./dist/index.polyfill.js"
+        "types": "./dist/type/index.edge-light.d.ts",
+        "default": "./dist/index.edge-light.js"
      },
      "import": {
        "types": "./dist/type/index.d.ts",
@@ -44,16 +44,6 @@
        "types": "./dist/type/index.d.ts",
        "default": "./dist/cjs/index.js"
      }
-    },
-    "./*": {
-      "import": {
-        "types": "./dist/type/*.d.ts",
-        "default": "./dist/*.js"
-      },
-      "require": {
-        "types": "./dist/type/*.d.ts",
-        "default": "./dist/cjs/*.js"
-      }
    }
  },
  "files": [
@@ -71,7 +61,9 @@
    "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file ../../.cjs.swcrc",
    "build:type": "tsc -p tsconfig.json",
    "postbuild": "node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"",
-    "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\""
+    "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\"",
+    "circular-check": "madge -c ./src/index.ts",
+    "test": "vitest"
  },
  "devDependencies": {
    "@aws-crypto/sha256-js": "^5.2.0",
@@ -79,7 +71,7 @@
    "@swc/core": "^1.5.5",
    "concurrently": "^8.2.2",
    "pathe": "^1.1.2",
-    "readable-stream": "^4.5.2"
+    "vitest": "^1.6.0"
  },
  "dependencies": {
    "@types/lodash": "^4.17.1",
@@ -87,8 +79,7 @@
  },
  "peerDependencies": {
    "@aws-crypto/sha256-js": "^5.2.0",
-    "pathe": "^1.1.2",
-    "readable-stream": "^4.5.2"
+    "pathe": "^1.1.2"
  },
  "peerDependenciesMeta": {
    "@aws-crypto/sha256-js": {
@@ -96,9 +87,6 @@
    },
    "pathe": {
      "optional": true
-    },
-    "readable-stream": {
-      "optional": true
    }
  }
 }
@@ -0,0 +1 @@
+Copy from https://github.com/bundled-es-modules/memfs/blob/main/index-esm.js
@@ -0,0 +1,9 @@
+/**
+ * Memory file system, used by edge runtime, worker runtime which doesn't have access to the file system.
+ *
+ * @module
+ */
+// @ts-expect-error
+import memFS from "./memfs/index.js";
+
+export const fs = memFS.promises;
@@ -0,0 +1,10 @@
+/**
+ * Node.js built-in file system module.
+ *
+ * Bun, Deno support node:fs/promises so we don't need to re-implement it.
+ *
+ * @module
+ */
+import fs from "node:fs/promises";
+
+export { fs };
@@ -0,0 +1,6 @@
+/**
+ * Edge light environment polyfill.
+ *
+ * @module
+ */
+export * from "./polyfill.js";
@@ -13,17 +13,15 @@
 */
 import { ok } from "node:assert";
 import { createHash, randomUUID } from "node:crypto";
-import fs from "node:fs/promises";
 import { EOL } from "node:os";
 import path from "node:path";
-import { pipeline } from "node:stream/promises";
 import {
  ReadableStream,
  TransformStream,
  WritableStream,
 } from "node:stream/web";
-import type { SHA256 } from "./index.polyfill.js";
-import type { CompleteFileSystem } from "./type.js";
+import { fs } from "./fs/node.js";
+import type { SHA256 } from "./polyfill.js";

 export function createSHA256(): SHA256 {
  const hash = createHash("sha256");
@@ -37,31 +35,14 @@ export function createSHA256(): SHA256 {
  };
 }

-export const defaultFS: CompleteFileSystem = {
-  writeFile: function (path: string, content: string) {
-    return fs.writeFile(path, content, "utf-8");
-  },
-  readRawFile(path: string): Promise<Buffer> {
-    return fs.readFile(path);
-  },
-  readFile: function (path: string) {
-    return fs.readFile(path, "utf-8");
-  },
-  access: fs.access,
-  mkdir: fs.mkdir,
-  readdir: fs.readdir,
-  stat: fs.stat,
-};
-
-export type * from "./type.js";
 export { AsyncLocalStorage, CustomEvent, getEnv, setEnvs } from "./utils.js";
 export {
  EOL,
  ReadableStream,
  TransformStream,
  WritableStream,
+  fs,
  ok,
  path,
-  pipeline,
  randomUUID,
 };
@@ -7,7 +7,7 @@
 */
 import { INTERNAL_ENV } from "./utils.js";

-export * from "./index.polyfill.js";
+export * from "./polyfill.js";

 export function getEnv(name: string): string | undefined {
  return INTERNAL_ENV[name];
@@ -1,19 +1,17 @@
 /**
- * Polyfill implementation some node.js APIs.
+ * Polyfill implementation for `@llamaindex/env`.
 *
 * The code should be compatible with any JS runtime.
 *
+ * Sometimes you should overwrite the polyfill with a native implementation.
+ *
 * @module
 */
 import { Sha256 } from "@aws-crypto/sha256-js";
 import pathe from "pathe";
-import { InMemoryFileSystem, type CompleteFileSystem } from "./type.js";
-// @ts-expect-error
-import { promises } from "readable-stream";
+import { fs } from "./fs/memory.js";

-const { pipeline } = promises;
-
-export { pathe as path, pipeline };
+export { fs, pathe as path };

 export interface SHA256 {
  update(data: string | Uint8Array): void;
@@ -23,8 +21,6 @@ export interface SHA256 {

 export const EOL = "\n";

-export const defaultFS: CompleteFileSystem = new InMemoryFileSystem();
-
 export function ok(value: unknown, message?: string): asserts value {
  if (!value) {
    const error = Error(message);
@@ -49,7 +45,6 @@ export function createSHA256(): SHA256 {
 export function randomUUID(): string {
  return crypto.randomUUID();
 }
-export * from "./type.js";

 // @ts-expect-error
 const ReadableStream = globalThis.ReadableStream;
@@ -1,73 +0,0 @@
-/**
- * A filesystem interface that is meant to be compatible with
- * the 'fs' module from Node.js.
- * Allows for the use of similar interface implementation on different JS runtimes.
- */
-export type GenericFileSystem = {
-  writeFile(path: string, content: string): Promise<void>;
-  /**
-   * Reads a file and returns its content as a raw buffer.
-   */
-  readRawFile(path: string): Promise<Buffer>;
-  /**
-   * Reads a file and returns its content as an utf-8 string.
-   */
-  readFile(path: string): Promise<string>;
-  access(path: string): Promise<void>;
-  mkdir(
-    path: string,
-    options: {
-      recursive: boolean;
-    },
-  ): Promise<string | undefined>;
-  mkdir(path: string): Promise<void>;
-};
-export type WalkableFileSystem = {
-  readdir(path: string): Promise<string[]>;
-  stat(path: string): Promise<any>;
-};
-export type CompleteFileSystem = GenericFileSystem & WalkableFileSystem;
-
-/**
- * A filesystem implementation that stores files in memory.
- */
-export class InMemoryFileSystem implements CompleteFileSystem {
-  private files: Record<string, any> = {};
-
-  async writeFile(
-    path: string,
-    content: string,
-    options?: unknown,
-  ): Promise<void> {
-    this.files[path] = structuredClone(content);
-  }
-
-  async readFile(path: string): Promise<string> {
-    if (!(path in this.files)) {
-      throw new Error(`File ${path} does not exist`);
-    }
-    return structuredClone(this.files[path]);
-  }
-
-  async access(path: string): Promise<void> {
-    if (!(path in this.files)) {
-      throw new Error(`File ${path} does not exist`);
-    }
-  }
-
-  async mkdir(path: string): Promise<undefined> {
-    this.files[path] = this.files[path] ?? null;
-  }
-
-  async readdir(path: string): Promise<string[]> {
-    throw new Error("Not implemented");
-  }
-
-  async stat(path: string): Promise<any> {
-    throw new Error("Not implemented");
-  }
-
-  async readRawFile(path: string): Promise<Buffer> {
-    throw new Error("Not implemented");
-  }
-}
@@ -0,0 +1,10 @@
+import { describe, expect, test } from "vitest";
+import { fs } from "../src/fs/memory.js";
+
+describe("memfs", () => {
+  test("should be able to read and write files", async () => {
+    await fs.writeFile("/foo", "bar");
+    const content = await fs.readFile("/foo", "utf8");
+    expect(content).toBe("bar");
+  });
+});
@@ -0,0 +1,15 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./lib",
+    "module": "node16",
+    "moduleResolution": "node16",
+    "target": "ESNext"
+  },
+  "include": ["./**/*.ts"],
+  "references": [
+    {
+      "path": "../tsconfig.json"
+    }
+  ]
+}
@@ -1,5 +1,15 @@
 # @llamaindex/experimental

+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [e072c45]
+- Updated dependencies [9e133ac]
+- Updated dependencies [447105a]
+- Updated dependencies [320be3f]
+  - llamaindex@0.3.11
+
 ## 0.0.27

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/experimental",
  "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.27",
+  "version": "0.0.28",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -40,6 +40,9 @@ importers:
      lint-staged:
        specifier: ^15.2.2
        version: 15.2.2
+      madge:
+        specifier: ^7.0.0
+        version: 7.0.0(typescript@5.4.5)
      prettier:
        specifier: ^3.2.5
        version: 3.2.5
@@ -138,8 +141,8 @@ importers:
        specifier: ^2.4.2
        version: 2.4.2
      chromadb:
-        specifier: ^1.8.1
-        version: 1.8.1(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.43.0(encoding@0.1.13))
+        specifier: ^1.7.3
+        version: 1.7.3(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.46.0(encoding@0.1.13))
      commander:
        specifier: ^12.0.0
        version: 12.0.0
@@ -386,8 +389,8 @@ importers:
        specifier: ^4.4.2
        version: 4.4.2
      chromadb:
-        specifier: ~1.8.1
-        version: 1.8.1(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.43.0(encoding@0.1.13))
+        specifier: ~1.7.3
+        version: 1.7.3(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.46.0(encoding@0.1.13))
      cohere-ai:
        specifier: ^7.9.5
        version: 7.9.5(encoding@0.1.13)
@@ -413,8 +416,8 @@ importers:
        specifier: ^1.0.0
        version: 1.0.0(encoding@0.1.13)
      openai:
-        specifier: ^4.43.0
-        version: 4.43.0(encoding@0.1.13)
+        specifier: ^4.46.0
+        version: 4.46.0(encoding@0.1.13)
      papaparse:
        specifier: ^5.4.1
        version: 5.4.1
@@ -461,9 +464,6 @@ importers:
      glob:
        specifier: ^10.3.12
        version: 10.3.12
-      madge:
-        specifier: ^7.0.0
-        version: 7.0.0(typescript@5.4.5)
      typescript:
        specifier: ^5.4.5
        version: 5.4.5
@@ -652,9 +652,9 @@ importers:
      pathe:
        specifier: ^1.1.2
        version: 1.1.2
-      readable-stream:
-        specifier: ^4.5.2
-        version: 4.5.2
+      vitest:
+        specifier: ^1.6.0
+        version: 1.6.0(@types/node@20.12.11)(terser@5.31.0)

  packages/experimental:
    dependencies:
@@ -3912,9 +3912,6 @@ packages:
  buffer@5.7.1:
    resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}

-  buffer@6.0.3:
-    resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
-
  builtin-modules@3.3.0:
    resolution: {integrity: sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==}
    engines: {node: '>=6'}
@@ -4075,8 +4072,8 @@ packages:
  chownr@1.1.4:
    resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}

-  chromadb@1.8.1:
-    resolution: {integrity: sha512-NpbYydbg4Uqt/9BXKgkZXn0fqpsh2Z1yjhkhKH+rcHMoq0pwI18BFSU2QU7Fk/ZypwGefW2AvqyE/3ZJIgy4QA==}
+  chromadb@1.7.3:
+    resolution: {integrity: sha512-3GgvQjpqgk5C89x5EuTDaXKbfrdqYDJ5UVyLQ3ZmwxnpetNc+HhRDGjkvXa5KSvpQ3lmKoyDoqnN4tZepfFkbw==}
    engines: {node: '>=14.17.0'}
    peerDependencies:
      '@google/generative-ai': ^0.1.1
@@ -7191,6 +7188,10 @@ packages:
    resolution: {integrity: sha512-4SMUB/XiqnO5IrEcdzEGGTcHoeXq7D/k82v36zoqSitrMUjenZXGH5JysIH7aF7Wr+gjvq0dT2mV6wLVKA7Seg==}
    hasBin: true

+  openai@4.46.0:
+    resolution: {integrity: sha512-l0Betzsx3WIjdagqQiH14hWmwYouUzUCcB1ENvKyfG5XOqh6YC2XT7OukzEBTnweutMC91pW2ToddWn8uyD4SA==}
+    hasBin: true
+
  opener@1.5.2:
    resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==}
    hasBin: true
@@ -7900,10 +7901,6 @@ packages:
  process-nextick-args@2.0.1:
    resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==}

-  process@0.11.10:
-    resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==}
-    engines: {node: '>= 0.6.0'}
-
  prompts@2.4.2:
    resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==}
    engines: {node: '>= 6'}
@@ -8155,10 +8152,6 @@ packages:
    resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
    engines: {node: '>= 6'}

-  readable-stream@4.5.2:
-    resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==}
-    engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
-
  readable-web-to-node-stream@3.0.2:
    resolution: {integrity: sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==}
    engines: {node: '>=8'}
@@ -13968,11 +13961,6 @@ snapshots:
      base64-js: 1.5.1
      ieee754: 1.2.1

-  buffer@6.0.3:
-    dependencies:
-      base64-js: 1.5.1
-      ieee754: 1.2.1
-
  builtin-modules@3.3.0: {}

  bunchee@5.1.5(typescript@5.4.5):
@@ -14162,14 +14150,14 @@ snapshots:

  chownr@1.1.4: {}

-  chromadb@1.8.1(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.43.0(encoding@0.1.13)):
+  chromadb@1.7.3(@google/generative-ai@0.11.0)(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.46.0(encoding@0.1.13)):
    dependencies:
      cliui: 8.0.1
      isomorphic-fetch: 3.0.0(encoding@0.1.13)
    optionalDependencies:
      '@google/generative-ai': 0.11.0
      cohere-ai: 7.9.5(encoding@0.1.13)
-      openai: 4.43.0(encoding@0.1.13)
+      openai: 4.46.0(encoding@0.1.13)
    transitivePeerDependencies:
      - encoding

@@ -18002,6 +17990,19 @@ snapshots:
    transitivePeerDependencies:
      - encoding

+  openai@4.46.0(encoding@0.1.13):
+    dependencies:
+      '@types/node': 18.19.33
+      '@types/node-fetch': 2.6.11
+      abort-controller: 3.0.0
+      agentkeepalive: 4.5.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.7.0(encoding@0.1.13)
+      web-streams-polyfill: 3.3.3
+    transitivePeerDependencies:
+      - encoding
+
  opener@1.5.2: {}

  option@0.2.4: {}
@@ -18669,8 +18670,6 @@ snapshots:

  process-nextick-args@2.0.1: {}

-  process@0.11.10: {}
-
  prompts@2.4.2:
    dependencies:
      kleur: 3.0.3
@@ -19001,14 +19000,6 @@ snapshots:
      string_decoder: 1.3.0
      util-deprecate: 1.0.2

-  readable-stream@4.5.2:
-    dependencies:
-      abort-controller: 3.0.0
-      buffer: 6.0.3
-      events: 3.3.0
-      process: 0.11.10
-      string_decoder: 1.3.0
-
  readable-web-to-node-stream@3.0.2:
    dependencies:
      readable-stream: 3.6.2
@@ -50,6 +50,9 @@
    {
      "path": "./packages/env/tsconfig.json"
    },
+    {
+      "path": "./packages/env/tests/tsconfig.json"
+    },
    {
      "path": "./examples"
    },
@@ -12,6 +12,7 @@
    "test": {
      "dependsOn": ["^build"]
    },
+    "circular-check": {},
    "e2e": {
      "dependsOn": ["^build"]
    },
Author	SHA1	Message	Date
github-actions[bot]	c517f35526	Release 0.3.11 (#835 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-05-16 16:35:57 -07:00
Alex Yang	e072c45393	fix: remove non-standard API `pipeline` (#850 )	2024-05-16 16:31:48 -07:00
Alex Yang	51241865f8	feat: improve BaseNode (#848 )	2024-05-16 16:29:16 -07:00
Thuc Pham	10c83485d2	fix: allow custom task query for agents (#846 )	2024-05-16 12:48:50 -07:00
Alex Yang	1e6a18ad2d	build: fix jsr release	2024-05-15 18:03:22 -07:00
Alex Yang	9e133ac10d	refactor: remove `defaultFS` from parameters (#841 )	2024-05-15 17:37:51 -07:00
Alex Yang	ba217eec2c	chore: remove `test.py` (#842 )	2024-05-15 16:47:39 -07:00
Alex Yang	64ef70b735	build: ignore example project	2024-05-15 16:10:08 -07:00
Alex Yang	6615aaa4ab	chore: use pnpm format Using `pnpm format:write` will cause two commits which is not expected	2024-05-15 13:11:53 -07:00
Parham Saidi	447105a6dc	fix: Gemini text chat - prevent sending broken messageContent and history (#822 )	2024-05-15 16:33:55 +07:00
Huu Le (Lee)	320be3fab6	chore: rollback chromadb version to 1.7.3 (#834 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2024-05-14 16:07:44 +07:00
Alex Yang	bbd9f85a45	chore: bump `openai` (#833 )	2024-05-13 12:53:12 -07:00
				`@@ -0,0 +1 @@`
				`Copy from https://github.com/bundled-es-modules/memfs/blob/main/index-esm.js`