Release 0.6.7 (#1244 )

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
fix: add serializer in doc store (#1243 )
2026-07-09 11:25:43 -04:00 · 2024-09-23 13:25:02 -07:00 · 2024-09-23 13:11:51 -07:00
27 changed files with 146 additions and 25 deletions
@@ -1,5 +1,12 @@
 # docs

+## 0.0.76
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
 ## 0.0.75

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "docs",
-  "version": "0.0.75",
+  "version": "0.0.76",
  "private": true,
  "scripts": {
    "docusaurus": "docusaurus",
@@ -1,5 +1,12 @@
 # @llamaindex/autotool

+## 3.0.7
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
 ## 3.0.6

 ### Patch Changes
@@ -1,5 +1,13 @@
 # @llamaindex/autotool-01-node-example

+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+  - @llamaindex/autotool@3.0.7
+
 ## 0.0.15

 ### Patch Changes
@@ -13,5 +13,5 @@
  "scripts": {
    "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
  },
-  "version": "0.0.15"
+  "version": "0.0.16"
 }
@@ -1,5 +1,13 @@
 # @llamaindex/autotool-02-next-example

+## 0.1.60
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+  - @llamaindex/autotool@3.0.7
+
 ## 0.1.59

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/autotool-02-next-example",
  "private": true,
-  "version": "0.1.59",
+  "version": "0.1.60",
  "scripts": {
    "dev": "next dev",
    "build": "next build",
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/autotool",
  "type": "module",
-  "version": "3.0.6",
+  "version": "3.0.7",
  "description": "auto transpile your JS function to LLM Agent compatible",
  "files": [
    "dist",
@@ -1,5 +1,12 @@
 # @llamaindex/experimental

+## 0.0.85
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
 ## 0.0.84

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/experimental",
  "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.84",
+  "version": "0.0.85",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,13 @@
 # llamaindex

+## 0.6.7
+
+### Patch Changes
+
+- 23bcc37: fix: add `serializer` in doc store
+
+  `PostgresDocumentStore` now will not use JSON.stringify for better performance
+
 ## 0.6.6

 ### Patch Changes
@@ -1,5 +1,12 @@
 # @llamaindex/cloudflare-worker-agent-test

+## 0.0.69
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
 ## 0.0.68

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.68",
+  "version": "0.0.69",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,12 @@
 # @llamaindex/next-agent-test

+## 0.1.69
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
 ## 0.1.68

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-agent-test",
-  "version": "0.1.68",
+  "version": "0.1.69",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,12 @@
 # test-edge-runtime

+## 0.1.68
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
 ## 0.1.67

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.67",
+  "version": "0.1.68",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,12 @@
 # @llamaindex/next-node-runtime

+## 0.0.50
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
 ## 0.0.49

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.49",
+  "version": "0.0.50",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,12 @@
 # @llamaindex/waku-query-engine-test

+## 0.0.69
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
 ## 0.0.68

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.68",
+  "version": "0.0.69",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,6 +1,6 @@
 {
  "name": "llamaindex",
-  "version": "0.6.6",
+  "version": "0.6.7",
  "license": "MIT",
  "type": "module",
  "keywords": [
@@ -1,7 +1,11 @@
 import type { BaseNode, TransformComponent } from "@llamaindex/core/schema";
 import { MetadataMode } from "@llamaindex/core/schema";
 import { createSHA256 } from "@llamaindex/env";
-import { docToJson, jsonToDoc } from "../storage/docStore/utils.js";
+import {
+  docToJson,
+  jsonSerializer,
+  jsonToDoc,
+} from "../storage/docStore/utils.js";
 import { SimpleKVStore } from "../storage/kvStore/SimpleKVStore.js";
 import type { BaseKVStore } from "../storage/kvStore/types.js";

@@ -53,7 +57,7 @@ export class IngestionCache {

  async put(hash: string, nodes: BaseNode[]) {
    const val = {
-      [this.nodesKey]: nodes.map((node) => docToJson(node)),
+      [this.nodesKey]: nodes.map((node) => docToJson(node, jsonSerializer)),
    };
    await this.cache.put(hash, val, this.collection);
  }
@@ -63,6 +67,8 @@ export class IngestionCache {
    if (!json || !json[this.nodesKey] || !Array.isArray(json[this.nodesKey])) {
      return undefined;
    }
-    return json[this.nodesKey].map((doc: any) => jsonToDoc(doc));
+    return json[this.nodesKey].map((doc: any) =>
+      jsonToDoc(doc, jsonSerializer),
+    );
  }
 }
@@ -29,7 +29,7 @@ export class KVDocumentStore extends BaseDocumentStore {
    for (const key in jsonDict) {
      const value = jsonDict[key];
      if (isValidDocJson(value)) {
-        docs[key] = jsonToDoc(value);
+        docs[key] = jsonToDoc(value, this.serializer);
      } else {
        console.warn(`Invalid JSON for docId ${key}`);
      }
@@ -52,7 +52,7 @@ export class KVDocumentStore extends BaseDocumentStore {
        );
      }
      const nodeKey = doc.id_;
-      const data = docToJson(doc);
+      const data = docToJson(doc, this.serializer);
      await this.kvstore.put(nodeKey, data, this.nodeCollection);
      const metadata: DocMetaData = { docHash: doc.hash };

@@ -94,7 +94,7 @@ export class KVDocumentStore extends BaseDocumentStore {
    if (!isValidDocJson(json)) {
      throw new Error(`Invalid JSON for docId ${docId}`);
    }
-    return jsonToDoc(json);
+    return jsonToDoc(json, this.serializer);
  }

  async getRefDocInfo(refDocId: string): Promise<RefDocInfo | undefined> {
@@ -4,6 +4,7 @@ import {
  type PostgresKVStoreConfig,
 } from "../kvStore/PostgresKVStore.js";
 import { KVDocumentStore } from "./KVDocumentStore.js";
+import { noneSerializer } from "./utils.js";

 const DEFAULT_TABLE_NAME = "llamaindex_doc_store";

@@ -12,6 +13,8 @@ export type PostgresDocumentStoreConfig = PostgresKVStoreConfig & {
 };

 export class PostgresDocumentStore extends KVDocumentStore {
+  serializer = noneSerializer;
+
  constructor(config?: PostgresDocumentStoreConfig) {
    const kvStore = new PostgresKVStore({
      schemaName: config?.schemaName,
@@ -3,6 +3,7 @@ import {
  DEFAULT_PERSIST_DIR,
 } from "@llamaindex/core/global";
 import { BaseNode } from "@llamaindex/core/schema";
+import { jsonSerializer, type Serializer } from "./utils.js";

 const defaultPersistPath = `${DEFAULT_PERSIST_DIR}/${DEFAULT_DOC_STORE_PERSIST_FILENAME}`;

@@ -12,6 +13,8 @@ export interface RefDocInfo {
 }

 export abstract class BaseDocumentStore {
+  serializer: Serializer<any> = jsonSerializer;
+
  // Save/load
  persist(persistPath: string = defaultPersistPath): void {
    // Persist the docstore to a file.
@@ -4,12 +4,35 @@ import { Document, ObjectType, TextNode } from "@llamaindex/core/schema";
 const TYPE_KEY = "__type__";
 const DATA_KEY = "__data__";

-type DocJson = {
-  [TYPE_KEY]: ObjectType;
-  [DATA_KEY]: string;
+export interface Serializer<T> {
+  toPersistence(data: Record<string, unknown>): T;
+  fromPersistence(data: T): Record<string, unknown>;
+}
+
+export const jsonSerializer: Serializer<string> = {
+  toPersistence(data) {
+    return JSON.stringify(data);
+  },
+  fromPersistence(data) {
+    return JSON.parse(data);
+  },
 };

-export function isValidDocJson(docJson: any): docJson is DocJson {
+export const noneSerializer: Serializer<Record<string, unknown>> = {
+  toPersistence(data) {
+    return data;
+  },
+  fromPersistence(data) {
+    return data;
+  },
+};
+
+type DocJson<Data> = {
+  [TYPE_KEY]: ObjectType;
+  [DATA_KEY]: Data;
+};
+
+export function isValidDocJson(docJson: any): docJson is DocJson<unknown> {
  return (
    typeof docJson === "object" &&
    docJson !== null &&
@@ -18,16 +41,22 @@ export function isValidDocJson(docJson: any): docJson is DocJson {
  );
 }

-export function docToJson(doc: BaseNode): DocJson {
+export function docToJson(
+  doc: BaseNode,
+  serializer: Serializer<unknown>,
+): DocJson<unknown> {
  return {
-    [DATA_KEY]: JSON.stringify(doc.toJSON()),
+    [DATA_KEY]: serializer.toPersistence(doc.toJSON()),
    [TYPE_KEY]: doc.type,
  };
 }

-export function jsonToDoc(docDict: DocJson): BaseNode {
+export function jsonToDoc<Data>(
+  docDict: DocJson<Data>,
+  serializer: Serializer<Data>,
+): BaseNode {
  const docType = docDict[TYPE_KEY];
-  const dataDict = JSON.parse(docDict[DATA_KEY]);
+  const dataDict = serializer.fromPersistence(docDict[DATA_KEY]) as any;
  let doc: BaseNode;

  if (docType === ObjectType.DOCUMENT) {
Author	SHA1	Message	Date
github-actions[bot]	22ae8d0166	Release 0.6.7 (#1244 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-09-23 13:25:02 -07:00
Goran	23bcc379a8	fix: add `serializer` in doc store (#1243 ) Co-authored-by: Alex Yang <himself65@outlook.com>	2024-09-23 13:11:51 -07:00