docs(changeset): feat: experimental package + json query engine

.changeset/clever-doors-invite.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat: experimental package + json query engine

package.json

@@ -14,7 +14,7 @@
     "release": "pnpm run build:release && changeset publish",
     "new-llamaindex": "pnpm run build:release && changeset version --ignore create-llama",
     "new-create-llama": "pnpm run build:release && changeset version --ignore llamaindex --ignore @llamaindex/core-test",
-    "new-snapshots": "pnpm run build:release && changeset version --snapshot"
+    "new-experimental": "pnpm run build:release && changeset version --ignore create-llama"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",

packages/core/src/agent/types.ts

@@ -3,6 +3,7 @@ import type {
   ChatEngineAgentParams,
   StreamingAgentChatResponse,
 } from "../engines/chat/index.js";
+
 import type { QueryEngineParamsNonStreaming } from "../types.js";
 
 export interface AgentWorker {

packages/core/src/index.ts

@@ -30,3 +30,4 @@ export * from "./selectors/index.js";
 export * from "./storage/index.js";
 export * from "./synthesizers/index.js";
 export * from "./tools/index.js";
+export * from "./types.js";

packages/experimental/.cjs.swcrc

@@ -0,0 +1,12 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    },
+    "target": "esnext"
+  },
+  "module": {
+    "type": "commonjs",
+    "ignoreDynamic": true
+  }
+}

packages/experimental/.swcrc

@@ -0,0 +1,8 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    },
+    "target": "esnext"
+  }
+}

packages/experimental/CHANGELOG.md

@@ -0,0 +1 @@
+# @llamaindex/experimental

packages/experimental/examples/jsonQueryEngine.ts

@@ -0,0 +1,119 @@
+import { JSONQueryEngine } from "@llamaindex/experimental";
+
+import { OpenAI, serviceContextFromDefaults } from "llamaindex";
+
+const jsonValue = {
+  blogPosts: [
+    {
+      id: 1,
+      title: "First blog post",
+      content: "This is my first blog post",
+    },
+    {
+      id: 2,
+      title: "Second blog post",
+      content: "This is my second blog post",
+    },
+  ],
+  comments: [
+    {
+      id: 1,
+      content: "Nice post!",
+      username: "jerry",
+      blogPostId: 1,
+    },
+    {
+      id: 2,
+      content: "Interesting thoughts",
+      username: "simon",
+      blogPostId: 2,
+    },
+    {
+      id: 3,
+      content: "Loved reading this!",
+      username: "simon",
+      blogPostId: 2,
+    },
+  ],
+};
+
+const jsonSchema = {
+  type: "object",
+  properties: {
+    blogPosts: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          id: {
+            type: "number",
+          },
+          title: {
+            type: "string",
+          },
+          content: {
+            type: "string",
+          },
+        },
+        required: ["id", "title", "content"],
+      },
+    },
+    comments: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          id: {
+            type: "number",
+          },
+          content: {
+            type: "string",
+          },
+          username: {
+            type: "string",
+          },
+          blogPostId: {
+            type: "number",
+          },
+        },
+        required: ["id", "content", "username", "blogPostId"],
+      },
+    },
+  },
+  required: ["blogPosts", "comments"],
+};
+
+async function main() {
+  const llm = new OpenAI({ model: "gpt-4" });
+
+  const serviceContext = serviceContextFromDefaults({
+    llm,
+  });
+
+  const jsonQueryEngine = new JSONQueryEngine({
+    jsonValue,
+    jsonSchema,
+    serviceContext,
+  });
+
+  const rawQueryEngine = new JSONQueryEngine({
+    jsonValue,
+    jsonSchema,
+    serviceContext,
+    synthesizeResponse: false,
+  });
+
+  const response = await jsonQueryEngine.query({
+    query: "give to me the comment with id 1",
+  });
+
+  const rawResponse = await rawQueryEngine.query({
+    query: "give me all simon comments",
+  });
+
+  console.log({ response });
+
+  console.log({ rawResponse });
+}
+
+main();

packages/experimental/examples/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "examples",
+  "private": true,
+  "version": "0.0.3",
+  "dependencies": {
+    "@llamaindex/experimental": "latest",
+    "llamaindex": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "^18.19.10",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3"
+  },
+  "scripts": {
+    "lint": "eslint ."
+  }
+}

packages/experimental/jsr.json

@@ -0,0 +1,8 @@
+{
+  "name": "@llamaindex/experimental",
+  "version": "0.0.5",
+  "exports": {
+    ".": "./src/index.ts",
+    "./type": "./src/type.ts"
+  }
+}

packages/experimental/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@llamaindex/experimental",
+  "description": "Experimental package for LlamaIndexTS",
+  "version": "0.0.2",
+  "type": "module",
+  "types": "dist/type/index.d.ts",
+  "main": "dist/cjs/index.js",
+  "exports": {
+    ".": {
+      "workerd": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/index.polyfill.js"
+      },
+      "edge-light": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/index.polyfill.js"
+      },
+      "import": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
+    },
+    "./*": {
+      "import": {
+        "types": "./dist/type/*.d.ts",
+        "default": "./dist/*.js"
+      },
+      "require": {
+        "types": "./dist/type/*.d.ts",
+        "default": "./dist/cjs/*.js"
+      }
+    }
+  },
+  "files": [
+    "dist",
+    "CHANGELOG.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/experimental"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "build": "rm -rf ./dist && pnpm run build:esm && pnpm run build:cjs && pnpm run build:type",
+    "build:esm": "swc src -d dist --strip-leading-paths --config-file ../../.swcrc",
+    "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file ../../.cjs.swcrc",
+    "build:type": "tsc -p tsconfig.json",
+    "postbuild": "node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"",
+    "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\""
+  },
+  "devDependencies": {
+    "@aws-crypto/sha256-js": "^5.2.0",
+    "@swc/cli": "^0.3.9",
+    "@swc/core": "^1.4.2",
+    "@types/jsonpath": "^0.2.4",
+    "concurrently": "^8.2.2",
+    "pathe": "^1.1.2"
+  },
+  "dependencies": {
+    "@types/lodash": "^4.14.202",
+    "@types/node": "^20.11.20",
+    "jsonpath": "^1.1.1",
+    "llamaindex": "workspace:*",
+    "lodash": "^4.17.21"
+  }
+}

packages/experimental/src/engines/query/JSONQueryEngine.ts

@@ -0,0 +1,209 @@
+import jsonpath from "jsonpath";
+
+import { Response } from "llamaindex";
+
+import { serviceContextFromDefaults, type ServiceContext } from "llamaindex";
+
+import type {
+  BaseQueryEngine,
+  QueryEngineParamsNonStreaming,
+  QueryEngineParamsStreaming,
+} from "llamaindex";
+
+import {
+  defaultJsonPathPrompt,
+  defaultResponseSynthesizePrompt,
+  type JSONPathPrompt,
+  type ResponseSynthesisPrompt,
+} from "./prompt.js";
+
+export type JSONSchemaType = Record<string, unknown>;
+
+function removeExtraQuotes(expr: string) {
+  let startIndex = 0;
+  let endIndex = expr.length;
+
+  // Trim the leading backticks and single quotes
+  while (
+    startIndex < endIndex &&
+    (expr[startIndex] === "`" || expr[startIndex] === "'")
+  ) {
+    startIndex++;
+  }
+
+  // Trim the trailing backticks and single quotes
+  while (
+    endIndex > startIndex &&
+    (expr[endIndex - 1] === "`" || expr[endIndex - 1] === "'")
+  ) {
+    endIndex--;
+  }
+
+  // Return the trimmed substring
+  return expr.substring(startIndex, endIndex);
+}
+
+export const defaultOutputProcessor = async ({
+  llmOutput,
+  jsonValue,
+}: {
+  llmOutput: string;
+  jsonValue: JSONSchemaType;
+}): Promise<Record<string, unknown>[]> => {
+  const expressions = llmOutput
+    .split(",")
+    .map((expr) => removeExtraQuotes(expr.trim()));
+
+  const results: Record<string, unknown>[] = [];
+
+  for (const expression of expressions) {
+    // get the key for example content from $.content
+    const key = expression.split(".").pop();
+
+    try {
+      const datums = jsonpath.query(jsonValue, expression);
+
+      if (!key) throw new Error(`Invalid JSON Path: ${expression}`);
+
+      for (const datum of datums) {
+        // in case there is a filter like [?(@.username=='simon')] without a key ie: $..comments[?(@.username=='simon').content]
+        if (key.includes("==")) {
+          results.push(datum);
+          continue;
+        }
+
+        results.push({
+          [key]: datum,
+        });
+      }
+    } catch (err) {
+      throw new Error(`Invalid JSON Path: ${expression}`);
+    }
+  }
+
+  return results;
+};
+
+type OutputProcessor = typeof defaultOutputProcessor;
+
+/**
+ * A JSON query engine that uses JSONPath to query a JSON object.
+ */
+export class JSONQueryEngine implements BaseQueryEngine {
+  jsonValue: JSONSchemaType;
+  jsonSchema: JSONSchemaType;
+  serviceContext: ServiceContext;
+  outputProcessor: OutputProcessor;
+  verbose: boolean;
+  jsonPathPrompt: JSONPathPrompt;
+  synthesizeResponse: boolean;
+  responseSynthesisPrompt: ResponseSynthesisPrompt;
+
+  constructor(init: {
+    jsonValue: JSONSchemaType;
+    jsonSchema: JSONSchemaType;
+    serviceContext?: ServiceContext;
+    jsonPathPrompt?: JSONPathPrompt;
+    outputProcessor?: OutputProcessor;
+    synthesizeResponse?: boolean;
+    responseSynthesisPrompt?: ResponseSynthesisPrompt;
+    verbose?: boolean;
+  }) {
+    this.jsonValue = init.jsonValue;
+    this.jsonSchema = init.jsonSchema;
+    this.serviceContext = init.serviceContext ?? serviceContextFromDefaults({});
+    this.jsonPathPrompt = init.jsonPathPrompt ?? defaultJsonPathPrompt;
+    this.outputProcessor = init.outputProcessor ?? defaultOutputProcessor;
+    this.verbose = init.verbose ?? false;
+    this.synthesizeResponse = init.synthesizeResponse ?? true;
+    this.responseSynthesisPrompt =
+      init.responseSynthesisPrompt ?? defaultResponseSynthesizePrompt;
+  }
+
+  getPrompts(): Record<string, unknown> {
+    return {
+      jsonPathPrompt: this.jsonPathPrompt,
+      responseSynthesisPrompt: this.responseSynthesisPrompt,
+    };
+  }
+
+  updatePrompts(prompts: {
+    jsonPathPrompt?: JSONPathPrompt;
+    responseSynthesisPrompt?: ResponseSynthesisPrompt;
+  }): void {
+    if (prompts.jsonPathPrompt) {
+      this.jsonPathPrompt = prompts.jsonPathPrompt;
+    }
+    if (prompts.responseSynthesisPrompt) {
+      this.responseSynthesisPrompt = prompts.responseSynthesisPrompt;
+    }
+  }
+
+  getPromptModules(): Record<string, unknown> {
+    return {};
+  }
+
+  getSchemaContext(): string {
+    return JSON.stringify(this.jsonSchema);
+  }
+
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { query, stream } = params;
+
+    if (stream) {
+      throw new Error("Streaming is not supported");
+    }
+
+    const schema = this.getSchemaContext();
+
+    const jsonPathResponseStr = await this.serviceContext.llm.complete({
+      prompt: this.jsonPathPrompt({ query, schema }),
+    });
+
+    if (this.verbose) {
+      console.log(
+        `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponseStr}\n\`\`\`\n`,
+      );
+    }
+
+    const jsonPathOutput = await this.outputProcessor({
+      llmOutput: jsonPathResponseStr.text,
+      jsonValue: this.jsonValue,
+    });
+
+    if (this.verbose) {
+      console.log(`> JSONPath Output: ${jsonPathOutput}\n`);
+    }
+
+    let responseStr;
+
+    if (this.synthesizeResponse) {
+      responseStr = await this.serviceContext.llm.complete({
+        prompt: this.responseSynthesisPrompt({
+          query,
+          jsonSchema: schema,
+          jsonPath: jsonPathResponseStr.text,
+          jsonPathValue: JSON.stringify(jsonPathOutput),
+        }),
+      });
+
+      responseStr = responseStr.text;
+    } else {
+      responseStr = JSON.stringify(jsonPathOutput);
+    }
+
+    const responseMetadata = {
+      jsonPathResponseStr,
+    };
+
+    const response = new Response(responseStr, []);
+
+    response.metadata = responseMetadata;
+
+    return response;
+  }
+}

packages/experimental/src/engines/query/index.ts

@@ -0,0 +1 @@
+export * from "./JSONQueryEngine.js";

packages/experimental/src/engines/query/prompt.ts

@@ -0,0 +1,36 @@
+export const defaultJsonPathPrompt = ({
+  query,
+  schema,
+}: {
+  query: string;
+  schema: string;
+}) => `
+We have provided a JSON schema below:
+${schema}
+Given a task, respond with a JSON Path query that can retrieve data from a JSON value that matches the schema.
+Task: ${query}
+JSONPath: 
+`;
+
+export type JSONPathPrompt = typeof defaultJsonPathPrompt;
+
+export const defaultResponseSynthesizePrompt = ({
+  query,
+  jsonSchema,
+  jsonPath,
+  jsonPathValue,
+}: {
+  query: string;
+  jsonSchema: string;
+  jsonPath: string;
+  jsonPathValue: string;
+}) => `
+Given a query, synthesize a response to satisfy the query using the JSON results. Only include details that are relevant to the query. If you don't know the answer, then say that.
+JSON Schema: ${jsonSchema}
+JSON Path: ${jsonPath}
+Value at path: ${jsonPathValue}
+Query: ${query}
+Response: 
+`;
+
+export type ResponseSynthesisPrompt = typeof defaultResponseSynthesizePrompt;

packages/experimental/src/index.ts

@@ -0,0 +1 @@
+export * from "./engines/query/index.js";

packages/experimental/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist/type",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo",
+    "emitDeclarationOnly": true,
+    "module": "node16",
+    "moduleResolution": "node16",
+    "types": ["node"]
+  },
+  "include": ["./src"],
+  "exclude": ["node_modules"],
+  "references": [
+    {
+      "path": "../core/tsconfig.json"
+    }
+  ]
+}

pnpm-lock.yaml

@@ -421,6 +421,43 @@ importers:
         specifier: ^13.5.6
         version: 13.5.6(react-dom@18.2.0)(react@18.2.0)
 
+  packages/experimental:
+    dependencies:
+      '@types/lodash':
+        specifier: ^4.14.202
+        version: 4.14.202
+      '@types/node':
+        specifier: ^20.11.20
+        version: 20.11.20
+      jsonpath:
+        specifier: ^1.1.1
+        version: 1.1.1
+      llamaindex:
+        specifier: workspace:*
+        version: link:../core
+      lodash:
+        specifier: ^4.17.21
+        version: 4.17.21
+    devDependencies:
+      '@aws-crypto/sha256-js':
+        specifier: ^5.2.0
+        version: 5.2.0
+      '@swc/cli':
+        specifier: ^0.3.9
+        version: 0.3.9(@swc/core@1.4.2)
+      '@swc/core':
+        specifier: ^1.4.2
+        version: 1.4.2
+      '@types/jsonpath':
+        specifier: ^0.2.4
+        version: 0.2.4
+      concurrently:
+        specifier: ^8.2.2
+        version: 8.2.2
+      pathe:
+        specifier: ^1.1.2
+        version: 1.1.2
+
   packages/tsconfig: {}
 
 packages:
@@ -2630,7 +2667,7 @@ packages:
     peerDependencies:
       react: '*'
     dependencies:
-      '@types/react': 18.2.48
+      '@types/react': 18.2.61
       prop-types: 15.8.1
       react: 18.2.0
 
@@ -4139,19 +4176,19 @@ packages:
     resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==}
     dependencies:
       '@types/connect': 3.4.38
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
 
   /@types/bonjour@3.5.13:
     resolution: {integrity: sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
 
   /@types/cacheable-request@6.0.3:
     resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==}
     dependencies:
       '@types/http-cache-semantics': 4.0.4
       '@types/keyv': 3.1.4
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
       '@types/responselike': 1.0.3
     dev: true
 
@@ -4163,7 +4200,7 @@ packages:
     resolution: {integrity: sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==}
     dependencies:
       '@types/express-serve-static-core': 4.17.42
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
 
   /@types/connect@3.4.38:
     resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==}
@@ -4173,7 +4210,7 @@ packages:
   /@types/cross-spawn@6.0.0:
     resolution: {integrity: sha512-evp2ZGsFw9YKprDbg8ySgC9NA15g3YgiI8ANkGmKKvvi0P2aDGYLPxQIC5qfeKNUOe3TjABVGuah6omPRpIYhg==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
     dev: true
 
   /@types/debug@4.1.12:
@@ -4204,7 +4241,7 @@ packages:
   /@types/express-serve-static-core@4.17.42:
     resolution: {integrity: sha512-ckM3jm2bf/MfB3+spLPWYPUH573plBFwpOhqQ2WottxYV85j1HQFlxmnTq57X1yHY9awZPig06hL/cLMgNWHIQ==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
       '@types/qs': 6.9.12
       '@types/range-parser': 1.2.7
       '@types/send': 0.17.4
@@ -4241,7 +4278,7 @@ packages:
   /@types/http-proxy@1.17.14:
     resolution: {integrity: sha512-SSrD0c1OQzlFX7pGu1eXxSEjemej64aaNPRhhVYUGqXh0BtldAAx37MG8btcumvpgKyZp1F5Gn3JkktdxiFv6w==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
 
   /@types/istanbul-lib-coverage@2.0.6:
     resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==}
@@ -4265,10 +4302,14 @@ packages:
   /@types/json5@0.0.29:
     resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==}
 
+  /@types/jsonpath@0.2.4:
+    resolution: {integrity: sha512-K3hxB8Blw0qgW6ExKgMbXQv2UPZBoE2GqLpVY+yr7nMD2Pq86lsuIzyAaiQ7eMqFL5B6di6pxSkogLJEyEHoGA==}
+    dev: true
+
   /@types/keyv@3.1.4:
     resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
     dev: true
 
   /@types/lodash-es@4.17.12:
@@ -4309,14 +4350,14 @@ packages:
   /@types/node-fetch@2.6.9:
     resolution: {integrity: sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
       form-data: 4.0.0
     dev: false
 
   /@types/node-forge@1.3.11:
     resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
 
   /@types/node@12.20.55:
     resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==}
@@ -4344,11 +4385,6 @@ packages:
       undici-types: 5.26.5
     dev: true
 
-  /@types/node@20.11.17:
-    resolution: {integrity: sha512-QmgQZGWu1Yw9TDyAP9ZzpFJKynYNeOvwMJmaxABfieQoVoiVOS6MN1WSpqpRcbeA5+RW82kraAVxCCJg+780Qw==}
-    dependencies:
-      undici-types: 5.26.5
-
   /@types/node@20.11.20:
     resolution: {integrity: sha512-7/rR21OS+fq8IyHTgtLkDK949uzsa6n8BkziAKtPVpugIkO6D+/ooXMvzXxDnZrmtXVfjb1bKQafYpb8s89LOg==}
     dependencies:
@@ -4367,7 +4403,7 @@ packages:
   /@types/papaparse@5.3.14:
     resolution: {integrity: sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
     dev: false
 
   /@types/parse-json@4.0.2:
@@ -4376,7 +4412,7 @@ packages:
   /@types/pg@8.11.0:
     resolution: {integrity: sha512-sDAlRiBNthGjNFfvt0k6mtotoVYVQ63pA8R4EMWka7crawSR60waVYR0HAgmPRs/e2YaeJTD/43OoZ3PFw80pw==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
       pg-protocol: 1.6.0
       pg-types: 4.0.1
     dev: false
@@ -4448,7 +4484,7 @@ packages:
   /@types/responselike@1.0.3:
     resolution: {integrity: sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
     dev: true
 
   /@types/retry@0.12.0:
@@ -4461,7 +4497,7 @@ packages:
   /@types/sax@1.2.7:
     resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
     dev: true
 
   /@types/scheduler@0.16.4:
@@ -4490,17 +4526,17 @@ packages:
     dependencies:
       '@types/http-errors': 2.0.4
       '@types/mime': 3.0.4
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
 
   /@types/sockjs@0.3.36:
     resolution: {integrity: sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
 
   /@types/tar@6.1.5:
     resolution: {integrity: sha512-qm2I/RlZij5RofuY7vohTpYNaYcrSQlN2MyjucQc7ZweDwaEWkdN/EeNh6e9zjK6uEm6PwjdMXkcj05BxZdX1Q==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
       minipass: 4.2.8
     dev: true
 
@@ -4531,7 +4567,7 @@ packages:
   /@types/ws@8.5.10:
     resolution: {integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
 
   /@types/yargs-parser@21.0.3:
     resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==}
@@ -7245,6 +7281,19 @@ packages:
     resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==}
     engines: {node: '>=12'}
 
+  /escodegen@1.14.3:
+    resolution: {integrity: sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==}
+    engines: {node: '>=4.0'}
+    hasBin: true
+    dependencies:
+      esprima: 4.0.1
+      estraverse: 4.3.0
+      esutils: 2.0.3
+      optionator: 0.8.3
+    optionalDependencies:
+      source-map: 0.6.1
+    dev: false
+
   /escodegen@2.1.0:
     resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==}
     engines: {node: '>=6.0'}
@@ -7613,6 +7662,12 @@ packages:
       acorn-jsx: 5.3.2(acorn@8.11.3)
       eslint-visitor-keys: 3.4.3
 
+  /esprima@1.2.2:
+    resolution: {integrity: sha512-+JpPZam9w5DuJ3Q67SqsMGtiHKENSMRVoxvArfJZK01/BfLEObtZ6orJa/MtoGNR/rfMgp5837T41PAmTwAv/A==}
+    engines: {node: '>=0.4.0'}
+    hasBin: true
+    dev: false
+
   /esprima@4.0.1:
     resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==}
     engines: {node: '>=4'}
@@ -7695,7 +7750,7 @@ packages:
     resolution: {integrity: sha512-EzV94NYKoO09GLXGjXj9JIlXijVck4ONSr5wiCWDvhsvj5jxSrzTmRU/9C1DyB6uToszLs8aifA6NQ7lEQdvFw==}
     engines: {node: '>= 0.8'}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
       require-like: 0.1.2
 
   /event-target-shim@5.0.1:
@@ -9453,7 +9508,7 @@ packages:
     engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
     dependencies:
       '@jest/types': 29.6.3
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
       chalk: 4.1.2
       ci-info: 3.9.0
       graceful-fs: 4.2.11
@@ -9463,7 +9518,7 @@ packages:
     resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==}
     engines: {node: '>= 10.13.0'}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
       merge-stream: 2.0.0
       supports-color: 8.1.1
 
@@ -9471,7 +9526,7 @@ packages:
     resolution: {integrity: sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==}
     engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
     dependencies:
-      '@types/node': 20.11.17
+      '@types/node': 20.11.20
       jest-util: 29.7.0
       merge-stream: 2.0.0
       supports-color: 8.1.1
@@ -9585,6 +9640,14 @@ packages:
     optionalDependencies:
       graceful-fs: 4.2.11
 
+  /jsonpath@1.1.1:
+    resolution: {integrity: sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==}
+    dependencies:
+      esprima: 1.2.2
+      static-eval: 2.0.2
+      underscore: 1.12.1
+    dev: false
+
   /jsx-ast-utils@3.3.3:
     resolution: {integrity: sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==}
     engines: {node: '>=4.0'}
@@ -9661,6 +9724,14 @@ packages:
     resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==}
     engines: {node: '>=6'}
 
+  /levn@0.3.0:
+    resolution: {integrity: sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==}
+    engines: {node: '>= 0.8.0'}
+    dependencies:
+      prelude-ls: 1.1.2
+      type-check: 0.3.2
+    dev: false
+
   /levn@0.4.1:
     resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==}
     engines: {node: '>= 0.8.0'}
@@ -11261,6 +11332,18 @@ packages:
     resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==}
     dev: false
 
+  /optionator@0.8.3:
+    resolution: {integrity: sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==}
+    engines: {node: '>= 0.8.0'}
+    dependencies:
+      deep-is: 0.1.4
+      fast-levenshtein: 2.0.6
+      levn: 0.3.0
+      prelude-ls: 1.1.2
+      type-check: 0.3.2
+      word-wrap: 1.2.5
+    dev: false
+
   /optionator@0.9.3:
     resolution: {integrity: sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==}
     engines: {node: '>= 0.8.0'}
@@ -12285,6 +12368,11 @@ packages:
       which-pm: 2.0.0
     dev: true
 
+  /prelude-ls@1.1.2:
+    resolution: {integrity: sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==}
+    engines: {node: '>= 0.8.0'}
+    dev: false
+
   /prelude-ls@1.2.1:
     resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==}
     engines: {node: '>= 0.8.0'}
@@ -13751,6 +13839,12 @@ packages:
     resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==}
     dev: true
 
+  /static-eval@2.0.2:
+    resolution: {integrity: sha512-N/D219Hcr2bPjLxPiV+TQE++Tsmrady7TqAJugLy7Xk1EumfDWS/f5dtBbkRCGE7wKKXuYockQoj8Rm2/pVKyg==}
+    dependencies:
+      escodegen: 1.14.3
+    dev: false
+
   /statuses@1.5.0:
     resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==}
     engines: {node: '>= 0.6'}
@@ -14552,6 +14646,13 @@ packages:
       turbo-windows-arm64: 1.12.3
     dev: true
 
+  /type-check@0.3.2:
+    resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==}
+    engines: {node: '>= 0.8.0'}
+    dependencies:
+      prelude-ls: 1.1.2
+    dev: false
+
   /type-check@0.4.0:
     resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==}
     engines: {node: '>= 0.8.0'}
@@ -14706,6 +14807,10 @@ packages:
       has-symbols: 1.0.3
       which-boxed-primitive: 1.0.2
 
+  /underscore@1.12.1:
+    resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==}
+    dev: false
+
   /underscore@1.13.6:
     resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}
     dev: false
@@ -15474,6 +15579,11 @@ packages:
       winston-transport: 4.6.0
     dev: false
 
+  /word-wrap@1.2.5:
+    resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==}
+    engines: {node: '>=0.10.0'}
+    dev: false
+
   /wordwrap@1.0.0:
     resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==}
     dev: true

tsconfig.json

@@ -16,7 +16,9 @@
       "llamaindex": ["./packages/core/src/index.ts"],
       "llamaindex/*": ["./packages/core/src/*.ts"],
       "@llamaindex/env": ["./packages/env/src/index.ts"],
-      "@llamaindex/env/*": ["./packages/env/src/*.ts"]
+      "@llamaindex/env/*": ["./packages/env/src/*.ts"],
+      "@llamaindex/experimental": ["./packages/experimental/src/index.ts"],
+      "@llamaindex/experimental/*": ["./packages/experimental/src/*.ts"]
     }
   },
   "files": [],
@@ -44,6 +46,9 @@
     },
     {
       "path": "./examples/readers"
+    },
+    {
+      "path": "./packages/experimental/tsconfig.json"
     }
   ]
 }