Release 0.6.20 (#1325)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

apps/docs/CHANGELOG.md

@@ -1,5 +1,22 @@
 # docs
 
+## 0.0.89
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 0.0.88
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
 ## 0.0.87
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.87",
+  "version": "0.0.89",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

packages/autotool/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/autotool
 
+## 3.0.20
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 3.0.19
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
 ## 3.0.18
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+  - @llamaindex/autotool@3.0.20
+
+## 0.0.28
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+  - @llamaindex/autotool@3.0.19
+
 ## 0.0.27
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.27"
+  "version": "0.0.29"
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.73
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+  - @llamaindex/autotool@3.0.20
+
+## 0.1.72
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+  - @llamaindex/autotool@3.0.19
+
 ## 0.1.71
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.71",
+  "version": "0.1.73",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "3.0.18",
+  "version": "3.0.20",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/core/tests/node-parser/text-splitter.test.ts

@@ -2,6 +2,7 @@ import {
   SentenceSplitter,
   splitBySentenceTokenizer,
 } from "@llamaindex/core/node-parser";
+import { Document } from "@llamaindex/core/schema";
 import { describe, expect, test } from "vitest";
 
 describe("sentence splitter", () => {
@@ -115,4 +116,26 @@ describe("sentence splitter", () => {
     const split = splitBySentenceTokenizer();
     expect(split(text)).toEqual([text]);
   });
+
+  test("split nodes with UUID IDs and correct relationships", () => {
+    const UUID_REGEX =
+      /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+    const sentenceSplitter = new SentenceSplitter();
+    const docId = "test-doc-id";
+    const doc = new Document({
+      id_: docId,
+      text: "This is a test sentence. This is another test sentence.",
+    });
+    const nodes = sentenceSplitter.getNodesFromDocuments([doc]);
+    nodes.forEach((node) => {
+      // test node id should match uuid regex
+      expect(node.id_).toMatch(UUID_REGEX);
+
+      // test source reference to the doc ID
+      const source = node.relationships?.SOURCE;
+      expect(source).toBeDefined();
+      expect(source).toHaveProperty("nodeId");
+      expect((source as { nodeId: string }).nodeId).toEqual(docId);
+    });
+  });
 });

packages/experimental/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/experimental
 
+## 0.0.98
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 0.0.97
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
 ## 0.0.96
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.96",
+  "version": "0.0.98",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,23 @@
 # llamaindex
 
+## 0.6.20
+
+### Patch Changes
+
+- 6a9a7b1: fix: take init api key into account
+- Updated dependencies [6a9a7b1]
+  - @llamaindex/openai@0.1.16
+  - @llamaindex/groq@0.0.15
+
+## 0.6.19
+
+### Patch Changes
+
+- 62cba52: Add ensureIndex function to LlamaCloudIndex
+- d265e96: fix: ignore resolving unpdf for nextjs
+- d30bbf7: Convert undefined values to null in LlamaCloud filters
+- 53fd00a: Fix getPipelineId in LlamaCloudIndex
+
 ## 0.6.18
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.82
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 0.0.81
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
 ## 0.0.80
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.80",
+  "version": "0.0.82",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/next-agent-test
 
+## 0.1.82
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 0.1.81
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
 ## 0.1.80
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.80",
+  "version": "0.1.82",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,22 @@
 # test-edge-runtime
 
+## 0.1.81
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 0.1.80
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
 ## 0.1.79
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.79",
+  "version": "0.1.81",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.63
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 0.0.62
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
 ## 0.0.61
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.61",
+  "version": "0.0.63",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.82
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - llamaindex@0.6.20
+
+## 0.0.81
+
+### Patch Changes
+
+- Updated dependencies [62cba52]
+- Updated dependencies [d265e96]
+- Updated dependencies [d30bbf7]
+- Updated dependencies [53fd00a]
+  - llamaindex@0.6.19
+
 ## 0.0.80
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.80",
+  "version": "0.0.82",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.6.18",
+  "version": "0.6.20",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/cloud/LLamaCloudFileService.ts

@@ -85,7 +85,7 @@ export class LLamaCloudFileService {
       await new Promise((resolve) => setTimeout(resolve, 100)); // Sleep for 100ms
     }
     throw new Error(
-      `File processing did not complete after ${maxAttempts} attempts.`,
+      `File processing did not complete after ${maxAttempts} attempts. Check your LlamaCloud index at https://cloud.llamaindex.ai/project/${projectId}/deploy/${pipelineId} for more details.`,
     );
   }
 

packages/llamaindex/src/cloud/LlamaCloudIndex.ts

@@ -1,19 +1,21 @@
 import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
 import type { BaseSynthesizer } from "@llamaindex/core/response-synthesizers";
-import type { Document, TransformComponent } from "@llamaindex/core/schema";
+import type { Document } from "@llamaindex/core/schema";
 import { RetrieverQueryEngine } from "../engines/query/RetrieverQueryEngine.js";
 import type { BaseNodePostprocessor } from "../postprocessors/types.js";
 import type { CloudRetrieveParams } from "./LlamaCloudRetriever.js";
 import { LlamaCloudRetriever } from "./LlamaCloudRetriever.js";
-import { getPipelineCreate } from "./config.js";
 import type { CloudConstructorParams } from "./type.js";
-import { getAppBaseUrl, getProjectId, initService } from "./utils.js";
+import {
+  getAppBaseUrl,
+  getPipelineId,
+  getProjectId,
+  initService,
+} from "./utils.js";
 
-import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api";
-import { SentenceSplitter } from "@llamaindex/core/node-parser";
+import { PipelinesService, type PipelineCreate } from "@llamaindex/cloud/api";
 import type { BaseRetriever } from "@llamaindex/core/retriever";
 import { getEnv } from "@llamaindex/env";
-import { OpenAIEmbedding } from "@llamaindex/openai";
 import { Settings } from "../Settings.js";
 
 export class LlamaCloudIndex {
@@ -28,10 +30,7 @@ export class LlamaCloudIndex {
     verbose = Settings.debug,
     raiseOnError = false,
   ): Promise<void> {
-    const pipelineId = await this.getPipelineId(
-      this.params.name,
-      this.params.projectName,
-    );
+    const pipelineId = await this.getPipelineId();
 
     if (verbose) {
       console.log("Waiting for pipeline ingestion: ");
@@ -78,10 +77,7 @@ export class LlamaCloudIndex {
     verbose = Settings.debug,
     raiseOnError = false,
   ): Promise<void> {
-    const pipelineId = await this.getPipelineId(
-      this.params.name,
-      this.params.projectName,
-    );
+    const pipelineId = await this.getPipelineId();
 
     if (verbose) {
       console.log("Loading data: ");
@@ -143,17 +139,13 @@ export class LlamaCloudIndex {
   public async getPipelineId(
     name?: string,
     projectName?: string,
+    organizationId?: string,
   ): Promise<string> {
-    const { data: pipelines } =
-      await PipelinesService.searchPipelinesApiV1PipelinesGet({
-        path: {
-          project_id: await this.getProjectId(projectName),
-          project_name: name ?? this.params.name,
-        },
-        throwOnError: true,
-      });
-
-    return pipelines[0]!.id;
+    return await getPipelineId(
+      name ?? this.params.name,
+      projectName ?? this.params.projectName,
+      organizationId ?? this.params.organizationId,
+    );
   }
 
   public async getProjectId(
@@ -166,75 +158,42 @@ export class LlamaCloudIndex {
     );
   }
 
+  /**
+   * Adds documents to the given index parameters. If the index does not exist, it will be created.
+   *
+   * @param params - An object containing the following properties:
+   *   - documents: An array of Document objects to be added to the index.
+   *   - verbose: Optional boolean to enable verbose logging.
+   *   - Additional properties from CloudConstructorParams.
+   * @returns A Promise that resolves to a new LlamaCloudIndex instance.
+   */
   static async fromDocuments(
     params: {
       documents: Document[];
-      transformations?: TransformComponent[];
       verbose?: boolean;
     } & CloudConstructorParams,
+    config?: {
+      embedding: PipelineCreate["embedding_config"];
+      transform: PipelineCreate["transform_config"];
+    },
   ): Promise<LlamaCloudIndex> {
-    initService(params);
-    const defaultTransformations: TransformComponent[] = [
-      new SentenceSplitter(),
-      new OpenAIEmbedding({
-        apiKey: getEnv("OPENAI_API_KEY"),
-      }),
-    ];
+    const index = new LlamaCloudIndex({ ...params });
+    await index.ensureIndex({ ...config, verbose: params.verbose ?? false });
+    await index.addDocuments(params.documents, params.verbose);
+    return index;
+  }
+
+  async addDocuments(documents: Document[], verbose?: boolean): Promise<void> {
     const apiUrl = getAppBaseUrl();
-
-    const pipelineCreateParams = await getPipelineCreate({
-      pipelineName: params.name,
-      pipelineType: "MANAGED",
-      inputNodes: params.documents,
-      transformations: params.transformations ?? defaultTransformations,
-    });
-
-    const { data: project } =
-      await ProjectsService.upsertProjectApiV1ProjectsPut({
-        path: {
-          organization_id: params.organizationId,
-        },
-        body: {
-          name: params.projectName ?? "default",
-        },
-        throwOnError: true,
-      });
-
-    if (!project.id) {
-      throw new Error("Project ID should be defined");
-    }
-
-    const { data: pipeline } =
-      await PipelinesService.upsertPipelineApiV1PipelinesPut({
-        path: {
-          project_id: project.id,
-        },
-        body: pipelineCreateParams.configured_transformations
-          ? {
-              name: params.name,
-              configured_transformations:
-                pipelineCreateParams.configured_transformations,
-            }
-          : {
-              name: params.name,
-            },
-        throwOnError: true,
-      });
-
-    if (!pipeline.id) {
-      throw new Error("Pipeline ID must be defined");
-    }
-
-    if (params.verbose) {
-      console.log(`Created pipeline ${pipeline.id} with name ${params.name}`);
-    }
+    const projectId = await this.getProjectId();
+    const pipelineId = await this.getPipelineId();
 
     await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
       {
         path: {
-          pipeline_id: pipeline.id,
+          pipeline_id: pipelineId,
         },
-        body: params.documents.map((doc) => ({
+        body: documents.map((doc) => ({
           metadata: doc.metadata,
           text: doc.text,
           excluded_embed_metadata_keys: doc.excludedEmbedMetadataKeys,
@@ -248,7 +207,7 @@ export class LlamaCloudIndex {
       const { data: pipelineStatus } =
         await PipelinesService.getPipelineStatusApiV1PipelinesPipelineIdStatusGet(
           {
-            path: { pipeline_id: pipeline.id },
+            path: { pipeline_id: pipelineId },
             throwOnError: true,
           },
         );
@@ -262,32 +221,30 @@ export class LlamaCloudIndex {
 
       if (pipelineStatus.status === "ERROR") {
         console.error(
-          `Some documents failed to ingest, check your pipeline logs at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
+          `Some documents failed to ingest, check your pipeline logs at ${apiUrl}/project/${projectId}/deploy/${pipelineId}`,
         );
         throw new Error("Some documents failed to ingest");
       }
 
       if (pipelineStatus.status === "PARTIAL_SUCCESS") {
         console.info(
-          `Documents ingestion partially succeeded, to check a more complete status check your pipeline at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
+          `Documents ingestion partially succeeded, to check a more complete status check your pipeline at ${apiUrl}/project/${projectId}/deploy/${pipelineId}`,
         );
         break;
       }
 
-      if (params.verbose) {
+      if (verbose) {
         process.stdout.write(".");
       }
 
       await new Promise((resolve) => setTimeout(resolve, 1000));
     }
 
-    if (params.verbose) {
+    if (verbose) {
       console.info(
-        `Ingestion completed, find your index at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
+        `Ingestion completed, find your index at ${apiUrl}/project/${projectId}/deploy/${pipelineId}`,
       );
     }
-
-    return new LlamaCloudIndex({ ...params });
   }
 
   asRetriever(params: CloudRetrieveParams = {}): BaseRetriever {
@@ -313,14 +270,7 @@ export class LlamaCloudIndex {
   }
 
   async insert(document: Document) {
-    const pipelineId = await this.getPipelineId(
-      this.params.name,
-      this.params.projectName,
-    );
-
-    if (!pipelineId) {
-      throw new Error("We couldn't find the pipeline ID for the given name");
-    }
+    const pipelineId = await this.getPipelineId();
 
     await PipelinesService.createBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPost(
       {
@@ -343,14 +293,7 @@ export class LlamaCloudIndex {
   }
 
   async delete(document: Document) {
-    const pipelineId = await this.getPipelineId(
-      this.params.name,
-      this.params.projectName,
-    );
-
-    if (!pipelineId) {
-      throw new Error("We couldn't find the pipeline ID for the given name");
-    }
+    const pipelineId = await this.getPipelineId();
 
     await PipelinesService.deletePipelineDocumentApiV1PipelinesPipelineIdDocumentsDocumentIdDelete(
       {
@@ -365,14 +308,7 @@ export class LlamaCloudIndex {
   }
 
   async refreshDoc(document: Document) {
-    const pipelineId = await this.getPipelineId(
-      this.params.name,
-      this.params.projectName,
-    );
-
-    if (!pipelineId) {
-      throw new Error("We couldn't find the pipeline ID for the given name");
-    }
+    const pipelineId = await this.getPipelineId();
 
     await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
       {
@@ -393,4 +329,71 @@ export class LlamaCloudIndex {
 
     await this.waitForDocumentIngestion([document.id_]);
   }
+
+  public async ensureIndex(config?: {
+    embedding?: PipelineCreate["embedding_config"];
+    transform?: PipelineCreate["transform_config"];
+    verbose?: boolean;
+  }): Promise<void> {
+    const projectId = await this.getProjectId();
+
+    const { data: pipelines } =
+      await PipelinesService.searchPipelinesApiV1PipelinesGet({
+        query: {
+          project_id: projectId,
+          pipeline_name: this.params.name,
+        },
+        throwOnError: true,
+      });
+
+    if (pipelines.length === 0) {
+      // no pipeline found, create a new one
+      let embeddingConfig = config?.embedding;
+      if (!embeddingConfig) {
+        // no embedding config provided, use OpenAI as default
+        const openAIApiKey = getEnv("OPENAI_API_KEY");
+        const embeddingModel = getEnv("EMBEDDING_MODEL");
+        if (!openAIApiKey || !embeddingModel) {
+          throw new Error(
+            "No embedding configuration provided. Fallback to OpenAI embedding model. OPENAI_API_KEY and EMBEDDING_MODEL environment variables must be set.",
+          );
+        }
+        embeddingConfig = {
+          type: "OPENAI_EMBEDDING",
+          component: {
+            api_key: openAIApiKey,
+            model_name: embeddingModel,
+          },
+        };
+      }
+
+      let transformConfig = config?.transform;
+      if (!transformConfig) {
+        transformConfig = {
+          mode: "auto",
+          chunk_size: 1024,
+          chunk_overlap: 200,
+        };
+      }
+
+      const { data: pipeline } =
+        await PipelinesService.upsertPipelineApiV1PipelinesPut({
+          path: {
+            project_id: projectId,
+          },
+          body: {
+            name: this.params.name,
+            embedding_config: embeddingConfig,
+            transform_config: transformConfig,
+          },
+          throwOnError: true,
+        });
+
+      if (config?.verbose) {
+        console.log(
+          `Created pipeline ${pipeline.id} with name ${pipeline.name}`,
+        );
+      }
+    }
+  }
 }

packages/llamaindex/src/cloud/LlamaCloudRetriever.ts

@@ -1,4 +1,5 @@
 import {
+  type MetadataFilter,
   type MetadataFilters,
   PipelinesService,
   type RetrievalParams,
@@ -11,7 +12,7 @@ import type { NodeWithScore } from "@llamaindex/core/schema";
 import { jsonToNode, ObjectType } from "@llamaindex/core/schema";
 import { extractText } from "@llamaindex/core/utils";
 import type { ClientParams, CloudConstructorParams } from "./type.js";
-import { getProjectId, initService } from "./utils.js";
+import { getPipelineId, initService } from "./utils.js";
 
 export type CloudRetrieveParams = Omit<
   RetrievalParams,
@@ -42,6 +43,24 @@ export class LlamaCloudRetriever extends BaseRetriever {
     });
   }
 
+  // LlamaCloud expects null values for filters, but LlamaIndexTS uses undefined for empty values
+  // This function converts the undefined values to null
+  private convertFilter(filters?: MetadataFilters): MetadataFilters | null {
+    if (!filters) return null;
+
+    const processFilter = (
+      filter: MetadataFilter | MetadataFilters,
+    ): MetadataFilter | MetadataFilters => {
+      if ("filters" in filter) {
+        // type MetadataFilters
+        return { ...filter, filters: filter.filters.map(processFilter) };
+      }
+      return { ...filter, value: filter.value ?? null };
+    };
+
+    return { ...filters, filters: filters.filters.map(processFilter) };
+  }
+
   constructor(params: CloudConstructorParams & CloudRetrieveParams) {
     super();
     this.clientParams = { apiKey: params.apiKey, baseUrl: params.baseUrl };
@@ -57,31 +76,24 @@ export class LlamaCloudRetriever extends BaseRetriever {
   }
 
   async _retrieve(query: QueryBundle): Promise<NodeWithScore[]> {
-    const { data: pipelines } =
-      await PipelinesService.searchPipelinesApiV1PipelinesGet({
-        query: {
-          project_id: await getProjectId(this.projectName, this.organizationId),
-          pipeline_name: this.pipelineName,
-        },
-        throwOnError: true,
-      });
+    const pipelineId = await getPipelineId(
+      this.pipelineName,
+      this.projectName,
+      this.organizationId,
+    );
 
-    if (pipelines.length === 0 || !pipelines[0]!.id) {
-      throw new Error(
-        `No pipeline found with name ${this.pipelineName} in project ${this.projectName}`,
-      );
-    }
+    const filters = this.convertFilter(this.retrieveParams.filters);
 
     const { data: results } =
       await PipelinesService.runSearchApiV1PipelinesPipelineIdRetrievePost({
         throwOnError: true,
         path: {
-          pipeline_id: pipelines[0]!.id,
+          pipeline_id: pipelineId,
         },
         body: {
           ...this.retrieveParams,
           query: extractText(query),
-          search_filters: this.retrieveParams.filters as MetadataFilters,
+          search_filters: filters,
           dense_similarity_top_k: this.retrieveParams.similarityTopK!,
         },
       });

packages/llamaindex/src/cloud/config.ts

@@ -1,55 +0,0 @@
-import type {
-  ConfiguredTransformationItem,
-  PipelineCreate,
-  PipelineType,
-} from "@llamaindex/cloud/api";
-import { SentenceSplitter } from "@llamaindex/core/node-parser";
-import { BaseNode, type TransformComponent } from "@llamaindex/core/schema";
-import { OpenAIEmbedding } from "@llamaindex/openai";
-
-export type GetPipelineCreateParams = {
-  pipelineName: string;
-  pipelineType: PipelineType;
-  transformations?: TransformComponent[];
-  inputNodes?: BaseNode[];
-};
-
-function getTransformationConfig(
-  transformation: TransformComponent,
-): ConfiguredTransformationItem {
-  if (transformation instanceof SentenceSplitter) {
-    return {
-      configurable_transformation_type: "SENTENCE_AWARE_NODE_PARSER",
-      component: {
-        chunk_size: transformation.chunkSize, // TODO: set to public in SentenceSplitter
-        chunk_overlap: transformation.chunkOverlap, // TODO: set to public in SentenceSplitter
-        include_metadata: transformation.includeMetadata,
-        include_prev_next_rel: transformation.includePrevNextRel,
-      },
-    };
-  }
-  if (transformation instanceof OpenAIEmbedding) {
-    return {
-      configurable_transformation_type: "OPENAI_EMBEDDING",
-      component: {
-        model: transformation.model,
-        api_key: transformation.apiKey,
-        embed_batch_size: transformation.embedBatchSize,
-        dimensions: transformation.dimensions,
-      },
-    };
-  }
-  throw new Error(`Unsupported transformation: ${typeof transformation}`);
-}
-
-export async function getPipelineCreate(
-  params: GetPipelineCreateParams,
-): Promise<PipelineCreate> {
-  const { pipelineName, pipelineType, transformations = [] } = params;
-
-  return {
-    name: pipelineName,
-    configured_transformations: transformations.map(getTransformationConfig),
-    pipeline_type: pipelineType,
-  };
-}

packages/llamaindex/src/cloud/utils.ts

@@ -1,4 +1,8 @@
-import { client, ProjectsService } from "@llamaindex/cloud/api";
+import {
+  client,
+  PipelinesService,
+  ProjectsService,
+} from "@llamaindex/cloud/api";
 import { DEFAULT_BASE_URL } from "@llamaindex/core/global";
 import { getEnv } from "@llamaindex/env";
 import type { ClientParams } from "./type.js";
@@ -66,3 +70,26 @@ export async function getProjectId(
 
   return project.id;
 }
+
+export async function getPipelineId(
+  name: string,
+  projectName: string,
+  organizationId?: string,
+): Promise<string> {
+  const { data: pipelines } =
+    await PipelinesService.searchPipelinesApiV1PipelinesGet({
+      query: {
+        project_id: await getProjectId(projectName, organizationId),
+        pipeline_name: name,
+      },
+      throwOnError: true,
+    });
+
+  if (pipelines.length === 0 || !pipelines[0]!.id) {
+    throw new Error(
+      `No pipeline found with name ${name} in project ${projectName}`,
+    );
+  }
+
+  return pipelines[0]!.id;
+}

packages/llamaindex/src/next.ts

@@ -37,6 +37,7 @@ export default function withLlamaIndex(config: any) {
     webpackConfig.resolve.alias = {
       ...webpackConfig.resolve.alias,
       "@google-cloud/vertexai": false,
+      unpdf: false,
     };
     // Following lines will fix issues with onnxruntime-node when using pnpm
     // See: https://github.com/vercel/next.js/issues/43433

packages/llm/groq/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/groq
 
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [6a9a7b1]
+  - @llamaindex/openai@0.1.16
+
 ## 0.0.14
 
 ### Patch Changes

packages/llm/groq/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/groq",
   "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.14",
+  "version": "0.0.15",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/llm/openai/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/openai
 
+## 0.1.16
+
+### Patch Changes
+
+- 6a9a7b1: fix: take init api key into account
+
 ## 0.1.15
 
 ### Patch Changes

packages/llm/openai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/openai",
   "description": "OpenAI Adapter for LlamaIndex",
-  "version": "0.1.15",
+  "version": "0.1.16",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/llm/openai/src/embedding.ts

@@ -114,7 +114,8 @@ export class OpenAIEmbedding extends BaseEmbedding {
             }),
         );
     } else {
-      this.apiKey = init?.session?.apiKey ?? getEnv("OPENAI_API_KEY");
+      this.apiKey =
+        init?.session?.apiKey ?? init?.apiKey ?? getEnv("OPENAI_API_KEY");
       this.lazySession = async () =>
         import("openai").then(({ OpenAI }) => {
           return (