docs: updated docs for retriever using param obj

.changeset/config.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://unpkg.com/@changesets/config@2.3.1/schema.json",
   "changelog": "@changesets/cli/changelog",
-  "commit": false,
+  "commit": true,
   "fixed": [],
   "linked": [],
   "access": "public",

.changeset/cuddly-cars-doubt.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add LlamaParse option when selecting a pdf file or a folder

.changeset/eight-dogs-burn.md

@@ -0,0 +1,12 @@
+---
+"llamaindex": patch
+"@llamaindex/core-test": patch
+---
+
+- Add missing exports:
+  - `IndexStructType`,
+  - `IndexDict`,
+  - `jsonToIndexStruct`,
+  - `IndexList`,
+  - `IndexStruct`
+- Fix `IndexDict.toJson()` method

.changeset/fresh-pugs-explain.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Add pipeline.register to create a managed index in LlamaCloud

.changeset/khaki-rivers-unite.md

@@ -2,4 +2,4 @@
 "llamaindex": patch
 ---
 
-feat: add wikipedia tool
+Add streaming to agents

.changeset/pretty-rings-press.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add embedding model option to create-llama

.changeset/quick-fans-battle.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-fix: support import subdirectory

.changeset/strange-ducks-kiss.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-feat: use claude3 with react agent

.changeset/strange-waves-sit.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": minor
+---
+
+Use parameter object for retrieve function of Retriever (to align usage with query function of QueryEngine)

.github/workflows/e2e.yml

@@ -0,0 +1,68 @@
+name: E2E Tests
+on:
+  push:
+    branches: [main]
+  pull_request:
+    paths:
+      - "packages/create-llama/**"
+      - ".github/workflows/e2e.yml"
+    branches: [main]
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  e2e:
+    name: create-llama
+    timeout-minutes: 60
+    strategy:
+      fail-fast: true
+      matrix:
+        node-version: [18, 20]
+        python-version: ["3.11"]
+        os: [macos-latest, windows-latest]
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - uses: pnpm/action-setup@v2
+      - name: Setup Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+      - name: Install dependencies
+        run: pnpm install
+      - name: Install Playwright Browsers
+        run: pnpm exec playwright install --with-deps
+        working-directory: ./packages/create-llama
+      - name: Build create-llama
+        run: pnpm run build
+        working-directory: ./packages/create-llama
+      - name: Pack
+        run: pnpm pack --pack-destination ./output
+        working-directory: ./packages/create-llama
+      - name: Extract Pack
+        run: tar -xvzf ./output/*.tgz -C ./output
+        working-directory: ./packages/create-llama
+      - name: Run Playwright tests
+        run: pnpm exec playwright test
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        working-directory: ./packages/create-llama
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: ./packages/create-llama/playwright-report/
+          retention-days: 30

.github/workflows/publish.yml

@@ -14,23 +14,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v2
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: ".nvmrc"
-          cache: "pnpm"
-      - name: Install dependencies
-        run: pnpm install
 
       - name: Publish @llamaindex/env
         run: npx jsr publish
         working-directory: packages/env
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Publish @llamaindex/core
         run: npx jsr publish --allow-slow-types
         working-directory: packages/core
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -44,24 +44,6 @@ jobs:
           name: typecheck-build-dist
           path: ./packages/core/dist
           if-no-files-found: error
-  core-edge-runtime:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v2
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: ".nvmrc"
-          cache: "pnpm"
-      - name: Install dependencies
-        run: pnpm install
-      - name: Build
-        run: pnpm run build --filter @llamaindex/edge
-      - name: Build Edge Runtime
-        run: pnpm run build
-        working-directory: ./packages/edge/e2e/test-edge-runtime
   typecheck-examples:
     runs-on: ubuntu-latest
 

CONTRIBUTING.md

@@ -79,27 +79,13 @@ That should start a webserver which will serve the docs on https://localhost:300
 
 Any changes you make should be reflected in the browser. If you need to regenerate the API docs and find that your TSDoc isn't getting the updates, feel free to remove apps/docs/api. It will automatically regenerate itself when you run pnpm start again.
 
-## Changeset
+## Publishing
 
-We use [changesets](https://github.com/changesets/changesets) for managing versions and changelogs. To create a new changeset, run:
-
-```
-pnpm changeset
-```
-
-Please send a descriptive changeset for each PR.
-
-## Publishing (maintainers only)
-
-To publish a new version of the library, first create a new version:
-
-```shell
-pnpm new-version
-```
-
-If everything looks good, commit the generated files and release the new version:
+To publish a new version of the library, run
 
 ```shell
+pnpm new-llamaindex
+pnpm new-create-llama
 pnpm release
 git push # push to the main branch
 git push --tags

README.md

@@ -121,42 +121,6 @@ const nextConfig = {
 module.exports = nextConfig;
 ```
 
-### NextJS with Milvus:
-
-As proto files are not loaded per default in NextJS, you'll need to add the following to your next.config.js to have it load the proto files.
-
-```js
-const path = require("path");
-const CopyWebpackPlugin = require("copy-webpack-plugin");
-
-// next.config.js
-/** @type {import('next').NextConfig} */
-const nextConfig = {
-  webpack: (config, { isServer }) => {
-    if (isServer) {
-      // Copy the proto files to the server build directory
-      config.plugins.push(
-        new CopyWebpackPlugin({
-          patterns: [
-            {
-              from: path.join(
-                __dirname,
-                "node_modules/@zilliz/milvus2-sdk-node/dist",
-              ),
-              to: path.join(__dirname, ".next"),
-            },
-          ],
-        }),
-      );
-    }
-    // Important: return the modified config
-    return config;
-  },
-};
-
-module.exports = nextConfig;
-```
-
 ## Supported LLMs:
 
 - OpenAI GPT-3.5-turbo and GPT-4

apps/docs/docs/modules/ingestion_pipeline/transformations.md

@@ -1,6 +1,6 @@
 # Transformations
 
-A transformation is something that takes a list of nodes as an input, and returns a list of nodes. Each component that implements the Transformation class has both a `transform` definition responsible for transforming the nodes.
+A transformation is something that takes a list of nodes as an input, and returns a list of nodes. Each component that implements the Transformatio class has both a `transform` definition responsible for transforming the nodes
 
 Currently, the following components are Transformation objects:
 

apps/docs/docs/modules/node_postprocessors/index.md

@@ -36,7 +36,7 @@ const processor = new SimilarityPostprocessor({
   similarityCutoff: 0.7,
 });
 
-const filteredNodes = await processor.postprocessNodes(nodes);
+const filteredNodes = processor.postprocessNodes(nodes);
 
 // cohere rerank: rerank nodes given query using trained model
 const reranker = new CohereRerank({

examples/CHANGELOG.md

@@ -1,14 +0,0 @@
-# examples
-
-## 0.0.4
-
-### Patch Changes
-
-- d2e8d0c: add support for Milvus vector store
-- Updated dependencies [d2e8d0c]
-- Updated dependencies [aefc326]
-- Updated dependencies [484a710]
-- Updated dependencies [d766bd0]
-- Updated dependencies [dd95927]
-- Updated dependencies [bf583a7]
-  - llamaindex@0.2.0

examples/agent/react_agent.ts

@@ -1,4 +1,4 @@
-import { Anthropic, FunctionTool, ReActAgent } from "llamaindex";
+import { FunctionTool, ReActAgent } from "llamaindex";
 
 // Define a function to sum two numbers
 function sumNumbers({ a, b }: { a: number; b: number }): number {
@@ -56,14 +56,8 @@ async function main() {
     parameters: divideJSON,
   });
 
-  const anthropic = new Anthropic({
-    apiKey: process.env.ANTHROPIC_API_KEY,
-    model: "claude-3-opus",
-  });
-
-  // Create an ReActAgent with the function tools
+  // Create an OpenAIAgent with the function tools
   const agent = new ReActAgent({
-    llm: anthropic,
     tools: [functionTool, functionTool2],
     verbose: true,
   });

examples/agent/wikipedia-tool.ts

@@ -1,23 +0,0 @@
-import { OpenAIAgent, WikipediaTool } from "llamaindex";
-
-async function main() {
-  const wikipediaTool = new WikipediaTool();
-
-  // Create an OpenAIAgent with the function tools
-  const agent = new OpenAIAgent({
-    tools: [wikipediaTool],
-    verbose: true,
-  });
-
-  // Chat with the agent
-  const response = await agent.chat({
-    message: "Where is Ho Chi Minh City?",
-  });
-
-  // Print the response
-  console.log(response);
-}
-
-main().then(() => {
-  console.log("Done");
-});

examples/anthropic/haiku.ts

@@ -1,19 +0,0 @@
-import { Anthropic } from "llamaindex";
-
-(async () => {
-  const anthropic = new Anthropic({
-    apiKey: process.env.ANTHROPIC_API_KEY,
-    model: "claude-3-haiku",
-  });
-  const result = await anthropic.chat({
-    messages: [
-      { content: "You want to talk in rhymes.", role: "system" },
-      {
-        content:
-          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
-        role: "user",
-      },
-    ],
-  });
-  console.log(result);
-})();

examples/astradb/README.md

@@ -32,10 +32,10 @@ run `ts-node astradb/example`
 
 This sample loads the same dataset of movie reviews as the Astra Portal sample dataset. (Feel free to load the data in your the Astra Data Explorer to compare)
 
-run `npx ts-node astradb/load`
+run `ts-node astradb/load`
 
 ### Use RAG to Query the data
 
 Check out your data in the Astra Data Explorer and change the sample query as you see fit.
 
-run `npx ts-node astradb/query`
+run `ts-node astradb/query`

examples/cloud/README.md

@@ -31,11 +31,3 @@ This example shows how to use the managed index with a query engine.
 ```shell
 pnpx ts-node cloud/query.ts
 ```
-
-## Pipeline
-
-This example shows how to create a managed index with a pipeline.
-
-```shell
-pnpx ts-node cloud/pipeline.ts
-```

examples/cloud/pipeline.ts

@@ -1,34 +0,0 @@
-import fs from "node:fs/promises";
-
-import {
-  Document,
-  IngestionPipeline,
-  OpenAIEmbedding,
-  SimpleNodeParser,
-} from "llamaindex";
-
-async function main() {
-  // Load essay from abramov.txt in Node
-  const path = "node_modules/llamaindex/examples/abramov.txt";
-
-  const essay = await fs.readFile(path, "utf-8");
-
-  // Create Document object with essay
-  const document = new Document({ text: essay, id_: path });
-  const pipeline = new IngestionPipeline({
-    name: "pipeline",
-    transformations: [
-      new SimpleNodeParser({ chunkSize: 1024, chunkOverlap: 20 }),
-      new OpenAIEmbedding({ apiKey: "api-key" }),
-    ],
-  });
-
-  const pipelineId = await pipeline.register({
-    documents: [document],
-    verbose: true,
-  });
-
-  console.log(`Pipeline with id ${pipelineId} successfully created.`);
-}
-
-main().catch(console.error);

examples/milvus/README.md

@@ -1,34 +0,0 @@
-# Milvus Vector Store
-
-Here are two sample scripts which work with loading and querying data from a Milvus Vector Store.
-
-## Prerequisites
-
-- An Milvus Vector Database
-  - Hosted https://milvus.io/
-  - Self Hosted https://milvus.io/docs/install_standalone-docker.md
-- An OpenAI API Key
-
-## Setup
-
-1. Set your env variables:
-
-- `MILVUS_ADDRESS`: Address of your Milvus Vector Store (like localhost:19530)
-- `MILVUS_USERNAME`: empty or username for your Milvus Vector Store
-- `MILVUS_PASSWORD`: empty or password for your Milvus Vector Store
-- `OPENAI_API_KEY`: Your OpenAI key
-
-2. `cd` Into the `examples` directory
-3. run `npm i`
-
-## Load the data
-
-This sample loads the same dataset of movie reviews as sample dataset. You can install https://github.com/zilliztech/attu to inspect the loaded data.
-
-run `npx ts-node milvus/load`
-
-## Use RAG to Query the data
-
-Check out your data in Attu and change the sample query as you see fit.
-
-run `npx ts-node milvus/query`

examples/milvus/load.ts

@@ -1,26 +0,0 @@
-import {
-  MilvusVectorStore,
-  PapaCSVReader,
-  storageContextFromDefaults,
-  VectorStoreIndex,
-} from "llamaindex";
-
-const collectionName = "movie_reviews";
-
-async function main() {
-  try {
-    const reader = new PapaCSVReader(false);
-    const docs = await reader.loadData("./data/movie_reviews.csv");
-
-    const vectorStore = new MilvusVectorStore({ collection: collectionName });
-
-    const ctx = await storageContextFromDefaults({ vectorStore });
-    const index = await VectorStoreIndex.fromDocuments(docs, {
-      storageContext: ctx,
-    });
-  } catch (e) {
-    console.error(e);
-  }
-}
-
-main();

examples/milvus/query.ts

@@ -1,30 +0,0 @@
-import {
-  MilvusVectorStore,
-  serviceContextFromDefaults,
-  VectorStoreIndex,
-} from "llamaindex";
-
-const collectionName = "movie_reviews";
-
-async function main() {
-  try {
-    const milvus = new MilvusVectorStore({ collection: collectionName });
-
-    const ctx = serviceContextFromDefaults();
-    const index = await VectorStoreIndex.fromVectorStore(milvus, ctx);
-
-    const retriever = await index.asRetriever({ similarityTopK: 20 });
-
-    const queryEngine = await index.asQueryEngine({ retriever });
-
-    const results = await queryEngine.query({
-      query: "What is the best reviewed movie?",
-    });
-
-    console.log(results.response);
-  } catch (e) {
-    console.error(e);
-  }
-}
-
-main();

examples/package.json

@@ -1,19 +1,16 @@
 {
   "name": "examples",
   "private": true,
-  "version": "0.0.4",
+  "version": "0.0.3",
   "dependencies": {
-    "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.4",
     "@notionhq/client": "^2.2.14",
     "@pinecone-database/pinecone": "^1.1.3",
-    "@zilliz/milvus2-sdk-node": "^2.3.5",
     "chromadb": "^1.8.1",
     "commander": "^11.1.0",
     "dotenv": "^16.4.1",
     "llamaindex": "latest",
-    "mongodb": "^6.2.0",
-    "pathe": "^1.1.2"
+    "mongodb": "^6.2.0"
   },
   "devDependencies": {
     "@types/node": "^18.19.10",

examples/qdrantdb/README.md

@@ -1,11 +0,0 @@
-# Qdrant Vector Store Example
-
-How to run `examples/qdrantdb/preFilters.ts`:
-
-Add your OpenAI API Key into a file called `.env` in the parent folder of this directory. It should look like this:
-
-```
-OPEN_API_KEY=sk-you-key
-```
-
-Now, open a new terminal window and inside `examples`, run `npx ts-node qdrantdb/preFilters.ts`.

examples/qdrantdb/preFilters.ts

@@ -1,82 +0,0 @@
-import * as dotenv from "dotenv";
-import {
-  CallbackManager,
-  Document,
-  MetadataMode,
-  QdrantVectorStore,
-  VectorStoreIndex,
-  serviceContextFromDefaults,
-  storageContextFromDefaults,
-} from "llamaindex";
-
-// Load environment variables from local .env file
-dotenv.config();
-
-const collectionName = "dog_colors";
-const qdrantUrl = "http://127.0.0.1:6333";
-
-async function main() {
-  try {
-    const docs = [
-      new Document({
-        text: "The dog is brown",
-        metadata: {
-          dogId: "1",
-        },
-      }),
-      new Document({
-        text: "The dog is red",
-        metadata: {
-          dogId: "2",
-        },
-      }),
-    ];
-    console.log("Creating QdrantDB vector store");
-    const qdrantVs = new QdrantVectorStore({ url: qdrantUrl, collectionName });
-    const ctx = await storageContextFromDefaults({ vectorStore: qdrantVs });
-
-    console.log("Embedding documents and adding to index");
-    const index = await VectorStoreIndex.fromDocuments(docs, {
-      storageContext: ctx,
-      serviceContext: serviceContextFromDefaults({
-        callbackManager: new CallbackManager({
-          onRetrieve: (data) => {
-            console.log(
-              "The retrieved nodes are:",
-              data.nodes.map((node) => node.node.getContent(MetadataMode.NONE)),
-            );
-          },
-        }),
-      }),
-    });
-
-    console.log(
-      "Querying index with no filters: Expected output: Brown probably",
-    );
-    const queryEngineNoFilters = index.asQueryEngine();
-    const noFilterResponse = await queryEngineNoFilters.query({
-      query: "What is the color of the dog?",
-    });
-    console.log("No filter response:", noFilterResponse.toString());
-    console.log("Querying index with dogId 2: Expected output: Red");
-    const queryEngineDogId2 = index.asQueryEngine({
-      preFilters: {
-        filters: [
-          {
-            key: "dogId",
-            value: "2",
-            filterType: "ExactMatch",
-          },
-        ],
-      },
-    });
-    const response = await queryEngineDogId2.query({
-      query: "What is the color of the dog?",
-    });
-    console.log("Filter with dogId 2 response:", response.toString());
-  } catch (e) {
-    console.error(e);
-  }
-}
-
-main();

examples/readers/CHANGELOG.md

@@ -1,11 +0,0 @@
-# llamaindex-loader-example
-
-### Patch Changes
-
-- Updated dependencies [d2e8d0c]
-- Updated dependencies [aefc326]
-- Updated dependencies [484a710]
-- Updated dependencies [d766bd0]
-- Updated dependencies [dd95927]
-- Updated dependencies [bf583a7]
-  - llamaindex@0.2.0

examples/readers/src/assemblyai.ts

@@ -1,6 +1,6 @@
 import { program } from "commander";
-import { VectorStoreIndex, type TranscribeParams } from "llamaindex";
-import { AudioTranscriptReader } from "llamaindex/readers";
+import { TranscribeParams, VectorStoreIndex } from "llamaindex";
+import { AudioTranscriptReader } from "llamaindex/readers/AssemblyAIReader";
 import { stdin as input, stdout as output } from "node:process";
 import { createInterface } from "node:readline/promises";
 

examples/readers/src/csv.ts

@@ -5,7 +5,7 @@ import {
   serviceContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
-import { PapaCSVReader } from "llamaindex/readers";
+import { PapaCSVReader } from "llamaindex/readers/CSVReader";
 
 async function main() {
   // Load CSV

examples/readers/src/custom-simple-directory-reader.ts

@@ -3,7 +3,7 @@ import {
   FILE_EXT_TO_READER,
   SimpleDirectoryReader,
   TextFileReader,
-} from "llamaindex/readers";
+} from "llamaindex/readers/SimpleDirectoryReader";
 
 class ZipReader implements BaseReader {
   loadData(...args: any[]): Promise<Document<Metadata>[]> {

examples/readers/src/docx.ts

@@ -1,5 +1,5 @@
-import { VectorStoreIndex } from "llamaindex/indices";
-import { DocxReader } from "llamaindex/readers";
+import { VectorStoreIndex } from "llamaindex";
+import { DocxReader } from "llamaindex/readers/DocxReader";
 
 const FILE_PATH = "../data/stars.docx";
 const SAMPLE_QUERY = "Information about Zodiac";

examples/readers/src/html.ts

@@ -1,5 +1,5 @@
-import { VectorStoreIndex } from "llamaindex/indices";
-import { HTMLReader } from "llamaindex/readers";
+import { VectorStoreIndex } from "llamaindex";
+import { HTMLReader } from "llamaindex/readers/HTMLReader";
 
 async function main() {
   // Load page

examples/readers/src/llamaparse.ts

@@ -1,5 +1,4 @@
-import { VectorStoreIndex } from "llamaindex/indices";
-import { LlamaParseReader } from "llamaindex/readers";
+import { LlamaParseReader, VectorStoreIndex } from "llamaindex";
 
 async function main() {
   // Load PDF using LlamaParse

examples/readers/src/markdown.ts

@@ -1,5 +1,5 @@
-import { VectorStoreIndex } from "llamaindex/indices";
-import { MarkdownReader } from "llamaindex/readers";
+import { VectorStoreIndex } from "llamaindex";
+import { MarkdownReader } from "llamaindex/readers/MarkdownReader";
 
 const FILE_PATH = "../data/planets.md";
 const SAMPLE_QUERY = "List all planets";

examples/readers/src/notion.ts

@@ -1,7 +1,7 @@
 import { Client } from "@notionhq/client";
 import { program } from "commander";
-import { VectorStoreIndex } from "llamaindex/indices";
-import { NotionReader } from "llamaindex/readers";
+import { VectorStoreIndex } from "llamaindex";
+import { NotionReader } from "llamaindex/readers/NotionReader";
 import { stdin as input, stdout as output } from "node:process";
 import { createInterface } from "node:readline/promises";
 

examples/readers/src/pdf.ts

@@ -1,5 +1,5 @@
-import { VectorStoreIndex } from "llamaindex/indices";
-import { PDFReader } from "llamaindex/readers";
+import { VectorStoreIndex } from "llamaindex";
+import { PDFReader } from "llamaindex/readers/PDFReader";
 
 async function main() {
   // Load PDF

examples/readers/src/pdf_fw.ts

@@ -1,7 +1,5 @@
-import { FireworksEmbedding } from "llamaindex/embeddings";
-import { VectorStoreIndex } from "llamaindex/indices";
-import { FireworksLLM } from "llamaindex/llm";
-import { PDFReader } from "llamaindex/readers";
+import { FireworksEmbedding, FireworksLLM, VectorStoreIndex } from "llamaindex";
+import { PDFReader } from "llamaindex/readers/PDFReader";
 
 import { serviceContextFromDefaults } from "llamaindex";
 

examples/readers/src/pdf_fw_openai.ts

@@ -1,7 +1,5 @@
-import { OpenAIEmbedding } from "llamaindex/embeddings";
-import { VectorStoreIndex } from "llamaindex/indices";
-import { OpenAI } from "llamaindex/llm";
-import { PDFReader } from "llamaindex/readers";
+import { OpenAI, OpenAIEmbedding, VectorStoreIndex } from "llamaindex";
+import { PDFReader } from "llamaindex/readers/PDFReader";
 
 import { serviceContextFromDefaults } from "llamaindex";
 

examples/readers/src/simple-directory-reader.ts

@@ -1,4 +1,4 @@
-import { SimpleDirectoryReader } from "llamaindex/readers";
+import { SimpleDirectoryReader } from "llamaindex/readers/SimpleDirectoryReader";
 // or
 // import { SimpleDirectoryReader } from 'llamaindex'
 

package.json

@@ -11,12 +11,10 @@
     "prepare": "husky",
     "test": "turbo run test",
     "type-check": "tsc -b --diagnostics",
-    "release": "pnpm run check-minor-version && pnpm run build:release && changeset publish",
-    "release-snapshot": "pnpm run check-minor-version && pnpm run build:release && changeset publish --tag snapshot",
-    "check-minor-version": "node ./scripts/check-minor-version",
-    "update-version": "node ./scripts/update-version",
-    "new-version": "pnpm run build:release && changeset version && pnpm run check-minor-version && pnpm run update-version",
-    "new-snapshot": "pnpm run build:release && changeset version --snapshot && pnpm run update-version"
+    "release": "pnpm run build:release && changeset publish",
+    "new-llamaindex": "pnpm run build:release && changeset version --ignore create-llama",
+    "new-create-llama": "pnpm run build:release && changeset version --ignore llamaindex",
+    "new-snapshots": "pnpm run build:release && changeset version --snapshot"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",

packages/core/CHANGELOG.md

@@ -1,33 +1,5 @@
 # llamaindex
 
-## 0.2.1
-
-### Patch Changes
-
-- 41210df: Add auto create milvus collection and add milvus node metadata
-- 137cf67: Use Pinecone namespaces for all operations
-- 259c842: Add support for edge runtime by using @llamaindex/edge
-
-## 0.2.0
-
-### Minor Changes
-
-- bf583a7: Use parameter object for retrieve function of Retriever (to align usage with query function of QueryEngine)
-
-### Patch Changes
-
-- d2e8d0c: add support for Milvus vector store
-- aefc326: feat: experimental package + json query engine
-- 484a710: - Add missing exports:
-  - `IndexStructType`,
-  - `IndexDict`,
-  - `jsonToIndexStruct`,
-  - `IndexList`,
-  - `IndexStruct`
-  - Fix `IndexDict.toJson()` method
-- d766bd0: Add streaming to agents
-- dd95927: add Claude Haiku support and update anthropic SDK
-
 ## 0.1.21
 
 ### Patch Changes

packages/core/package.json

@@ -1,14 +1,12 @@
 {
   "name": "llamaindex",
-  "version": "0.2.1",
-  "expectedMinorVersion": "2",
+  "version": "0.1.21",
   "license": "MIT",
   "type": "module",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.18.0",
+    "@anthropic-ai/sdk": "^0.15.0",
     "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.4",
-    "@grpc/grpc-js": "^1.10.2",
     "@llamaindex/cloud": "0.0.4",
     "@llamaindex/env": "workspace:*",
     "@mistralai/mistralai": "^0.0.10",
@@ -20,13 +18,12 @@
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.0",
     "@xenova/transformers": "^2.15.0",
-    "@zilliz/milvus2-sdk-node": "^2.3.5",
     "assemblyai": "^4.2.2",
     "chromadb": "~1.7.3",
     "cohere-ai": "^7.7.5",
+    "file-type": "^18.7.0",
     "js-tiktoken": "^1.0.10",
     "lodash": "^4.17.21",
-    "magic-bytes.js": "^1.10.0",
     "mammoth": "^1.6.0",
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.3.0",
@@ -41,8 +38,7 @@
     "rake-modified": "^1.0.8",
     "replicate": "^0.25.2",
     "string-strip-html": "^13.4.6",
-    "wink-nlp": "^1.14.3",
-    "wikipedia": "^2.1.2"
+    "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
     "@swc/cli": "^0.3.9",
@@ -63,191 +59,15 @@
         "types": "./dist/type/index.d.ts",
         "default": "./dist/index.js"
       },
+      "edge-light": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/index.edge-light.js"
+      },
       "require": {
         "types": "./dist/type/index.d.ts",
         "default": "./dist/cjs/index.js"
       }
     },
-    "./agent": {
-      "import": {
-        "types": "./dist/type/agent/index.d.ts",
-        "default": "./dist/agent/index.js"
-      },
-      "require": {
-        "types": "./dist/type/agent/index.d.ts",
-        "default": "./dist/cjs/agent/index.js"
-      }
-    },
-    "./cloud": {
-      "import": {
-        "types": "./dist/type/cloud/index.d.ts",
-        "default": "./dist/cloud/index.js"
-      },
-      "require": {
-        "types": "./dist/type/cloud/index.d.ts",
-        "default": "./dist/cjs/cloud/index.js"
-      }
-    },
-    "./embeddings": {
-      "import": {
-        "types": "./dist/type/embeddings/index.d.ts",
-        "default": "./dist/embeddings/index.js"
-      },
-      "require": {
-        "types": "./dist/type/embeddings/index.d.ts",
-        "default": "./dist/cjs/embeddings/index.js"
-      }
-    },
-    "./engines": {
-      "import": {
-        "types": "./dist/type/engines/index.d.ts",
-        "default": "./dist/engines/index.js"
-      },
-      "require": {
-        "types": "./dist/type/engines/index.d.ts",
-        "default": "./dist/cjs/engines/index.js"
-      }
-    },
-    "./evaluation": {
-      "import": {
-        "types": "./dist/type/evaluation/index.d.ts",
-        "default": "./dist/evaluation/index.js"
-      },
-      "require": {
-        "types": "./dist/type/evaluation/index.d.ts",
-        "default": "./dist/cjs/evaluation/index.js"
-      }
-    },
-    "./extractors": {
-      "import": {
-        "types": "./dist/type/extractors/index.d.ts",
-        "default": "./dist/extractors/index.js"
-      },
-      "require": {
-        "types": "./dist/type/extractors/index.d.ts",
-        "default": "./dist/cjs/extractors/index.js"
-      }
-    },
-    "./indices": {
-      "import": {
-        "types": "./dist/type/indices/index.d.ts",
-        "default": "./dist/indices/index.js"
-      },
-      "require": {
-        "types": "./dist/type/indices/index.d.ts",
-        "default": "./dist/cjs/indices/index.js"
-      }
-    },
-    "./ingestion": {
-      "import": {
-        "types": "./dist/type/ingestion/index.d.ts",
-        "default": "./dist/ingestion/index.js"
-      },
-      "require": {
-        "types": "./dist/type/ingestion/index.d.ts",
-        "default": "./dist/cjs/ingestion/index.js"
-      }
-    },
-    "./llm": {
-      "import": {
-        "types": "./dist/type/llm/index.d.ts",
-        "default": "./dist/llm/index.js"
-      },
-      "require": {
-        "types": "./dist/type/llm/index.d.ts",
-        "default": "./dist/cjs/llm/index.js"
-      }
-    },
-    "./nodeParsers": {
-      "import": {
-        "types": "./dist/type/nodeParsers/index.d.ts",
-        "default": "./dist/nodeParsers/index.js"
-      },
-      "require": {
-        "types": "./dist/type/nodeParsers/index.d.ts",
-        "default": "./dist/cjs/nodeParsers/index.js"
-      }
-    },
-    "./objects": {
-      "import": {
-        "types": "./dist/type/objects/index.d.ts",
-        "default": "./dist/objects/index.js"
-      },
-      "require": {
-        "types": "./dist/type/objects/index.d.ts",
-        "default": "./dist/cjs/objects/index.js"
-      }
-    },
-    "./postprocessors": {
-      "import": {
-        "types": "./dist/type/postprocessors/index.d.ts",
-        "default": "./dist/postprocessors/index.js"
-      },
-      "require": {
-        "types": "./dist/type/postprocessors/index.d.ts",
-        "default": "./dist/cjs/postprocessors/index.js"
-      }
-    },
-    "./prompts": {
-      "import": {
-        "types": "./dist/type/prompts/index.d.ts",
-        "default": "./dist/prompts/index.js"
-      },
-      "require": {
-        "types": "./dist/type/prompts/index.d.ts",
-        "default": "./dist/cjs/prompts/index.js"
-      }
-    },
-    "./readers": {
-      "import": {
-        "types": "./dist/type/readers/index.d.ts",
-        "default": "./dist/readers/index.js"
-      },
-      "require": {
-        "types": "./dist/type/readers/index.d.ts",
-        "default": "./dist/cjs/readers/index.js"
-      }
-    },
-    "./selectors": {
-      "import": {
-        "types": "./dist/type/selectors/index.d.ts",
-        "default": "./dist/selectors/index.js"
-      },
-      "require": {
-        "types": "./dist/type/selectors/index.d.ts",
-        "default": "./dist/cjs/selectors/index.js"
-      }
-    },
-    "./storage": {
-      "import": {
-        "types": "./dist/type/storage/index.d.ts",
-        "default": "./dist/storage/index.js"
-      },
-      "require": {
-        "types": "./dist/type/storage/index.d.ts",
-        "default": "./dist/cjs/storage/index.js"
-      }
-    },
-    "./synthesizers": {
-      "import": {
-        "types": "./dist/type/synthesizers/index.d.ts",
-        "default": "./dist/synthesizers/index.js"
-      },
-      "require": {
-        "types": "./dist/type/synthesizers/index.d.ts",
-        "default": "./dist/cjs/synthesizers/index.js"
-      }
-    },
-    "./tools": {
-      "import": {
-        "types": "./dist/type/tools/index.d.ts",
-        "default": "./dist/tools/index.js"
-      },
-      "require": {
-        "types": "./dist/type/tools/index.d.ts",
-        "default": "./dist/cjs/tools/index.js"
-      }
-    },
     "./*": {
       "import": {
         "types": "./dist/type/*.d.ts",
@@ -272,9 +92,9 @@
   "scripts": {
     "lint": "eslint .",
     "build": "rm -rf ./dist && pnpm run build:esm && pnpm run build:cjs && pnpm run build:type",
-    "build:esm": "swc src -d dist --strip-leading-paths --config-file ../../.swcrc",
-    "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file ../../.cjs.swcrc",
-    "build:type": "tsc -p tsconfig.json",
+    "build:esm": "swc src -d dist --strip-leading-paths --config-file .swcrc",
+    "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file .cjs.swcrc",
+    "build:type": "pnpm run -w type-check",
     "copy": "cp -r ../../README.md ../../LICENSE .",
     "postbuild": "pnpm run copy && node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"",
     "circular-check": "madge -c ./src/index.ts",

packages/core/src/agent/openai/worker.ts

@@ -1,3 +1,5 @@
+// Assuming that the necessary interfaces and classes (like BaseTool, OpenAI, ChatMessage, CallbackManager, etc.) are defined elsewhere
+
 import { randomUUID } from "@llamaindex/env";
 import { Response } from "../../Response.js";
 import type { CallbackManager } from "../../callbacks/CallbackManager.js";

packages/core/src/agent/react/formatter.ts

@@ -60,7 +60,7 @@ export class ReActChatFormatter implements BaseAgentChatFormatter {
       } else {
         message = {
           content: reasoningStep.getContent(),
-          role: "assistant",
+          role: "system",
         };
       }
 

packages/core/src/agent/react/worker.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from "@llamaindex/env";
+import { randomUUID } from "crypto";
 import { CallbackManager } from "../../callbacks/CallbackManager.js";
 import { AgentChatResponse } from "../../engines/chat/index.js";
 import type { ChatResponse, LLM } from "../../llm/index.js";
@@ -17,6 +17,7 @@ import {
   ObservationReasoningStep,
   ResponseReasoningStep,
 } from "./types.js";
+
 type ReActAgentWorkerParams = {
   tools: BaseTool[];
   llm?: LLM;

packages/core/src/agent/runner/base.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from "@llamaindex/env";
+import { randomUUID } from "crypto";
 import { CallbackManager } from "../../callbacks/CallbackManager.js";
 import type { ChatEngineAgentParams } from "../../engines/chat/index.js";
 import {
@@ -12,6 +12,7 @@ import type { BaseMemory } from "../../memory/types.js";
 import type { AgentWorker, TaskStepOutput } from "../types.js";
 import { Task, TaskStep } from "../types.js";
 import { AgentState, BaseAgentRunner, TaskState } from "./types.js";
+
 const validateStepFromArgs = (
   taskId: string,
   input?: string | null,

packages/core/src/agent/types.ts

@@ -3,7 +3,6 @@ import type {
   ChatEngineAgentParams,
   StreamingAgentChatResponse,
 } from "../engines/chat/index.js";
-
 import type { QueryEngineParamsNonStreaming } from "../types.js";
 
 export interface AgentWorker {

packages/core/src/cloud/config.ts

@@ -1,81 +0,0 @@
-import type { PlatformApi } from "@llamaindex/cloud";
-import { BaseNode, Document } from "../Node.js";
-import { OpenAIEmbedding } from "../embeddings/OpenAIEmbedding.js";
-import type { TransformComponent } from "../ingestion/types.js";
-import { SimpleNodeParser } from "../nodeParsers/SimpleNodeParser.js";
-
-export type GetPipelineCreateParams = {
-  pipelineName: string;
-  pipelineType: any; // TODO: PlatformApi.PipelineType is not exported
-  transformations?: TransformComponent[];
-  inputNodes?: BaseNode[];
-};
-
-function getTransformationConfig(
-  transformation: TransformComponent,
-): PlatformApi.ConfiguredTransformationItem {
-  if (transformation instanceof SimpleNodeParser) {
-    return {
-      configurableTransformationType: "SENTENCE_AWARE_NODE_PARSER",
-      component: {
-        // TODO: API returns 422 if these parameters are included
-        // chunkSize: transformation.textSplitter.chunkSize, // TODO: set to public in SentenceSplitter
-        // chunkOverlap: transformation.textSplitter.chunkOverlap, // TODO: set to public in SentenceSplitter
-        // includeMetadata: transformation.includeMetadata,
-        // includePrevNextRel: transformation.includePrevNextRel,
-      },
-    };
-  }
-  if (transformation instanceof OpenAIEmbedding) {
-    return {
-      configurableTransformationType: "OPENAI_EMBEDDING",
-      component: {
-        modelName: transformation.model,
-        apiKey: transformation.apiKey,
-        embedBatchSize: transformation.embedBatchSize,
-        dimensions: transformation.dimensions,
-      },
-    };
-  }
-  throw new Error(`Unsupported transformation: ${typeof transformation}`);
-}
-
-function getDataSourceConfig(node: BaseNode): PlatformApi.DataSourceCreate {
-  if (node instanceof Document) {
-    return {
-      name: node.id_,
-      sourceType: "DOCUMENT",
-      component: {
-        id: node.id_,
-        text: node.text,
-        textTemplate: node.textTemplate,
-        startCharIdx: node.startCharIdx,
-        endCharIdx: node.endCharIdx,
-        metadataSeparator: node.metadataSeparator,
-        excludedEmbedMetadataKeys: node.excludedEmbedMetadataKeys,
-        excludedLlmMetadataKeys: node.excludedLlmMetadataKeys,
-        extraInfo: node.metadata,
-      },
-    };
-  }
-  throw new Error(`Unsupported node: ${typeof node}`);
-}
-
-export async function getPipelineCreate(
-  params: GetPipelineCreateParams,
-): Promise<PlatformApi.PipelineCreate> {
-  const {
-    pipelineName,
-    pipelineType,
-    transformations = [],
-    inputNodes = [],
-  } = params;
-
-  return {
-    name: pipelineName,
-    configuredTransformations: transformations.map(getTransformationConfig),
-    dataSources: inputNodes.map(getDataSourceConfig),
-    dataSinks: [],
-    pipelineType,
-  };
-}

packages/core/src/cloud/types.ts

@@ -1,6 +1,5 @@
 import type { ServiceContext } from "../ServiceContext.js";
 
-export const DEFAULT_PIPELINE_NAME = "default";
 export const DEFAULT_PROJECT_NAME = "default";
 export const DEFAULT_BASE_URL = "https://api.cloud.llamaindex.ai";
 

packages/core/src/cloud/utils.ts

@@ -3,20 +3,12 @@ import { getEnv } from "@llamaindex/env";
 import type { ClientParams } from "./types.js";
 import { DEFAULT_BASE_URL } from "./types.js";
 
-function getBaseUrl(baseUrl?: string): string {
-  return baseUrl ?? getEnv("LLAMA_CLOUD_BASE_URL") ?? DEFAULT_BASE_URL;
-}
-
-export function getAppBaseUrl(baseUrl?: string): string {
-  return getBaseUrl(baseUrl).replace(/api\./, "");
-}
-
 export async function getClient({
   apiKey,
   baseUrl,
 }: ClientParams = {}): Promise<PlatformApiClient> {
   // Get the environment variables or use defaults
-  baseUrl = getBaseUrl(baseUrl);
+  baseUrl = baseUrl ?? getEnv("LLAMA_CLOUD_BASE_URL") ?? DEFAULT_BASE_URL;
   apiKey = apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
 
   const { PlatformApiClient } = await import("@llamaindex/cloud");

packages/core/src/embeddings/utils.ts

@@ -1,6 +1,5 @@
 import { defaultFS } from "@llamaindex/env";
 import _ from "lodash";
-import { filetypemime } from "magic-bytes.js";
 import type { ImageType } from "../Node.js";
 import { DEFAULT_SIMILARITY_TOP_K } from "../constants.js";
 import { VectorStoreQueryMode } from "../storage/vectorStore/types.js";
@@ -200,12 +199,13 @@ export function getTopKMMREmbeddings(
 }
 
 async function blobToDataUrl(input: Blob) {
+  const { fileTypeFromBuffer } = await import("file-type");
   const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
+  const type = await fileTypeFromBuffer(buffer);
+  if (!type) {
     throw new Error("Unsupported image type");
   }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
+  return "data:" + type.mime + ";base64," + buffer.toString("base64");
 }
 
 export async function readImage(input: ImageType) {

packages/core/src/index.edge.ts

@@ -1,31 +0,0 @@
-export * from "./ChatHistory.js";
-export * from "./GlobalsHelper.js";
-export * from "./Node.js";
-export * from "./OutputParser.js";
-export * from "./Prompt.js";
-export * from "./PromptHelper.js";
-export * from "./QuestionGenerator.js";
-export * from "./Response.js";
-export * from "./Retriever.js";
-export * from "./ServiceContext.js";
-export * from "./TextSplitter.js";
-export * from "./agent/index.js";
-export * from "./callbacks/CallbackManager.js";
-export * from "./cloud/index.js";
-export * from "./constants.js";
-export * from "./embeddings/index.js";
-export * from "./engines/chat/index.js";
-export * from "./engines/query/index.js";
-export * from "./evaluation/index.js";
-export * from "./extractors/index.js";
-export * from "./indices/index.js";
-export * from "./ingestion/index.js";
-export * from "./llm/index.js";
-export * from "./nodeParsers/index.js";
-export * from "./objects/index.js";
-export * from "./postprocessors/index.js";
-export * from "./prompts/index.js";
-export * from "./selectors/index.js";
-export * from "./synthesizers/index.js";
-export * from "./tools/index.js";
-export * from "./types.js";

packages/core/src/index.ts

@@ -1,3 +1,32 @@
-export * from "./index.edge.js";
+export * from "./ChatHistory.js";
+export * from "./GlobalsHelper.js";
+export * from "./Node.js";
+export * from "./OutputParser.js";
+export * from "./Prompt.js";
+export * from "./PromptHelper.js";
+export * from "./QuestionGenerator.js";
+export * from "./Response.js";
+export * from "./Retriever.js";
+export * from "./ServiceContext.js";
+export * from "./TextSplitter.js";
+export * from "./agent/index.js";
+export * from "./callbacks/CallbackManager.js";
+export * from "./cloud/index.js";
+export * from "./constants.js";
+export * from "./embeddings/index.js";
+export * from "./engines/chat/index.js";
+export * from "./engines/query/index.js";
+export * from "./evaluation/index.js";
+export * from "./extractors/index.js";
+export * from "./indices/index.js";
+export * from "./ingestion/index.js";
+export * from "./llm/index.js";
+export * from "./nodeParsers/index.js";
+export * from "./objects/index.js";
+export * from "./postprocessors/index.js";
+export * from "./prompts/index.js";
 export * from "./readers/index.js";
+export * from "./selectors/index.js";
 export * from "./storage/index.js";
+export * from "./synthesizers/index.js";
+export * from "./tools/index.js";

packages/core/src/indices/keyword/index.ts

@@ -13,9 +13,8 @@ import type { ServiceContext } from "../../ServiceContext.js";
 import { serviceContextFromDefaults } from "../../ServiceContext.js";
 import { RetrieverQueryEngine } from "../../engines/query/index.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
-import type { StorageContext } from "../../storage/StorageContext.js";
-import { storageContextFromDefaults } from "../../storage/StorageContext.js";
-import type { BaseDocumentStore } from "../../storage/docStore/types.js";
+import type { BaseDocumentStore, StorageContext } from "../../storage/index.js";
+import { storageContextFromDefaults } from "../../storage/index.js";
 import type { BaseSynthesizer } from "../../synthesizers/index.js";
 import type { BaseQueryEngine } from "../../types.js";
 import type { BaseIndexInit } from "../BaseIndex.js";

packages/core/src/indices/summary/index.ts

@@ -8,12 +8,12 @@ import type { ServiceContext } from "../../ServiceContext.js";
 import { serviceContextFromDefaults } from "../../ServiceContext.js";
 import { RetrieverQueryEngine } from "../../engines/query/index.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
-import type { StorageContext } from "../../storage/StorageContext.js";
-import { storageContextFromDefaults } from "../../storage/StorageContext.js";
 import type {
   BaseDocumentStore,
   RefDocInfo,
-} from "../../storage/docStore/types.js";
+  StorageContext,
+} from "../../storage/index.js";
+import { storageContextFromDefaults } from "../../storage/index.js";
 import type { BaseSynthesizer } from "../../synthesizers/index.js";
 import {
   CompactAndRefine,

packages/core/src/indices/vectorStore/index.ts

@@ -24,15 +24,15 @@ import { ClipEmbedding } from "../../embeddings/index.js";
 import { RetrieverQueryEngine } from "../../engines/query/RetrieverQueryEngine.js";
 import { runTransformations } from "../../ingestion/index.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/types.js";
-import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";
-import type { BaseIndexStore } from "../../storage/indexStore/types.js";
 import type {
+  BaseIndexStore,
   MetadataFilters,
+  StorageContext,
   VectorStore,
   VectorStoreQuery,
   VectorStoreQueryResult,
-} from "../../storage/vectorStore/types.js";
+} from "../../storage/index.js";
 import { VectorStoreQueryMode } from "../../storage/vectorStore/types.js";
 import type { BaseSynthesizer } from "../../synthesizers/types.js";
 import type { BaseQueryEngine } from "../../types.js";

packages/core/src/ingestion/IngestionPipeline.ts

@@ -1,12 +1,4 @@
-import type { PlatformApiClient } from "@llamaindex/cloud";
 import type { BaseNode, Document } from "../Node.js";
-import { getPipelineCreate } from "../cloud/config.js";
-import {
-  DEFAULT_PIPELINE_NAME,
-  DEFAULT_PROJECT_NAME,
-  type ClientParams,
-} from "../cloud/types.js";
-import { getAppBaseUrl, getClient } from "../cloud/utils.js";
 import type { BaseReader } from "../readers/type.js";
 import type { BaseDocumentStore } from "../storage/docStore/types.js";
 import type { VectorStore } from "../storage/vectorStore/types.js";
@@ -63,16 +55,11 @@ export class IngestionPipeline {
   docStoreStrategy: DocStoreStrategy = DocStoreStrategy.UPSERTS;
   cache?: IngestionCache;
   disableCache: boolean = false;
-  client?: PlatformApiClient;
-  clientParams?: ClientParams;
-  projectName: string = DEFAULT_PROJECT_NAME;
-  name: string = DEFAULT_PIPELINE_NAME;
 
   private _docStoreStrategy?: TransformComponent;
 
-  constructor(init?: Partial<IngestionPipeline> & ClientParams) {
+  constructor(init?: Partial<IngestionPipeline>) {
     Object.assign(this, init);
-    this.clientParams = { apiKey: init?.apiKey, baseUrl: init?.baseUrl };
     this._docStoreStrategy = createDocStoreStrategy(
       this.docStoreStrategy,
       this.docStore,
@@ -128,50 +115,4 @@ export class IngestionPipeline {
     }
     return nodes;
   }
-
-  private async getClient(): Promise<PlatformApiClient> {
-    if (!this.client) {
-      this.client = await getClient(this.clientParams);
-    }
-    return this.client;
-  }
-
-  async register(params: {
-    documents?: Document[];
-    nodes?: BaseNode[];
-    verbose?: boolean;
-  }): Promise<string> {
-    const client = await this.getClient();
-
-    const inputNodes = await this.prepareInput(params.documents, params.nodes);
-    const project = await client.project.upsertProject({
-      name: this.projectName,
-    });
-    if (!project.id) {
-      throw new Error("Project ID should be defined");
-    }
-
-    // upload
-    const pipeline = await client.project.upsertPipelineForProject(
-      project.id,
-      await getPipelineCreate({
-        pipelineName: this.name,
-        pipelineType: "PLAYGROUND",
-        transformations: this.transformations,
-        inputNodes,
-      }),
-    );
-    if (!pipeline.id) {
-      throw new Error("Pipeline ID must be defined");
-    }
-
-    // Print playground URL if not running remote
-    if (params.verbose) {
-      console.log(
-        `Pipeline available at: ${getAppBaseUrl(this.clientParams?.baseUrl)}/project/${project.id}/playground/${pipeline.id}`,
-      );
-    }
-
-    return pipeline.id;
-  }
 }

packages/core/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts

@@ -1,6 +1,5 @@
 import type { BaseNode } from "../../Node.js";
-import type { BaseDocumentStore } from "../../storage/docStore/types.js";
-import type { VectorStore } from "../../storage/vectorStore/types.js";
+import type { BaseDocumentStore, VectorStore } from "../../storage/index.js";
 import { classify } from "./classify.js";
 
 /**

packages/core/src/llm/LLM.ts

@@ -620,7 +620,6 @@ export const ALL_AVAILABLE_ANTHROPIC_LEGACY_MODELS = {
 export const ALL_AVAILABLE_V3_MODELS = {
   "claude-3-opus": { contextWindow: 200000 },
   "claude-3-sonnet": { contextWindow: 200000 },
-  "claude-3-haiku": { contextWindow: 200000 },
 };
 
 export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
@@ -631,7 +630,6 @@ export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
 const AVAILABLE_ANTHROPIC_MODELS_WITHOUT_DATE: { [key: string]: string } = {
   "claude-3-opus": "claude-3-opus-20240229",
   "claude-3-sonnet": "claude-3-sonnet-20240229",
-  "claude-3-haiku": "claude-3-haiku-20240307",
 } as { [key in keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS]: string };
 
 /**

packages/core/src/readers/LlamaParseReader.ts

@@ -1,5 +1,4 @@
 import { defaultFS, getEnv, type GenericFileSystem } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";
 
@@ -110,10 +109,11 @@ export class LlamaParseReader implements FileReader {
   }
 
   private async getMimeType(data: Buffer): Promise<string> {
-    const mimes = filetypemime(data);
-    if (!mimes.includes("application/pdf")) {
+    const { fileTypeFromBuffer } = await import("file-type");
+    const type = await fileTypeFromBuffer(data);
+    if (type?.mime !== "application/pdf") {
       throw new Error("Currently, only PDF files are supported.");
     }
-    return "application/pdf";
+    return type.mime;
   }
 }

packages/core/src/readers/SimpleDirectoryReader.edge.ts

@@ -1,126 +0,0 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
-import { Document } from "../Node.js";
-import { walk } from "../storage/FileSystem.js";
-import { TextFileReader } from "./TextFileReader.js";
-import type { BaseReader } from "./type.js";
-
-type ReaderCallback = (
-  category: "file" | "directory",
-  name: string,
-  status: ReaderStatus,
-  message?: string,
-) => boolean;
-enum ReaderStatus {
-  STARTED = 0,
-  COMPLETE,
-  ERROR,
-}
-
-export type SimpleDirectoryReaderLoadDataParams = {
-  directoryPath: string;
-  fs?: CompleteFileSystem;
-  defaultReader?: BaseReader | null;
-  fileExtToReader?: Record<string, BaseReader>;
-};
-
-/**
- * Read all the documents in a directory.
- * By default, supports the list of file types
- * in the FILE_EXT_TO_READER map.
- */
-export class SimpleDirectoryReader implements BaseReader {
-  constructor(private observer?: ReaderCallback) {}
-
-  async loadData(
-    params: SimpleDirectoryReaderLoadDataParams,
-  ): Promise<Document[]>;
-  async loadData(directoryPath: string): Promise<Document[]>;
-  async loadData(
-    params: SimpleDirectoryReaderLoadDataParams | string,
-  ): Promise<Document[]> {
-    if (typeof params === "string") {
-      params = { directoryPath: params };
-    }
-
-    const {
-      directoryPath,
-      fs = defaultFS,
-      defaultReader = new TextFileReader(),
-      fileExtToReader,
-    } = params;
-
-    // Observer can decide to skip the directory
-    if (
-      !this.doObserverCheck("directory", directoryPath, ReaderStatus.STARTED)
-    ) {
-      return [];
-    }
-
-    const docs: Document[] = [];
-    for await (const filePath of walk(fs, directoryPath)) {
-      try {
-        const fileExt = path.extname(filePath).slice(1).toLowerCase();
-
-        // Observer can decide to skip each file
-        if (!this.doObserverCheck("file", filePath, ReaderStatus.STARTED)) {
-          // Skip this file
-          continue;
-        }
-
-        let reader: BaseReader;
-
-        if (fileExtToReader && fileExt in fileExtToReader) {
-          reader = fileExtToReader[fileExt];
-        } else if (defaultReader != null) {
-          reader = defaultReader;
-        } else {
-          const msg = `No reader for file extension of ${filePath}`;
-          console.warn(msg);
-
-          // In an error condition, observer's false cancels the whole process.
-          if (
-            !this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)
-          ) {
-            return [];
-          }
-
-          continue;
-        }
-
-        const fileDocs = await reader.loadData(filePath, fs);
-
-        // Observer can still cancel addition of the resulting docs from this file
-        if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) {
-          docs.push(...fileDocs);
-        }
-      } catch (e) {
-        const msg = `Error reading file ${filePath}: ${e}`;
-        console.error(msg);
-
-        // In an error condition, observer's false cancels the whole process.
-        if (!this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)) {
-          return [];
-        }
-      }
-    }
-
-    // After successful import of all files, directory completion
-    // is only a notification for observer, cannot be cancelled.
-    this.doObserverCheck("directory", directoryPath, ReaderStatus.COMPLETE);
-
-    return docs;
-  }
-
-  private doObserverCheck(
-    category: "file" | "directory",
-    name: string,
-    status: ReaderStatus,
-    message?: string,
-  ): boolean {
-    if (this.observer) {
-      return this.observer(category, name, status, message);
-    }
-    return true;
-  }
-}

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -1,17 +1,40 @@
+import type { CompleteFileSystem } from "@llamaindex/env";
+import { defaultFS, path } from "@llamaindex/env";
 import { Document } from "../Node.js";
+import { walk } from "../storage/FileSystem.js";
 import { PapaCSVReader } from "./CSVReader.js";
 import { DocxReader } from "./DocxReader.js";
 import { HTMLReader } from "./HTMLReader.js";
 import { ImageReader } from "./ImageReader.js";
 import { MarkdownReader } from "./MarkdownReader.js";
 import { PDFReader } from "./PDFReader.js";
-import {
-  SimpleDirectoryReader as EdgeSimpleDirectoryReader,
-  type SimpleDirectoryReaderLoadDataParams,
-} from "./SimpleDirectoryReader.edge.js";
-import { TextFileReader } from "./TextFileReader.js";
 import type { BaseReader } from "./type.js";
 
+type ReaderCallback = (
+  category: "file" | "directory",
+  name: string,
+  status: ReaderStatus,
+  message?: string,
+) => boolean;
+enum ReaderStatus {
+  STARTED = 0,
+  COMPLETE,
+  ERROR,
+}
+
+/**
+ * Read a .txt file
+ */
+export class TextFileReader implements BaseReader {
+  async loadData(
+    file: string,
+    fs: CompleteFileSystem = defaultFS,
+  ): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file);
+    return [new Document({ text: dataBuffer, id_: file })];
+  }
+}
+
 export const FILE_EXT_TO_READER: Record<string, BaseReader> = {
   txt: new TextFileReader(),
   pdf: new PDFReader(),
@@ -26,12 +49,21 @@ export const FILE_EXT_TO_READER: Record<string, BaseReader> = {
   gif: new ImageReader(),
 };
 
+export type SimpleDirectoryReaderLoadDataParams = {
+  directoryPath: string;
+  fs?: CompleteFileSystem;
+  defaultReader?: BaseReader | null;
+  fileExtToReader?: Record<string, BaseReader>;
+};
+
 /**
  * Read all the documents in a directory.
  * By default, supports the list of file types
  * in the FILE_EXT_TO_READER map.
  */
-export class SimpleDirectoryReader extends EdgeSimpleDirectoryReader {
+export class SimpleDirectoryReader implements BaseReader {
+  constructor(private observer?: ReaderCallback) {}
+
   async loadData(
     params: SimpleDirectoryReaderLoadDataParams,
   ): Promise<Document[]>;
@@ -42,7 +74,85 @@ export class SimpleDirectoryReader extends EdgeSimpleDirectoryReader {
     if (typeof params === "string") {
       params = { directoryPath: params };
     }
-    params.fileExtToReader = params.fileExtToReader ?? FILE_EXT_TO_READER;
-    return super.loadData(params);
+
+    const {
+      directoryPath,
+      fs = defaultFS,
+      defaultReader = new TextFileReader(),
+      fileExtToReader = FILE_EXT_TO_READER,
+    } = params;
+
+    // Observer can decide to skip the directory
+    if (
+      !this.doObserverCheck("directory", directoryPath, ReaderStatus.STARTED)
+    ) {
+      return [];
+    }
+
+    const docs: Document[] = [];
+    for await (const filePath of walk(fs, directoryPath)) {
+      try {
+        const fileExt = path.extname(filePath).slice(1).toLowerCase();
+
+        // Observer can decide to skip each file
+        if (!this.doObserverCheck("file", filePath, ReaderStatus.STARTED)) {
+          // Skip this file
+          continue;
+        }
+
+        let reader: BaseReader;
+
+        if (fileExt in fileExtToReader) {
+          reader = fileExtToReader[fileExt];
+        } else if (defaultReader != null) {
+          reader = defaultReader;
+        } else {
+          const msg = `No reader for file extension of ${filePath}`;
+          console.warn(msg);
+
+          // In an error condition, observer's false cancels the whole process.
+          if (
+            !this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)
+          ) {
+            return [];
+          }
+
+          continue;
+        }
+
+        const fileDocs = await reader.loadData(filePath, fs);
+
+        // Observer can still cancel addition of the resulting docs from this file
+        if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) {
+          docs.push(...fileDocs);
+        }
+      } catch (e) {
+        const msg = `Error reading file ${filePath}: ${e}`;
+        console.error(msg);
+
+        // In an error condition, observer's false cancels the whole process.
+        if (!this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)) {
+          return [];
+        }
+      }
+    }
+
+    // After successful import of all files, directory completion
+    // is only a notification for observer, cannot be cancelled.
+    this.doObserverCheck("directory", directoryPath, ReaderStatus.COMPLETE);
+
+    return docs;
+  }
+
+  private doObserverCheck(
+    category: "file" | "directory",
+    name: string,
+    status: ReaderStatus,
+    message?: string,
+  ): boolean {
+    if (this.observer) {
+      return this.observer(category, name, status, message);
+    }
+    return true;
   }
 }

packages/core/src/readers/TextFileReader.ts

@@ -1,18 +0,0 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
-import { Document } from "../Node.js";
-import type { BaseReader } from "./type.js";
-
-/**
- * Read a .txt file
- */
-
-export class TextFileReader implements BaseReader {
-  async loadData(
-    file: string,
-    fs: CompleteFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
-    return [new Document({ text: dataBuffer, id_: file })];
-  }
-}

packages/core/src/readers/index.ts

@@ -9,5 +9,4 @@ export * from "./NotionReader.js";
 export * from "./PDFReader.js";
 export * from "./SimpleDirectoryReader.js";
 export * from "./SimpleMongoReader.js";
-export * from "./TextFileReader.js";
 export * from "./type.js";

packages/core/src/storage/index.ts

@@ -11,7 +11,6 @@ export { SimpleKVStore } from "./kvStore/SimpleKVStore.js";
 export * from "./kvStore/types.js";
 export { AstraDBVectorStore } from "./vectorStore/AstraDBVectorStore.js";
 export { ChromaVectorStore } from "./vectorStore/ChromaVectorStore.js";
-export { MilvusVectorStore } from "./vectorStore/MilvusVectorStore.js";
 export { MongoDBAtlasVectorSearch } from "./vectorStore/MongoDBAtlasVectorStore.js";
 export { PGVectorStore } from "./vectorStore/PGVectorStore.js";
 export { PineconeVectorStore } from "./vectorStore/PineconeVectorStore.js";

packages/core/src/storage/vectorStore/MilvusVectorStore.ts

@@ -1,214 +0,0 @@
-/* eslint-disable turbo/no-undeclared-env-vars */
-import type { ChannelOptions } from "@grpc/grpc-js";
-import {
-  DataType,
-  MilvusClient,
-  type ClientConfig,
-  type DeleteReq,
-  type RowData,
-} from "@zilliz/milvus2-sdk-node";
-import { BaseNode, MetadataMode, type Metadata } from "../../Node.js";
-import type {
-  VectorStore,
-  VectorStoreQuery,
-  VectorStoreQueryResult,
-} from "./types.js";
-import { metadataDictToNode, nodeToMetadata } from "./utils.js";
-
-export class MilvusVectorStore implements VectorStore {
-  public storesText: boolean = true;
-  public isEmbeddingQuery?: boolean;
-  private flatMetadata: boolean = true;
-
-  private milvusClient: MilvusClient;
-  private collectionInitialized = false;
-  private collectionName: string;
-
-  private idKey: string;
-  private contentKey: string;
-  private metadataKey: string;
-  private embeddingKey: string;
-
-  constructor(
-    init?: Partial<{ milvusClient: MilvusClient }> & {
-      params?: {
-        configOrAddress: ClientConfig | string;
-        ssl?: boolean;
-        username?: string;
-        password?: string;
-        channelOptions?: ChannelOptions;
-      };
-      collection?: string;
-      idKey?: string;
-      contentKey?: string;
-      metadataKey?: string;
-      embeddingKey?: string;
-    },
-  ) {
-    if (init?.milvusClient) {
-      this.milvusClient = init.milvusClient;
-    } else {
-      const configOrAddress =
-        init?.params?.configOrAddress ?? process.env.MILVUS_ADDRESS;
-      const ssl = init?.params?.ssl ?? process.env.MILVUS_SSL === "true";
-      const username = init?.params?.username ?? process.env.MILVUS_USERNAME;
-      const password = init?.params?.password ?? process.env.MILVUS_PASSWORD;
-
-      if (!configOrAddress) {
-        throw new Error("Must specify MILVUS_ADDRESS via env variable.");
-      }
-      this.milvusClient = new MilvusClient(
-        configOrAddress,
-        ssl,
-        username,
-        password,
-        init?.params?.channelOptions,
-      );
-    }
-
-    this.collectionName = init?.collection ?? "llamacollection";
-    this.idKey = init?.idKey ?? "id";
-    this.contentKey = init?.contentKey ?? "content";
-    this.metadataKey = init?.metadataKey ?? "metadata";
-    this.embeddingKey = init?.embeddingKey ?? "embedding";
-  }
-
-  public client(): MilvusClient {
-    return this.milvusClient;
-  }
-
-  private async createCollection() {
-    await this.milvusClient.createCollection({
-      collection_name: this.collectionName,
-      fields: [
-        {
-          name: this.idKey,
-          data_type: DataType.VarChar,
-          is_primary_key: true,
-          max_length: 200,
-        },
-        {
-          name: this.embeddingKey,
-          data_type: DataType.FloatVector,
-          dim: 1536,
-        },
-        {
-          name: this.contentKey,
-          data_type: DataType.VarChar,
-          max_length: 9000,
-        },
-        {
-          name: this.metadataKey,
-          data_type: DataType.JSON,
-        },
-      ],
-    });
-    await this.milvusClient.createIndex({
-      collection_name: this.collectionName,
-      field_name: this.embeddingKey,
-    });
-  }
-
-  private async ensureCollection(): Promise<void> {
-    if (!this.collectionInitialized) {
-      await this.milvusClient.connectPromise;
-
-      // Check collection exists
-      const isCollectionExist = await this.milvusClient.hasCollection({
-        collection_name: this.collectionName,
-      });
-      if (!isCollectionExist.value) {
-        await this.createCollection();
-      }
-
-      await this.milvusClient.loadCollectionSync({
-        collection_name: this.collectionName,
-      });
-      this.collectionInitialized = true;
-    }
-  }
-
-  public async add(nodes: BaseNode<Metadata>[]): Promise<string[]> {
-    await this.ensureCollection();
-
-    const result = await this.milvusClient.insert({
-      collection_name: this.collectionName,
-      data: nodes.map((node) => {
-        const metadata = nodeToMetadata(
-          node,
-          true,
-          this.contentKey,
-          this.flatMetadata,
-        );
-
-        const entry: RowData = {
-          [this.idKey]: node.id_,
-          [this.embeddingKey]: node.getEmbedding(),
-          [this.contentKey]: node.getContent(MetadataMode.NONE),
-          [this.metadataKey]: metadata,
-        };
-
-        return entry;
-      }),
-    });
-
-    if (!result.IDs) {
-      return [];
-    }
-
-    if ("int_id" in result.IDs) {
-      return result.IDs.int_id.data.map((i) => String(i));
-    }
-
-    return result.IDs.str_id.data.map((s) => String(s));
-  }
-
-  public async delete(
-    refDocId: string,
-    deleteOptions?: Omit<DeleteReq, "ids">,
-  ): Promise<void> {
-    this.ensureCollection();
-
-    await this.milvusClient.delete({
-      ids: [refDocId],
-      collection_name: this.collectionName,
-      ...deleteOptions,
-    });
-  }
-
-  public async query(
-    query: VectorStoreQuery,
-    _options?: any,
-  ): Promise<VectorStoreQueryResult> {
-    await this.ensureCollection();
-
-    const found = await this.milvusClient.search({
-      collection_name: this.collectionName,
-      limit: query.similarityTopK,
-      vector: query.queryEmbedding,
-    });
-
-    const nodes: BaseNode<Metadata>[] = [];
-    const similarities: number[] = [];
-    const ids: string[] = [];
-
-    found.results.forEach((result) => {
-      const node = metadataDictToNode(result.metadata);
-      node.setContent(result.content);
-      nodes.push(node);
-
-      similarities.push(result.score);
-      ids.push(String(result.id));
-    });
-
-    return {
-      nodes,
-      similarities,
-      ids,
-    };
-  }
-
-  public async persist() {
-    // no need to do anything
-  }
-}

packages/core/src/storage/vectorStore/PineconeVectorStore.ts

@@ -73,7 +73,7 @@ export class PineconeVectorStore implements VectorStore {
 
   async index() {
     const db: Pinecone = await this.getDb();
-    return db.index(this.indexName).namespace(this.namespace);
+    return await db.index(this.indexName);
   }
 
   /**
@@ -82,8 +82,8 @@ export class PineconeVectorStore implements VectorStore {
    * @returns The result of the delete query.
    */
   async clearIndex() {
-    const idx = await this.index();
-    return await idx.deleteAll();
+    const db: Pinecone = await this.getDb();
+    return await db.index(this.indexName).deleteAll();
   }
 
   /**
@@ -155,10 +155,12 @@ export class PineconeVectorStore implements VectorStore {
     };
 
     const idx = await this.index();
-    const results = await idx.query(options);
+    const results = await idx.namespace(this.namespace).query(options);
 
     const idList = results.matches.map((row) => row.id);
-    const records: FetchResponse<any> = await idx.fetch(idList);
+    const records: FetchResponse<any> = await idx
+      .namespace(this.namespace)
+      .fetch(idList);
     const rows = Object.values(records.records);
 
     const nodes = rows.map((row) => {

packages/core/src/storage/vectorStore/QdrantVectorStore.ts

@@ -289,7 +289,7 @@ export class QdrantVectorStore implements VectorStore {
    * @param query The VectorStoreQuery to be used
    */
   private async buildQueryFilter(query: VectorStoreQuery) {
-    if (!query.docIds && !query.queryStr && !query.filters) {
+    if (!query.docIds && !query.queryStr) {
       return null;
     }
 

packages/core/src/tools/WikipediaTool.ts

@@ -1,54 +0,0 @@
-import { default as wiki } from "wikipedia";
-import type { BaseTool, ToolMetadata } from "../types.js";
-
-export type WikipediaToolParams = {
-  metadata?: ToolMetadata;
-};
-
-type WikipediaCallParams = {
-  query: string;
-  lang?: string;
-};
-
-const DEFAULT_META_DATA: ToolMetadata = {
-  name: "wikipedia_tool",
-  description: "A tool that uses a query engine to search Wikipedia.",
-  parameters: {
-    type: "object",
-    properties: {
-      query: {
-        type: "string",
-        description: "The query to search for",
-      },
-    },
-    required: ["query"],
-  },
-};
-
-export class WikipediaTool implements BaseTool {
-  private readonly DEFAULT_LANG = "en";
-  metadata: ToolMetadata;
-
-  constructor(params?: WikipediaToolParams) {
-    this.metadata = params?.metadata || DEFAULT_META_DATA;
-  }
-
-  async loadData(
-    page: string,
-    lang: string = this.DEFAULT_LANG,
-  ): Promise<string> {
-    wiki.default.setLang(lang);
-    const pageResult = await wiki.default.page(page, { autoSuggest: false });
-    const content = await pageResult.content();
-    return content;
-  }
-
-  async call({
-    query,
-    lang = this.DEFAULT_LANG,
-  }: WikipediaCallParams): Promise<string> {
-    const searchResult = await wiki.default.search(query);
-    if (searchResult.results.length === 0) return "No search results.";
-    return await this.loadData(searchResult.results[0].title, lang);
-  }
-}

packages/core/src/tools/index.ts

@@ -1,5 +1,4 @@
 export * from "./QueryEngineTool.js";
-export * from "./WikipediaTool.js";
 export * from "./functionTool.js";
 export * from "./types.js";
 export * from "./utils.js";

packages/core/tests/CHANGELOG.md

@@ -1,13 +0,0 @@
-# @llamaindex/core-test
-
-## 0.0.2
-
-### Patch Changes
-
-- 484a710: - Add missing exports:
-  - `IndexStructType`,
-  - `IndexDict`,
-  - `jsonToIndexStruct`,
-  - `IndexList`,
-  - `IndexStruct`
-  - Fix `IndexDict.toJson()` method

packages/core/tests/StorageContext.test.ts

@@ -1,40 +1,21 @@
-import {
-  storageContextFromDefaults,
-  type StorageContext,
-} from "llamaindex/storage/StorageContext";
+import { storageContextFromDefaults } from "llamaindex/storage/StorageContext";
 import { existsSync, rmSync } from "node:fs";
-import { mkdtemp } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import {
-  afterAll,
-  beforeAll,
-  describe,
-  expect,
-  test,
-  vi,
-  vitest,
-} from "vitest";
-const testDir = await mkdtemp(join(tmpdir(), "test-"));
+import { describe, expect, test, vi, vitest } from "vitest";
+
 vitest.spyOn(console, "error");
 
 describe("StorageContext", () => {
-  let storageContext: StorageContext;
-
-  beforeAll(async () => {
-    storageContext = await storageContextFromDefaults({
-      persistDir: testDir,
-    });
-  });
-
   test("initializes", async () => {
     vi.mocked(console.error).mockImplementation(() => {}); // silence console.error
 
-    expect(existsSync(testDir)).toBe(true);
-    expect(storageContext).toBeDefined();
-  });
+    const storageContext = await storageContextFromDefaults({
+      persistDir: "/tmp/test_dir",
+    });
 
-  afterAll(() => {
-    rmSync(testDir, { recursive: true });
+    expect(existsSync("/tmp/test_dir")).toBe(true);
+    expect(storageContext).toBeDefined();
+
+    // cleanup
+    rmSync("/tmp/test_dir", { recursive: true });
   });
 });

packages/core/tests/indices/SummaryIndex.test.ts

@@ -1,39 +1,48 @@
+import type { ServiceContext } from "llamaindex";
 import {
   Document,
+  OpenAI,
+  OpenAIEmbedding,
   SummaryIndex,
   VectorStoreIndex,
+  serviceContextFromDefaults,
   storageContextFromDefaults,
-  type ServiceContext,
-  type StorageContext,
 } from "llamaindex";
-import { rmSync } from "node:fs";
-import { mkdtemp } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
-
-const testDir = await mkdtemp(join(tmpdir(), "test-"));
+import { beforeAll, describe, expect, it, vi } from "vitest";
+import {
+  mockEmbeddingModel,
+  mockLlmGeneration,
+} from "../utility/mockOpenAI.js";
 
+// Mock the OpenAI getOpenAISession function during testing
 vi.mock("llamaindex/llm/open_ai", () => {
   return {
     getOpenAISession: vi.fn().mockImplementation(() => null),
   };
 });
 
-import { mockServiceContext } from "../utility/mockServiceContext.js";
-
 describe("SummaryIndex", () => {
   let serviceContext: ServiceContext;
-  let storageContext: StorageContext;
 
-  beforeAll(async () => {
-    serviceContext = mockServiceContext();
-    storageContext = await storageContextFromDefaults({
-      persistDir: testDir,
+  beforeAll(() => {
+    const embeddingModel = new OpenAIEmbedding();
+    const llm = new OpenAI();
+
+    mockEmbeddingModel(embeddingModel);
+    mockLlmGeneration({ languageModel: llm });
+
+    const ctx = serviceContextFromDefaults({
+      embedModel: embeddingModel,
+      llm,
     });
+
+    serviceContext = ctx;
   });
 
   it("SummaryIndex and VectorStoreIndex must be able to share the same storage context", async () => {
+    const storageContext = await storageContextFromDefaults({
+      persistDir: "/tmp/test_dir",
+    });
     const documents = [new Document({ text: "lorem ipsem", id_: "1" })];
     const vectorIndex = await VectorStoreIndex.fromDocuments(documents, {
       serviceContext,
@@ -46,8 +55,4 @@ describe("SummaryIndex", () => {
     });
     expect(summaryIndex).toBeDefined();
   });
-
-  afterAll(() => {
-    rmSync(testDir, { recursive: true });
-  });
 });

packages/core/tests/indices/VectorStoreIndex.test.ts

@@ -1,70 +0,0 @@
-import type { ServiceContext, StorageContext } from "llamaindex";
-import {
-  Document,
-  VectorStoreIndex,
-  storageContextFromDefaults,
-} from "llamaindex";
-import { rmSync } from "node:fs";
-import { mkdtemp } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterAll, beforeAll, describe, expect, test, vi } from "vitest";
-
-const testDir = await mkdtemp(join(tmpdir(), "test-"));
-
-vi.mock("llamaindex/llm/open_ai", () => {
-  return {
-    getOpenAISession: vi.fn().mockImplementation(() => null),
-  };
-});
-
-import { mockServiceContext } from "../utility/mockServiceContext.js";
-
-describe.sequential("VectorStoreIndex", () => {
-  let serviceContext: ServiceContext;
-  let storageContext: StorageContext;
-  let testStrategy: (
-    // strategy?: DocStoreStrategy,
-    runs?: number,
-  ) => Promise<Array<number>>;
-
-  beforeAll(async () => {
-    serviceContext = mockServiceContext();
-    storageContext = await storageContextFromDefaults({
-      persistDir: testDir,
-    });
-    testStrategy = async (
-      // strategy?: DocStoreStrategy,
-      runs: number = 2,
-    ): Promise<Array<number>> => {
-      const documents = [new Document({ text: "lorem ipsem", id_: "1" })];
-      const entries = [];
-      for (let i = 0; i < runs; i++) {
-        await VectorStoreIndex.fromDocuments(documents, {
-          serviceContext,
-          storageContext,
-          // docStoreStrategy: strategy,
-        });
-        const docs = await storageContext.docStore.docs();
-        entries.push(Object.keys(docs).length);
-      }
-      return entries;
-    };
-  });
-
-  test("fromDocuments does not stores duplicates per default", async () => {
-    const entries = await testStrategy();
-    expect(entries[0]).toBe(entries[1]);
-  });
-
-  // test("fromDocuments ignores duplicates in upserts", async () => {
-  //   const entries = await testStrategy(DocStoreStrategy.DUPLICATES_ONLY);
-  //   expect(entries[0]).toBe(entries[1]);
-  // });
-
-  afterAll(async () => {
-    // TODO: VectorStoreIndex.fromDocuments running twice is causing a cleanup issue
-    await new Promise((resolve) => setTimeout(resolve, 100));
-    rmSync(testDir, { recursive: true });
-  });
-});

packages/core/tests/objects/ObjectIndex.test.ts

@@ -2,10 +2,17 @@ import type { ServiceContext } from "llamaindex";
 import {
   FunctionTool,
   ObjectIndex,
+  OpenAI,
+  OpenAIEmbedding,
   SimpleToolNodeMapping,
   VectorStoreIndex,
+  serviceContextFromDefaults,
 } from "llamaindex";
 import { beforeAll, describe, expect, test, vi } from "vitest";
+import {
+  mockEmbeddingModel,
+  mockLlmGeneration,
+} from "../utility/mockOpenAI.js";
 
 vi.mock("llamaindex/llm/open_ai", () => {
   return {
@@ -13,13 +20,22 @@ vi.mock("llamaindex/llm/open_ai", () => {
   };
 });
 
-import { mockServiceContext } from "../utility/mockServiceContext.js";
-
 describe("ObjectIndex", () => {
   let serviceContext: ServiceContext;
 
   beforeAll(() => {
-    serviceContext = mockServiceContext();
+    const embeddingModel = new OpenAIEmbedding();
+    const llm = new OpenAI();
+
+    mockEmbeddingModel(embeddingModel);
+    mockLlmGeneration({ languageModel: llm });
+
+    const ctx = serviceContextFromDefaults({
+      embedModel: embeddingModel,
+      llm,
+    });
+
+    serviceContext = ctx;
   });
 
   test("test_object_with_tools", async () => {

packages/core/tests/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core-test",
   "private": true,
-  "version": "0.0.2",
+  "version": "0.0.1",
   "type": "module",
   "scripts": {
     "test": "vitest run"

packages/core/tests/utility/mockServiceContext.ts

@@ -1,23 +0,0 @@
-import {
-  OpenAI,
-  OpenAIEmbedding,
-  serviceContextFromDefaults,
-} from "llamaindex";
-
-import {
-  mockEmbeddingModel,
-  mockLlmGeneration,
-} from "../utility/mockOpenAI.js";
-
-export function mockServiceContext() {
-  const embeddingModel = new OpenAIEmbedding();
-  const llm = new OpenAI();
-
-  mockEmbeddingModel(embeddingModel);
-  mockLlmGeneration({ languageModel: llm });
-
-  return serviceContextFromDefaults({
-    embedModel: embeddingModel,
-    llm,
-  });
-}

packages/create-llama/.eslintrc

@@ -0,0 +1,19 @@
+{
+  "root": false,
+  "rules": {
+    "turbo/no-undeclared-env-vars": [
+      "error",
+      {
+        "allowList": [
+          "OPENAI_API_KEY",
+          "npm_config_user_agent",
+          "http_proxy",
+          "https_proxy",
+          "MODEL",
+          "NEXT_PUBLIC_CHAT_API",
+          "NEXT_PUBLIC_MODEL"
+        ]
+      }
+    ]
+  }
+}

packages/create-llama/CHANGELOG.md

@@ -0,0 +1,160 @@
+# create-llama
+
+## 0.0.26
+
+### Patch Changes
+
+- 09d532e: feat: generate llama pack project from llama index
+- cfdd6db: feat: add pinecone support to create llama
+- ef25d69: upgrade llama-index package to version v0.10.7 for create-llama app
+- 50dfd7b: update fastapi for CVE-2024-24762
+
+## 0.0.25
+
+### Patch Changes
+
+- d06a85b: Add option to create an agent by selecting tools (Google, Wikipedia)
+- 7b7329b: Added latest turbo models for GPT-3.5 and GPT 4
+
+## 0.0.24
+
+### Patch Changes
+
+- ba95ca3: Use condense plus context chat engine for FastAPI as default
+
+## 0.0.23
+
+### Patch Changes
+
+- c680af6: Fixed issues with locating templates path
+
+## 0.0.22
+
+### Patch Changes
+
+- 6dd401e: Add an option to provide an URL and chat with the website data (FastAPI only)
+- e9b87ef: Select a folder as data source and support more file types (.pdf, .doc, .docx, .xls, .xlsx, .csv)
+
+## 0.0.20
+
+### Patch Changes
+
+- 27d55fd: Add an option to provide an URL and chat with the website data
+
+## 0.0.19
+
+### Patch Changes
+
+- 3a29a80: Add node_modules to gitignore in Express backends
+- fe03aaa: feat: generate llama pack example
+
+## 0.0.18
+
+### Patch Changes
+
+- 88d3b41: fix packaging
+
+## 0.0.17
+
+### Patch Changes
+
+- fa17f7e: Add an option that allows the user to run the generated app
+- 9e5d8e1: Add an option to select a local PDF file as data source
+
+## 0.0.16
+
+### Patch Changes
+
+- a73942d: Fix: Bundle mongo dependency with NextJS
+- 9492cc6: Feat: Added option to automatically install dependencies (for Python and TS)
+- f74dea5: Feat: Show images in chat messages using GPT4 Vision (Express and NextJS only)
+
+## 0.0.15
+
+### Patch Changes
+
+- 8e124e5: feat: support showing image on chat message
+
+## 0.0.14
+
+### Patch Changes
+
+- 2e6b36e: fix: re-organize file structure
+- 2b356c8: fix: relative path incorrect
+
+## 0.0.13
+
+### Patch Changes
+
+- Added PostgreSQL vector store (for Typescript and Python)
+- Improved async handling in FastAPI
+
+## 0.0.12
+
+### Patch Changes
+
+- 9c5e22a: Added cross-env so frontends with Express/FastAPI backends are working under Windows
+- 5ab65eb: Bring Python templates with TS templates to feature parity
+- 9c5e22a: Added vector DB selector to create-llama (starting with MongoDB support)
+
+## 0.0.11
+
+### Patch Changes
+
+- 2aeb341: - Added option to create a new project based on community templates
+  - Added OpenAI model selector for NextJS projects
+  - Added GPT4 Vision support (and file upload)
+
+## 0.0.10
+
+### Patch Changes
+
+- Bugfixes (thanks @marcusschiesser)
+
+## 0.0.9
+
+### Patch Changes
+
+- acfe232: Deployment fixes (thanks @seldo)
+
+## 0.0.8
+
+### Patch Changes
+
+- 8cdb07f: Fix Next deployment (thanks @seldo and @marcusschiesser)
+
+## 0.0.7
+
+### Patch Changes
+
+- 9f9f293: Added more to README and made it easier to switch models (thanks @seldo)
+
+## 0.0.6
+
+### Patch Changes
+
+- 4431ec7: Label bug fix (thanks @marcusschiesser)
+
+## 0.0.5
+
+### Patch Changes
+
+- 25257f4: Fix issue where it doesn't find OpenAI Key when running npm run generate (#182) (thanks @RayFernando1337)
+
+## 0.0.4
+
+### Patch Changes
+
+- 031e926: Update create-llama readme (thanks @logan-markewich)
+
+## 0.0.3
+
+### Patch Changes
+
+- 91b42a3: change version (thanks @marcusschiesser)
+
+## 0.0.2
+
+### Patch Changes
+
+- e2a6805: Hello Create Llama (thanks @marcusschiesser)

packages/create-llama/LICENSE.md

@@ -0,0 +1,9 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 LlamaIndex, Vercel, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

packages/create-llama/README.md

@@ -0,0 +1,126 @@
+# Create LlamaIndex App
+
+The easiest way to get started with [LlamaIndex](https://www.llamaindex.ai/) is by using `create-llama`. This CLI tool enables you to quickly start building a new LlamaIndex application, with everything set up for you.
+
+Just run
+
+```bash
+npx create-llama@latest
+```
+
+to get started, or see below for more options. Once your app is generated, run
+
+```bash
+npm run dev
+```
+
+to start the development server. You can then visit [http://localhost:3000](http://localhost:3000) to see your app.
+
+## What you'll get
+
+- A Next.js-powered front-end. The app is set up as a chat interface that can answer questions about your data (see below)
+  - You can style it with HTML and CSS, or you can optionally use components from [shadcn/ui](https://ui.shadcn.com/)
+- Your choice of 3 back-ends:
+  - **Next.js**: if you select this option, you’ll have a full stack Next.js application that you can deploy to a host like [Vercel](https://vercel.com/) in just a few clicks. This uses [LlamaIndex.TS](https://www.npmjs.com/package/llamaindex), our TypeScript library.
+  - **Express**: if you want a more traditional Node.js application you can generate an Express backend. This also uses LlamaIndex.TS.
+  - **Python FastAPI**: if you select this option you’ll get a backend powered by the [llama-index python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
+- The back-end has a single endpoint that allows you to send the state of your chat and receive additional responses
+- You can choose whether you want a streaming or non-streaming back-end (if you're not sure, we recommend streaming)
+- You can choose whether you want to use `ContextChatEngine` or `SimpleChatEngine`
+  - `SimpleChatEngine` will just talk to the LLM directly without using your data
+  - `ContextChatEngine` will use your data to answer questions (see below).
+- The app uses OpenAI by default, so you'll need an OpenAI API key, or you can customize it to use any of the dozens of LLMs we support.
+
+## Using your data
+
+If you've enabled `ContextChatEngine`, you can supply your own data and the app will index it and answer questions. Your generated app will have a folder called `data`:
+
+- With the Next.js backend this is `./data`
+- With the Express or Python backend this is in `./backend/data`
+
+The app will ingest any supported files you put in this directory. Your Next.js and Express apps use LlamaIndex.TS so they will be able to ingest any PDF, text, CSV, Markdown, Word and HTML files. The Python backend can read even more types, including video and audio files.
+
+Before you can use your data, you need to index it. If you're using the Next.js or Express apps, run:
+
+```bash
+npm run generate
+```
+
+Then re-start your app. Remember you'll need to re-run `generate` if you add new files to your `data` folder. If you're using the Python backend, you can trigger indexing of your data by deleting the `./storage` folder and re-starting the app.
+
+## Don't want a front-end?
+
+It's optional! If you've selected the Python or Express back-ends, just delete the `frontend` folder and you'll get an API without any front-end code.
+
+## Customizing the LLM
+
+By default the app will use OpenAI's gpt-3.5-turbo model. If you want to use GPT-4, you can modify this by editing a file:
+
+- In the Next.js backend, edit `./app/api/chat/route.ts` and replace `gpt-3.5-turbo` with `gpt-4`
+- In the Express backend, edit `./backend/src/controllers/chat.controller.ts` and likewise replace `gpt-3.5-turbo` with `gpt-4`
+- In the Python backend, edit `./backend/app/utils/index.py` and once again replace `gpt-3.5-turbo` with `gpt-4`
+
+You can also replace OpenAI with one of our [dozens of other supported LLMs](https://docs.llamaindex.ai/en/stable/module_guides/models/llms/modules.html).
+
+## Example
+
+The simplest thing to do is run `create-llama` in interactive mode:
+
+```bash
+npx create-llama@latest
+# or
+npm create llama@latest
+# or
+yarn create llama
+# or
+pnpm create llama@latest
+```
+
+You will be asked for the name of your project, along with other configuration options, something like this:
+
+```bash
+>> npm create llama@latest
+Need to install the following packages:
+  create-llama@latest
+Ok to proceed? (y) y
+✔ What is your project named? … my-app
+✔ Which template would you like to use? › Chat with streaming
+✔ Which framework would you like to use? › NextJS
+✔ Which UI would you like to use? › Just HTML
+✔ Which chat engine would you like to use? › ContextChatEngine
+✔ Please provide your OpenAI API key (leave blank to skip): …
+✔ Would you like to use ESLint? … No / Yes
+Creating a new LlamaIndex app in /home/my-app.
+```
+
+### Running non-interactively
+
+You can also pass command line arguments to set up a new project
+non-interactively. See `create-llama --help`:
+
+```bash
+create-llama <project-directory> [options]
+
+Options:
+  -V, --version                      output the version number
+
+  --use-npm
+
+    Explicitly tell the CLI to bootstrap the app using npm
+
+  --use-pnpm
+
+    Explicitly tell the CLI to bootstrap the app using pnpm
+
+  --use-yarn
+
+    Explicitly tell the CLI to bootstrap the app using Yarn
+
+```
+
+## LlamaIndex Documentation
+
+- [TS/JS docs](https://ts.llamaindex.ai/)
+- [Python docs](https://docs.llamaindex.ai/en/stable/)
+
+Inspired by and adapted from [create-next-app](https://github.com/vercel/next.js/tree/canary/packages/create-next-app)

packages/create-llama/create-app.ts

@@ -0,0 +1,144 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import path from "path";
+import { green, yellow } from "picocolors";
+import { tryGitInit } from "./helpers/git";
+import { isFolderEmpty } from "./helpers/is-folder-empty";
+import { getOnline } from "./helpers/is-online";
+import { isWriteable } from "./helpers/is-writeable";
+import { makeDir } from "./helpers/make-dir";
+
+import fs from "fs";
+import terminalLink from "terminal-link";
+import type { InstallTemplateArgs } from "./helpers";
+import { installTemplate } from "./helpers";
+import { templatesDir } from "./helpers/dir";
+import { toolsRequireConfig } from "./helpers/tools";
+
+export type InstallAppArgs = Omit<
+  InstallTemplateArgs,
+  "appName" | "root" | "isOnline" | "customApiPath"
+> & {
+  appPath: string;
+  frontend: boolean;
+};
+
+export async function createApp({
+  template,
+  framework,
+  engine,
+  ui,
+  appPath,
+  packageManager,
+  eslint,
+  frontend,
+  openAiKey,
+  llamaCloudKey,
+  model,
+  embeddingModel,
+  communityProjectPath,
+  llamapack,
+  vectorDb,
+  externalPort,
+  postInstallAction,
+  dataSource,
+  tools,
+}: InstallAppArgs): Promise<void> {
+  const root = path.resolve(appPath);
+
+  if (!(await isWriteable(path.dirname(root)))) {
+    console.error(
+      "The application path is not writable, please check folder permissions and try again.",
+    );
+    console.error(
+      "It is likely you do not have write permissions for this folder.",
+    );
+    process.exit(1);
+  }
+
+  const appName = path.basename(root);
+
+  await makeDir(root);
+  if (!isFolderEmpty(root, appName)) {
+    process.exit(1);
+  }
+
+  const useYarn = packageManager === "yarn";
+  const isOnline = !useYarn || (await getOnline());
+
+  console.log(`Creating a new LlamaIndex app in ${green(root)}.`);
+  console.log();
+
+  const args = {
+    appName,
+    root,
+    template,
+    framework,
+    engine,
+    ui,
+    packageManager,
+    isOnline,
+    eslint,
+    openAiKey,
+    llamaCloudKey,
+    model,
+    embeddingModel,
+    communityProjectPath,
+    llamapack,
+    vectorDb,
+    externalPort,
+    postInstallAction,
+    dataSource,
+    tools,
+  };
+
+  if (frontend) {
+    // install backend
+    const backendRoot = path.join(root, "backend");
+    await makeDir(backendRoot);
+    await installTemplate({ ...args, root: backendRoot, backend: true });
+    // install frontend
+    const frontendRoot = path.join(root, "frontend");
+    await makeDir(frontendRoot);
+    await installTemplate({
+      ...args,
+      root: frontendRoot,
+      framework: "nextjs",
+      customApiPath: `http://localhost:${externalPort ?? 8000}/api/chat`,
+      backend: false,
+    });
+    // copy readme for fullstack
+    await fs.promises.copyFile(
+      path.join(templatesDir, "README-fullstack.md"),
+      path.join(root, "README.md"),
+    );
+  } else {
+    await installTemplate({ ...args, backend: true, forBackend: framework });
+  }
+
+  process.chdir(root);
+  if (tryGitInit(root)) {
+    console.log("Initialized a git repository.");
+    console.log();
+  }
+
+  if (toolsRequireConfig(tools)) {
+    console.log(
+      yellow(
+        `You have selected tools that require configuration. Please configure them in the ${terminalLink(
+          "tools_config.json",
+          `file://${root}/tools_config.json`,
+        )} file.`,
+      ),
+    );
+  }
+  console.log("");
+  console.log(`${green("Success!")} Created ${appName} at ${appPath}`);
+
+  console.log(
+    `Now have a look at the ${terminalLink(
+      "README.md",
+      `file://${root}/README.md`,
+    )} and learn how to get started.`,
+  );
+  console.log();
+}

packages/create-llama/e2e/basic.spec.ts

@@ -0,0 +1,145 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import { expect, test } from "@playwright/test";
+import { ChildProcess } from "child_process";
+import fs from "fs";
+import path from "path";
+import type {
+  TemplateEngine,
+  TemplateFramework,
+  TemplatePostInstallAction,
+  TemplateType,
+  TemplateUI,
+} from "../helpers";
+import { createTestDir, runCreateLlama, type AppType } from "./utils";
+
+const templateTypes: TemplateType[] = ["streaming", "simple"];
+const templateFrameworks: TemplateFramework[] = [
+  "nextjs",
+  "express",
+  "fastapi",
+];
+const templateEngines: TemplateEngine[] = ["simple", "context"];
+const templateUIs: TemplateUI[] = ["shadcn", "html"];
+const templatePostInstallActions: TemplatePostInstallAction[] = [
+  "none",
+  "runApp",
+];
+
+for (const templateType of templateTypes) {
+  for (const templateFramework of templateFrameworks) {
+    for (const templateEngine of templateEngines) {
+      for (const templateUI of templateUIs) {
+        for (const templatePostInstallAction of templatePostInstallActions) {
+          if (templateFramework === "nextjs" && templateType === "simple") {
+            // nextjs doesn't support simple templates - skip tests
+            continue;
+          }
+          const appType: AppType =
+            templateFramework === "express" || templateFramework === "fastapi"
+              ? templateType === "simple"
+                ? "--no-frontend" // simple templates don't have frontends
+                : "--frontend"
+              : "";
+          if (appType === "--no-frontend" && templateUI !== "html") {
+            // if there's no frontend, don't iterate over UIs
+            continue;
+          }
+          test.describe(`try create-llama ${templateType} ${templateFramework} ${templateEngine} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
+            let port: number;
+            let externalPort: number;
+            let cwd: string;
+            let name: string;
+            let appProcess: ChildProcess;
+            // Only test without using vector db for now
+            const vectorDb = "none";
+
+            test.beforeAll(async () => {
+              port = Math.floor(Math.random() * 10000) + 10000;
+              externalPort = port + 1;
+              cwd = await createTestDir();
+              const result = await runCreateLlama(
+                cwd,
+                templateType,
+                templateFramework,
+                templateEngine,
+                templateUI,
+                vectorDb,
+                appType,
+                port,
+                externalPort,
+                templatePostInstallAction,
+              );
+              name = result.projectName;
+              appProcess = result.appProcess;
+            });
+
+            test("App folder should exist", async () => {
+              const dirExists = fs.existsSync(path.join(cwd, name));
+              expect(dirExists).toBeTruthy();
+            });
+            test("Frontend should have a title", async ({ page }) => {
+              test.skip(templatePostInstallAction !== "runApp");
+              test.skip(appType === "--no-frontend");
+              await page.goto(`http://localhost:${port}`);
+              await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
+            });
+
+            test("Frontend should be able to submit a message and receive a response", async ({
+              page,
+            }) => {
+              test.skip(templatePostInstallAction !== "runApp");
+              test.skip(appType === "--no-frontend");
+              await page.goto(`http://localhost:${port}`);
+              await page.fill("form input", "hello");
+              const [response] = await Promise.all([
+                page.waitForResponse(
+                  (res) => {
+                    return (
+                      res.url().includes("/api/chat") && res.status() === 200
+                    );
+                  },
+                  {
+                    timeout: 1000 * 60,
+                  },
+                ),
+                page.click("form button[type=submit]"),
+              ]);
+              const text = await response.text();
+              console.log("AI response when submitting message: ", text);
+              expect(response.ok()).toBeTruthy();
+            });
+
+            test("Backend should response when calling API", async ({
+              request,
+            }) => {
+              test.skip(templatePostInstallAction !== "runApp");
+              test.skip(appType !== "--no-frontend");
+              const backendPort = appType === "" ? port : externalPort;
+              const response = await request.post(
+                `http://localhost:${backendPort}/api/chat`,
+                {
+                  data: {
+                    messages: [
+                      {
+                        role: "user",
+                        content: "Hello",
+                      },
+                    ],
+                  },
+                },
+              );
+              const text = await response.text();
+              console.log("AI response when calling API: ", text);
+              expect(response.ok()).toBeTruthy();
+            });
+
+            // clean processes
+            test.afterAll(async () => {
+              appProcess?.kill();
+            });
+          });
+        }
+      }
+    }
+  }
+}

packages/create-llama/e2e/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "es2019",
+    "module": "esnext",
+    "moduleResolution": "node",
+    "strict": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "declaration": false,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "incremental": true,
+    "tsBuildInfoFile": "./lib/.tsbuildinfo"
+  },
+  "include": ["./**/*.ts"]
+}

packages/create-llama/e2e/utils.ts

@@ -0,0 +1,181 @@
+import { ChildProcess, exec } from "child_process";
+import crypto from "node:crypto";
+import { mkdir } from "node:fs/promises";
+import * as path from "path";
+import waitPort from "wait-port";
+import {
+  TemplateEngine,
+  TemplateFramework,
+  TemplatePostInstallAction,
+  TemplateType,
+  TemplateUI,
+  TemplateVectorDB,
+} from "../helpers";
+
+export type AppType = "--frontend" | "--no-frontend" | "";
+const MODEL = "gpt-3.5-turbo";
+const EMBEDDING_MODEL = "text-embedding-ada-002";
+export type CreateLlamaResult = {
+  projectName: string;
+  appProcess: ChildProcess;
+};
+
+// eslint-disable-next-line max-params
+export async function checkAppHasStarted(
+  frontend: boolean,
+  framework: TemplateFramework,
+  port: number,
+  externalPort: number,
+  timeout: number,
+) {
+  if (frontend) {
+    await Promise.all([
+      waitPort({
+        host: "localhost",
+        port: port,
+        timeout,
+      }),
+      waitPort({
+        host: "localhost",
+        port: externalPort,
+        timeout,
+      }),
+    ]).catch((err) => {
+      console.error(err);
+      throw err;
+    });
+  } else {
+    let wPort: number;
+    if (framework === "nextjs") {
+      wPort = port;
+    } else {
+      wPort = externalPort;
+    }
+    await waitPort({
+      host: "localhost",
+      port: wPort,
+      timeout,
+    }).catch((err) => {
+      console.error(err);
+      throw err;
+    });
+  }
+}
+
+// eslint-disable-next-line max-params
+export async function runCreateLlama(
+  cwd: string,
+  templateType: TemplateType,
+  templateFramework: TemplateFramework,
+  templateEngine: TemplateEngine,
+  templateUI: TemplateUI,
+  vectorDb: TemplateVectorDB,
+  appType: AppType,
+  port: number,
+  externalPort: number,
+  postInstallAction: TemplatePostInstallAction,
+): Promise<CreateLlamaResult> {
+  const createLlama = path.join(
+    __dirname,
+    "..",
+    "output",
+    "package",
+    "dist",
+    "index.js",
+  );
+
+  const name = [
+    templateType,
+    templateFramework,
+    templateEngine,
+    templateUI,
+    appType,
+  ].join("-");
+  const command = [
+    "node",
+    createLlama,
+    name,
+    "--template",
+    templateType,
+    "--framework",
+    templateFramework,
+    "--engine",
+    templateEngine,
+    "--ui",
+    templateUI,
+    "--vector-db",
+    vectorDb,
+    "--model",
+    MODEL,
+    "--embedding-model",
+    EMBEDDING_MODEL,
+    "--open-ai-key",
+    process.env.OPENAI_API_KEY || "testKey",
+    appType,
+    "--eslint",
+    "--use-npm",
+    "--port",
+    port,
+    "--external-port",
+    externalPort,
+    "--post-install-action",
+    postInstallAction,
+    "--tools",
+    "none",
+    "--no-llama-parse",
+  ].join(" ");
+  console.log(`running command '${command}' in ${cwd}`);
+  const appProcess = exec(command, {
+    cwd,
+    env: {
+      ...process.env,
+    },
+  });
+  appProcess.stderr?.on("data", (data) => {
+    console.log(data.toString());
+  });
+  appProcess.on("exit", (code) => {
+    if (code !== 0 && code !== null) {
+      throw new Error(`create-llama command was failed!`);
+    }
+  });
+
+  // Wait for app to start
+  if (postInstallAction === "runApp") {
+    await checkAppHasStarted(
+      appType === "--frontend",
+      templateFramework,
+      port,
+      externalPort,
+      1000 * 60 * 5,
+    );
+  } else {
+    // wait create-llama to exit
+    // we don't test install dependencies for now, so just set timeout for 10 seconds
+    await new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        reject(new Error("create-llama timeout error"));
+      }, 1000 * 10);
+      appProcess.on("exit", (code) => {
+        if (code !== 0 && code !== null) {
+          clearTimeout(timeout);
+          reject(new Error("create-llama command was failed!"));
+        } else {
+          clearTimeout(timeout);
+          resolve(undefined);
+        }
+      });
+    });
+  }
+
+  return {
+    projectName: name,
+    appProcess,
+  };
+}
+
+export async function createTestDir() {
+  const cwd = path.join(__dirname, ".cache", crypto.randomUUID());
+  await mkdir(cwd, { recursive: true });
+  return cwd;
+}

packages/create-llama/helpers/constant.ts

@@ -0,0 +1,6 @@
+export const COMMUNITY_OWNER = "run-llama";
+export const COMMUNITY_REPO = "create_llama_projects";
+export const LLAMA_PACK_OWNER = "run-llama";
+export const LLAMA_PACK_REPO = "llama_index";
+export const LLAMA_PACK_FOLDER = "llama-index-packs";
+export const LLAMA_PACK_FOLDER_PATH = `${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/main/${LLAMA_PACK_FOLDER}`;

packages/create-llama/helpers/copy.ts

@@ -0,0 +1,50 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { async as glob } from "fast-glob";
+import fs from "fs";
+import path from "path";
+
+interface CopyOption {
+  cwd?: string;
+  rename?: (basename: string) => string;
+  parents?: boolean;
+}
+
+const identity = (x: string) => x;
+
+export const copy = async (
+  src: string | string[],
+  dest: string,
+  { cwd, rename = identity, parents = true }: CopyOption = {},
+) => {
+  const source = typeof src === "string" ? [src] : src;
+
+  if (source.length === 0 || !dest) {
+    throw new TypeError("`src` and `dest` are required");
+  }
+
+  const sourceFiles = await glob(source, {
+    cwd,
+    dot: true,
+    absolute: false,
+    stats: false,
+  });
+
+  const destRelativeToCwd = cwd ? path.resolve(cwd, dest) : dest;
+
+  return Promise.all(
+    sourceFiles.map(async (p) => {
+      const dirname = path.dirname(p);
+      const basename = rename(path.basename(p));
+
+      const from = cwd ? path.resolve(cwd, p) : p;
+      const to = parents
+        ? path.join(destRelativeToCwd, dirname, basename)
+        : path.join(destRelativeToCwd, basename);
+
+      // Ensure the destination directory exists
+      await fs.promises.mkdir(path.dirname(to), { recursive: true });
+
+      return fs.promises.copyFile(from, to);
+    }),
+  );
+};

packages/create-llama/helpers/dir.ts

@@ -0,0 +1,3 @@
+import path from "path";
+
+export const templatesDir = path.join(__dirname, "..", "templates");

packages/create-llama/helpers/get-pkg-manager.ts

@@ -0,0 +1,15 @@
+export type PackageManager = "npm" | "pnpm" | "yarn";
+
+export function getPkgManager(): PackageManager {
+  const userAgent = process.env.npm_config_user_agent || "";
+
+  if (userAgent.startsWith("yarn")) {
+    return "yarn";
+  }
+
+  if (userAgent.startsWith("pnpm")) {
+    return "pnpm";
+  }
+
+  return "npm";
+}

packages/create-llama/helpers/git.ts

@@ -0,0 +1,58 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { execSync } from "child_process";
+import fs from "fs";
+import path from "path";
+
+function isInGitRepository(): boolean {
+  try {
+    execSync("git rev-parse --is-inside-work-tree", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+function isInMercurialRepository(): boolean {
+  try {
+    execSync("hg --cwd . root", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+function isDefaultBranchSet(): boolean {
+  try {
+    execSync("git config init.defaultBranch", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function tryGitInit(root: string): boolean {
+  let didInit = false;
+  try {
+    execSync("git --version", { stdio: "ignore" });
+    if (isInGitRepository() || isInMercurialRepository()) {
+      return false;
+    }
+
+    execSync("git init", { stdio: "ignore" });
+    didInit = true;
+
+    if (!isDefaultBranchSet()) {
+      execSync("git checkout -b main", { stdio: "ignore" });
+    }
+
+    execSync("git add -A", { stdio: "ignore" });
+    execSync('git commit -m "Initial commit from Create Llama"', {
+      stdio: "ignore",
+    });
+    return true;
+  } catch (e) {
+    if (didInit) {
+      try {
+        fs.rmSync(path.join(root, ".git"), { recursive: true, force: true });
+      } catch (_) {}
+    }
+    return false;
+  }
+}

packages/create-llama/helpers/index.ts

@@ -0,0 +1,247 @@
+import { copy } from "./copy";
+import { callPackageManager } from "./install";
+
+import fs from "fs/promises";
+import path from "path";
+import { cyan } from "picocolors";
+
+import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./constant";
+import { templatesDir } from "./dir";
+import { PackageManager } from "./get-pkg-manager";
+import { installLlamapackProject } from "./llama-pack";
+import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry";
+import { installPythonTemplate } from "./python";
+import { downloadAndExtractRepo } from "./repo";
+import {
+  FileSourceConfig,
+  InstallTemplateArgs,
+  TemplateDataSource,
+  TemplateFramework,
+  TemplateVectorDB,
+  WebSourceConfig,
+} from "./types";
+import { installTSTemplate } from "./typescript";
+
+const createEnvLocalFile = async (
+  root: string,
+  opts?: {
+    openAiKey?: string;
+    llamaCloudKey?: string;
+    vectorDb?: TemplateVectorDB;
+    model?: string;
+    embeddingModel?: string;
+    framework?: TemplateFramework;
+    dataSource?: TemplateDataSource;
+  },
+) => {
+  const envFileName = ".env";
+  let content = "";
+
+  const model = opts?.model || "gpt-3.5-turbo";
+  content += `MODEL=${model}\n`;
+  if (opts?.framework === "nextjs") {
+    content += `NEXT_PUBLIC_MODEL=${model}\n`;
+  }
+  console.log("\nUsing OpenAI model: ", model, "\n");
+
+  if (opts?.openAiKey) {
+    content += `OPENAI_API_KEY=${opts?.openAiKey}\n`;
+  }
+
+  if (opts?.embeddingModel) {
+    content += `EMBEDDING_MODEL=${opts?.embeddingModel}\n`;
+  }
+
+  if (opts?.llamaCloudKey) {
+    content += `LLAMA_CLOUD_API_KEY=${opts?.llamaCloudKey}\n`;
+  }
+
+  switch (opts?.vectorDb) {
+    case "mongo": {
+      content += `# For generating a connection URI, see https://www.mongodb.com/docs/guides/atlas/connection-string\n`;
+      content += `MONGO_URI=\n`;
+      content += `MONGODB_DATABASE=\n`;
+      content += `MONGODB_VECTORS=\n`;
+      content += `MONGODB_VECTOR_INDEX=\n`;
+      break;
+    }
+    case "pg": {
+      content += `# For generating a connection URI, see https://docs.timescale.com/use-timescale/latest/services/create-a-service\n`;
+      content += `PG_CONNECTION_STRING=\n`;
+      break;
+    }
+    case "pinecone": {
+      content += `PINECONE_API_KEY=\n`;
+      content += `PINECONE_ENVIRONMENT=\n`;
+      content += `PINECONE_INDEX_NAME=\n`;
+      break;
+    }
+  }
+
+  switch (opts?.dataSource?.type) {
+    case "web": {
+      const webConfig = opts?.dataSource.config as WebSourceConfig;
+      content += `# web loader config\n`;
+      content += `BASE_URL=${webConfig.baseUrl}\n`;
+      content += `URL_PREFIX=${webConfig.baseUrl}\n`;
+      content += `MAX_DEPTH=${webConfig.depth}\n`;
+      break;
+    }
+  }
+
+  if (content) {
+    await fs.writeFile(path.join(root, envFileName), content);
+    console.log(`Created '${envFileName}' file. Please check the settings.`);
+  }
+};
+
+const generateContextData = async (
+  framework: TemplateFramework,
+  packageManager?: PackageManager,
+  openAiKey?: string,
+  vectorDb?: TemplateVectorDB,
+) => {
+  if (packageManager) {
+    const runGenerate = `${cyan(
+      framework === "fastapi"
+        ? "poetry run python app/engine/generate.py"
+        : `${packageManager} run generate`,
+    )}`;
+    const hasOpenAiKey = openAiKey || process.env["OPENAI_API_KEY"];
+    const hasVectorDb = vectorDb && vectorDb !== "none";
+    if (framework === "fastapi") {
+      if (hasOpenAiKey && !hasVectorDb && isHavingPoetryLockFile()) {
+        console.log(`Running ${runGenerate} to generate the context data.`);
+        const result = tryPoetryRun("python app/engine/generate.py");
+        if (!result) {
+          console.log(`Failed to run ${runGenerate}.`);
+          process.exit(1);
+        }
+        console.log(`Generated context data`);
+        return;
+      }
+    } else {
+      if (hasOpenAiKey && vectorDb === "none") {
+        console.log(`Running ${runGenerate} to generate the context data.`);
+        await callPackageManager(packageManager, true, ["run", "generate"]);
+        return;
+      }
+    }
+
+    const settings = [];
+    if (!hasOpenAiKey) settings.push("your OpenAI key");
+    if (hasVectorDb) settings.push("your Vector DB environment variables");
+    const settingsMessage =
+      settings.length > 0 ? `After setting ${settings.join(" and ")}, ` : "";
+    const generateMessage = `run ${runGenerate} to generate the context data.`;
+    console.log(`\n${settingsMessage}${generateMessage}\n\n`);
+  }
+};
+
+const copyContextData = async (
+  root: string,
+  dataSource?: TemplateDataSource,
+) => {
+  const destPath = path.join(root, "data");
+
+  const dataSourceConfig = dataSource?.config as FileSourceConfig;
+
+  // Copy file
+  if (dataSource?.type === "file") {
+    if (dataSourceConfig.path) {
+      console.log(`\nCopying file to ${cyan(destPath)}\n`);
+      await fs.mkdir(destPath, { recursive: true });
+      await fs.copyFile(
+        dataSourceConfig.path,
+        path.join(destPath, path.basename(dataSourceConfig.path)),
+      );
+    } else {
+      console.log("Missing file path in config");
+      process.exit(1);
+    }
+    return;
+  }
+
+  // Copy folder
+  if (dataSource?.type === "folder") {
+    const srcPath =
+      dataSourceConfig.path ?? path.join(templatesDir, "components", "data");
+    console.log(`\nCopying data to ${cyan(destPath)}\n`);
+    await copy("**", destPath, {
+      parents: true,
+      cwd: srcPath,
+    });
+    return;
+  }
+};
+
+const installCommunityProject = async ({
+  root,
+  communityProjectPath,
+}: Pick<InstallTemplateArgs, "root" | "communityProjectPath">) => {
+  console.log("\nInstalling community project:", communityProjectPath!);
+  await downloadAndExtractRepo(root, {
+    username: COMMUNITY_OWNER,
+    name: COMMUNITY_REPO,
+    branch: "main",
+    filePath: communityProjectPath!,
+  });
+};
+
+export const installTemplate = async (
+  props: InstallTemplateArgs & { backend: boolean },
+) => {
+  process.chdir(props.root);
+
+  if (props.template === "community" && props.communityProjectPath) {
+    await installCommunityProject(props);
+    return;
+  }
+
+  if (props.template === "llamapack" && props.llamapack) {
+    await installLlamapackProject(props);
+    return;
+  }
+
+  if (props.framework === "fastapi") {
+    await installPythonTemplate(props);
+  } else {
+    await installTSTemplate(props);
+  }
+
+  if (props.backend) {
+    // This is a backend, so we need to copy the test data and create the env file.
+
+    // Copy the environment file to the target directory.
+    await createEnvLocalFile(props.root, {
+      openAiKey: props.openAiKey,
+      llamaCloudKey: props.llamaCloudKey,
+      vectorDb: props.vectorDb,
+      model: props.model,
+      embeddingModel: props.embeddingModel,
+      framework: props.framework,
+      dataSource: props.dataSource,
+    });
+
+    if (props.engine === "context") {
+      await copyContextData(props.root, props.dataSource);
+      if (
+        props.postInstallAction === "runApp" ||
+        props.postInstallAction === "dependencies"
+      ) {
+        await generateContextData(
+          props.framework,
+          props.packageManager,
+          props.openAiKey,
+          props.vectorDb,
+        );
+      }
+    }
+  } else {
+    // this is a frontend for a full-stack app, create .env file with model information
+    const content = `MODEL=${props.model}\nNEXT_PUBLIC_MODEL=${props.model}\n`;
+    await fs.writeFile(path.join(props.root, ".env"), content);
+  }
+};
+
+export * from "./types";

packages/create-llama/helpers/install.ts

@@ -0,0 +1,50 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import spawn from "cross-spawn";
+import { yellow } from "picocolors";
+import type { PackageManager } from "./get-pkg-manager";
+
+/**
+ * Spawn a package manager installation based on user preference.
+ *
+ * @returns A Promise that resolves once the installation is finished.
+ */
+export async function callPackageManager(
+  /** Indicate which package manager to use. */
+  packageManager: PackageManager,
+  /** Indicate whether there is an active Internet connection.*/
+  isOnline: boolean,
+  args: string[] = ["install"],
+): Promise<void> {
+  if (!isOnline) {
+    console.log(
+      yellow("You appear to be offline.\nFalling back to the local cache."),
+    );
+    args.push("--offline");
+  }
+  /**
+   * Return a Promise that resolves once the installation is finished.
+   */
+  return new Promise((resolve, reject) => {
+    /**
+     * Spawn the installation process.
+     */
+    const child = spawn(packageManager, args, {
+      stdio: "inherit",
+      env: {
+        ...process.env,
+        ADBLOCK: "1",
+        // we set NODE_ENV to development as pnpm skips dev
+        // dependencies when production
+        NODE_ENV: "development",
+        DISABLE_OPENCOLLECTIVE: "1",
+      },
+    });
+    child.on("close", (code) => {
+      if (code !== 0) {
+        reject({ command: `${packageManager} ${args.join(" ")}` });
+        return;
+      }
+      resolve();
+    });
+  });
+}

packages/create-llama/helpers/is-folder-empty.ts

@@ -0,0 +1,62 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import fs from "fs";
+import path from "path";
+import { blue, green } from "picocolors";
+
+export function isFolderEmpty(root: string, name: string): boolean {
+  const validFiles = [
+    ".DS_Store",
+    ".git",
+    ".gitattributes",
+    ".gitignore",
+    ".gitlab-ci.yml",
+    ".hg",
+    ".hgcheck",
+    ".hgignore",
+    ".idea",
+    ".npmignore",
+    ".travis.yml",
+    "LICENSE",
+    "Thumbs.db",
+    "docs",
+    "mkdocs.yml",
+    "npm-debug.log",
+    "yarn-debug.log",
+    "yarn-error.log",
+    "yarnrc.yml",
+    ".yarn",
+  ];
+
+  const conflicts = fs
+    .readdirSync(root)
+    .filter((file) => !validFiles.includes(file))
+    // Support IntelliJ IDEA-based editors
+    .filter((file) => !/\.iml$/.test(file));
+
+  if (conflicts.length > 0) {
+    console.log(
+      `The directory ${green(name)} contains files that could conflict:`,
+    );
+    console.log();
+    for (const file of conflicts) {
+      try {
+        const stats = fs.lstatSync(path.join(root, file));
+        if (stats.isDirectory()) {
+          console.log(`  ${blue(file)}/`);
+        } else {
+          console.log(`  ${file}`);
+        }
+      } catch {
+        console.log(`  ${file}`);
+      }
+    }
+    console.log();
+    console.log(
+      "Either try using a new directory name, or remove the files listed above.",
+    );
+    console.log();
+    return false;
+  }
+
+  return true;
+}

packages/create-llama/helpers/is-online.ts

@@ -0,0 +1,40 @@
+import { execSync } from "child_process";
+import dns from "dns";
+import url from "url";
+
+function getProxy(): string | undefined {
+  if (process.env.https_proxy) {
+    return process.env.https_proxy;
+  }
+
+  try {
+    const httpsProxy = execSync("npm config get https-proxy").toString().trim();
+    return httpsProxy !== "null" ? httpsProxy : undefined;
+  } catch (e) {
+    return;
+  }
+}
+
+export function getOnline(): Promise<boolean> {
+  return new Promise((resolve) => {
+    dns.lookup("registry.yarnpkg.com", (registryErr) => {
+      if (!registryErr) {
+        return resolve(true);
+      }
+
+      const proxy = getProxy();
+      if (!proxy) {
+        return resolve(false);
+      }
+
+      const { hostname } = url.parse(proxy);
+      if (!hostname) {
+        return resolve(false);
+      }
+
+      dns.lookup(hostname, (proxyErr) => {
+        resolve(proxyErr == null);
+      });
+    });
+  });
+}

packages/create-llama/helpers/is-url.ts

@@ -0,0 +1,8 @@
+export function isUrl(url: string): boolean {
+  try {
+    new URL(url);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}