fix: fix syncing edge with core version

Co-authored-by: Michael Schramm <michael@tucan.ai>

.changeset/clever-doors-invite.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-feat: experimental package + json query engine

.changeset/eight-dogs-burn.md

@@ -1,12 +0,0 @@
----
-"llamaindex": patch
-"@llamaindex/core-test": patch
----
-
-- Add missing exports:
-  - `IndexStructType`,
-  - `IndexDict`,
-  - `jsonToIndexStruct`,
-  - `IndexList`,
-  - `IndexStruct`
-- Fix `IndexDict.toJson()` method

.changeset/eighty-waves-march.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add auto create milvus collection and add milvus node metadata

.changeset/few-chicken-exercise.md

@@ -1,5 +0,0 @@
----
-"create-llama": patch
----
-
-Add "Start in VSCode" option to postInstallAction

.changeset/fluffy-squids-play.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Use Pinecone namespaces for all operations

.changeset/khaki-rivers-unite.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Add streaming to agents

.changeset/plenty-onions-relate.md

@@ -1,5 +0,0 @@
----
-"create-llama": patch
----
-
-Add devcontainers to generated code

.changeset/strange-waves-sit.md

@@ -1,5 +0,0 @@
----
-"llamaindex": minor
----
-
-Use parameter object for retrieve function of Retriever (to align usage with query function of QueryEngine)

.changeset/yellow-chicken-study.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add support for edge runtime by using @llamaindex/edge

.github/workflows/e2e.yml

@@ -1,68 +0,0 @@
-name: E2E Tests
-on:
-  push:
-    branches: [main]
-  pull_request:
-    paths:
-      - "packages/create-llama/**"
-      - ".github/workflows/e2e.yml"
-    branches: [main]
-
-env:
-  POETRY_VERSION: "1.6.1"
-
-jobs:
-  e2e:
-    name: create-llama
-    timeout-minutes: 60
-    strategy:
-      fail-fast: true
-      matrix:
-        node-version: [18, 20]
-        python-version: ["3.11"]
-        os: [macos-latest, windows-latest]
-    defaults:
-      run:
-        shell: bash
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-        with:
-          version: ${{ env.POETRY_VERSION }}
-      - uses: pnpm/action-setup@v2
-      - name: Setup Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: "pnpm"
-      - name: Install dependencies
-        run: pnpm install
-      - name: Install Playwright Browsers
-        run: pnpm exec playwright install --with-deps
-        working-directory: ./packages/create-llama
-      - name: Build create-llama
-        run: pnpm run build
-        working-directory: ./packages/create-llama
-      - name: Pack
-        run: pnpm pack --pack-destination ./output
-        working-directory: ./packages/create-llama
-      - name: Extract Pack
-        run: tar -xvzf ./output/*.tgz -C ./output
-        working-directory: ./packages/create-llama
-      - name: Run Playwright tests
-        run: pnpm exec playwright test
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        working-directory: ./packages/create-llama
-      - uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: ./packages/create-llama/playwright-report/
-          retention-days: 30

.github/workflows/publish.yml

@@ -14,6 +14,14 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+      - name: Install dependencies
+        run: pnpm install
 
       - name: Publish @llamaindex/env
         run: npx jsr publish

.github/workflows/test.yml

@@ -44,6 +44,24 @@ jobs:
           name: typecheck-build-dist
           path: ./packages/core/dist
           if-no-files-found: error
+  core-edge-runtime:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+      - name: Install dependencies
+        run: pnpm install
+      - name: Build
+        run: pnpm run build --filter @llamaindex/edge
+      - name: Build Edge Runtime
+        run: pnpm run build
+        working-directory: ./packages/edge/e2e/test-edge-runtime
   typecheck-examples:
     runs-on: ubuntu-latest
 

CONTRIBUTING.md

@@ -84,8 +84,7 @@ Any changes you make should be reflected in the browser. If you need to regenera
 To publish a new version of the library, run
 
 ```shell
-pnpm new-llamaindex
-pnpm new-create-llama
+pnpm new-version
 pnpm release
 git push # push to the main branch
 git push --tags

README.md

@@ -121,6 +121,42 @@ const nextConfig = {
 module.exports = nextConfig;
 ```
 
+### NextJS with Milvus:
+
+As proto files are not loaded per default in NextJS, you'll need to add the following to your next.config.js to have it load the proto files.
+
+```js
+const path = require("path");
+const CopyWebpackPlugin = require("copy-webpack-plugin");
+
+// next.config.js
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  webpack: (config, { isServer }) => {
+    if (isServer) {
+      // Copy the proto files to the server build directory
+      config.plugins.push(
+        new CopyWebpackPlugin({
+          patterns: [
+            {
+              from: path.join(
+                __dirname,
+                "node_modules/@zilliz/milvus2-sdk-node/dist",
+              ),
+              to: path.join(__dirname, ".next"),
+            },
+          ],
+        }),
+      );
+    }
+    // Important: return the modified config
+    return config;
+  },
+};
+
+module.exports = nextConfig;
+```
+
 ## Supported LLMs:
 
 - OpenAI GPT-3.5-turbo and GPT-4

apps/docs/docs/modules/ingestion_pipeline/transformations.md

@@ -1,6 +1,6 @@
 # Transformations
 
-A transformation is something that takes a list of nodes as an input, and returns a list of nodes. Each component that implements the Transformatio class has both a `transform` definition responsible for transforming the nodes
+A transformation is something that takes a list of nodes as an input, and returns a list of nodes. Each component that implements the Transformation class has both a `transform` definition responsible for transforming the nodes.
 
 Currently, the following components are Transformation objects:
 

apps/docs/docs/modules/node_postprocessors/index.md

@@ -36,7 +36,7 @@ const processor = new SimilarityPostprocessor({
   similarityCutoff: 0.7,
 });
 
-const filteredNodes = processor.postprocessNodes(nodes);
+const filteredNodes = await processor.postprocessNodes(nodes);
 
 // cohere rerank: rerank nodes given query using trained model
 const reranker = new CohereRerank({

examples/CHANGELOG.md

@@ -0,0 +1,14 @@
+# examples
+
+## 0.0.4
+
+### Patch Changes
+
+- d2e8d0c: add support for Milvus vector store
+- Updated dependencies [d2e8d0c]
+- Updated dependencies [aefc326]
+- Updated dependencies [484a710]
+- Updated dependencies [d766bd0]
+- Updated dependencies [dd95927]
+- Updated dependencies [bf583a7]
+  - llamaindex@0.2.0

examples/anthropic/haiku.ts

@@ -0,0 +1,19 @@
+import { Anthropic } from "llamaindex";
+
+(async () => {
+  const anthropic = new Anthropic({
+    apiKey: process.env.ANTHROPIC_API_KEY,
+    model: "claude-3-haiku",
+  });
+  const result = await anthropic.chat({
+    messages: [
+      { content: "You want to talk in rhymes.", role: "system" },
+      {
+        content:
+          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
+        role: "user",
+      },
+    ],
+  });
+  console.log(result);
+})();

examples/astradb/README.md

@@ -32,10 +32,10 @@ run `ts-node astradb/example`
 
 This sample loads the same dataset of movie reviews as the Astra Portal sample dataset. (Feel free to load the data in your the Astra Data Explorer to compare)
 
-run `ts-node astradb/load`
+run `npx ts-node astradb/load`
 
 ### Use RAG to Query the data
 
 Check out your data in the Astra Data Explorer and change the sample query as you see fit.
 
-run `ts-node astradb/query`
+run `npx ts-node astradb/query`

examples/milvus/README.md

@@ -0,0 +1,34 @@
+# Milvus Vector Store
+
+Here are two sample scripts which work with loading and querying data from a Milvus Vector Store.
+
+## Prerequisites
+
+- An Milvus Vector Database
+  - Hosted https://milvus.io/
+  - Self Hosted https://milvus.io/docs/install_standalone-docker.md
+- An OpenAI API Key
+
+## Setup
+
+1. Set your env variables:
+
+- `MILVUS_ADDRESS`: Address of your Milvus Vector Store (like localhost:19530)
+- `MILVUS_USERNAME`: empty or username for your Milvus Vector Store
+- `MILVUS_PASSWORD`: empty or password for your Milvus Vector Store
+- `OPENAI_API_KEY`: Your OpenAI key
+
+2. `cd` Into the `examples` directory
+3. run `npm i`
+
+## Load the data
+
+This sample loads the same dataset of movie reviews as sample dataset. You can install https://github.com/zilliztech/attu to inspect the loaded data.
+
+run `npx ts-node milvus/load`
+
+## Use RAG to Query the data
+
+Check out your data in Attu and change the sample query as you see fit.
+
+run `npx ts-node milvus/query`

examples/milvus/load.ts

@@ -0,0 +1,26 @@
+import {
+  MilvusVectorStore,
+  PapaCSVReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+
+const collectionName = "movie_reviews";
+
+async function main() {
+  try {
+    const reader = new PapaCSVReader(false);
+    const docs = await reader.loadData("./data/movie_reviews.csv");
+
+    const vectorStore = new MilvusVectorStore({ collection: collectionName });
+
+    const ctx = await storageContextFromDefaults({ vectorStore });
+    const index = await VectorStoreIndex.fromDocuments(docs, {
+      storageContext: ctx,
+    });
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+main();

examples/milvus/query.ts

@@ -0,0 +1,30 @@
+import {
+  MilvusVectorStore,
+  serviceContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+
+const collectionName = "movie_reviews";
+
+async function main() {
+  try {
+    const milvus = new MilvusVectorStore({ collection: collectionName });
+
+    const ctx = serviceContextFromDefaults();
+    const index = await VectorStoreIndex.fromVectorStore(milvus, ctx);
+
+    const retriever = await index.asRetriever({ similarityTopK: 20 });
+
+    const queryEngine = await index.asQueryEngine({ retriever });
+
+    const results = await queryEngine.query({
+      query: "What is the best reviewed movie?",
+    });
+
+    console.log(results.response);
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+main();

examples/package.json

@@ -1,16 +1,19 @@
 {
   "name": "examples",
   "private": true,
-  "version": "0.0.3",
+  "version": "0.0.4",
   "dependencies": {
+    "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.4",
     "@notionhq/client": "^2.2.14",
     "@pinecone-database/pinecone": "^1.1.3",
+    "@zilliz/milvus2-sdk-node": "^2.3.5",
     "chromadb": "^1.8.1",
     "commander": "^11.1.0",
     "dotenv": "^16.4.1",
     "llamaindex": "latest",
-    "mongodb": "^6.2.0"
+    "mongodb": "^6.2.0",
+    "pathe": "^1.1.2"
   },
   "devDependencies": {
     "@types/node": "^18.19.10",

examples/qdrantdb/README.md

@@ -0,0 +1,11 @@
+# Qdrant Vector Store Example
+
+How to run `examples/qdrantdb/preFilters.ts`:
+
+Add your OpenAI API Key into a file called `.env` in the parent folder of this directory. It should look like this:
+
+```
+OPEN_API_KEY=sk-you-key
+```
+
+Now, open a new terminal window and inside `examples`, run `npx ts-node qdrantdb/preFilters.ts`.

examples/qdrantdb/preFilters.ts

@@ -0,0 +1,82 @@
+import * as dotenv from "dotenv";
+import {
+  CallbackManager,
+  Document,
+  MetadataMode,
+  QdrantVectorStore,
+  VectorStoreIndex,
+  serviceContextFromDefaults,
+  storageContextFromDefaults,
+} from "llamaindex";
+
+// Load environment variables from local .env file
+dotenv.config();
+
+const collectionName = "dog_colors";
+const qdrantUrl = "http://127.0.0.1:6333";
+
+async function main() {
+  try {
+    const docs = [
+      new Document({
+        text: "The dog is brown",
+        metadata: {
+          dogId: "1",
+        },
+      }),
+      new Document({
+        text: "The dog is red",
+        metadata: {
+          dogId: "2",
+        },
+      }),
+    ];
+    console.log("Creating QdrantDB vector store");
+    const qdrantVs = new QdrantVectorStore({ url: qdrantUrl, collectionName });
+    const ctx = await storageContextFromDefaults({ vectorStore: qdrantVs });
+
+    console.log("Embedding documents and adding to index");
+    const index = await VectorStoreIndex.fromDocuments(docs, {
+      storageContext: ctx,
+      serviceContext: serviceContextFromDefaults({
+        callbackManager: new CallbackManager({
+          onRetrieve: (data) => {
+            console.log(
+              "The retrieved nodes are:",
+              data.nodes.map((node) => node.node.getContent(MetadataMode.NONE)),
+            );
+          },
+        }),
+      }),
+    });
+
+    console.log(
+      "Querying index with no filters: Expected output: Brown probably",
+    );
+    const queryEngineNoFilters = index.asQueryEngine();
+    const noFilterResponse = await queryEngineNoFilters.query({
+      query: "What is the color of the dog?",
+    });
+    console.log("No filter response:", noFilterResponse.toString());
+    console.log("Querying index with dogId 2: Expected output: Red");
+    const queryEngineDogId2 = index.asQueryEngine({
+      preFilters: {
+        filters: [
+          {
+            key: "dogId",
+            value: "2",
+            filterType: "ExactMatch",
+          },
+        ],
+      },
+    });
+    const response = await queryEngineDogId2.query({
+      query: "What is the color of the dog?",
+    });
+    console.log("Filter with dogId 2 response:", response.toString());
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+main();

examples/readers/CHANGELOG.md

@@ -0,0 +1,11 @@
+# llamaindex-loader-example
+
+### Patch Changes
+
+- Updated dependencies [d2e8d0c]
+- Updated dependencies [aefc326]
+- Updated dependencies [484a710]
+- Updated dependencies [d766bd0]
+- Updated dependencies [dd95927]
+- Updated dependencies [bf583a7]
+  - llamaindex@0.2.0

examples/readers/src/custom-simple-directory-reader.ts

@@ -2,8 +2,8 @@ import type { BaseReader, Document, Metadata } from "llamaindex";
 import {
   FILE_EXT_TO_READER,
   SimpleDirectoryReader,
-  TextFileReader,
 } from "llamaindex/readers/SimpleDirectoryReader";
+import { TextFileReader } from "llamaindex/readers/TextFileReader";
 
 class ZipReader implements BaseReader {
   loadData(...args: any[]): Promise<Document<Metadata>[]> {

package.json

@@ -11,10 +11,12 @@
     "prepare": "husky",
     "test": "turbo run test",
     "type-check": "tsc -b --diagnostics",
-    "release": "pnpm run build:release && changeset publish",
-    "new-llamaindex": "pnpm run build:release && changeset version --ignore create-llama",
-    "new-create-llama": "pnpm run build:release && changeset version --ignore llamaindex --ignore @llamaindex/core-test",
-    "new-experimental": "pnpm run build:release && changeset version --ignore create-llama"
+    "release": "pnpm run check-minor-version && pnpm run build:release && changeset publish",
+    "release-snapshot": "pnpm run check-minor-version && pnpm run build:release && changeset publish --tag snapshot",
+    "check-minor-version": "node ./scripts/check-minor-version",
+    "update-version": "node ./scripts/update-version",
+    "new-version": "pnpm run build:release && changeset version && pnpm run check-minor-version && pnpm run update-version",
+    "new-snapshot": "pnpm run build:release && changeset version --snapshot && pnpm run update-version"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",

packages/core/CHANGELOG.md

@@ -1,5 +1,25 @@
 # llamaindex
 
+## 0.2.0
+
+### Minor Changes
+
+- bf583a7: Use parameter object for retrieve function of Retriever (to align usage with query function of QueryEngine)
+
+### Patch Changes
+
+- d2e8d0c: add support for Milvus vector store
+- aefc326: feat: experimental package + json query engine
+- 484a710: - Add missing exports:
+  - `IndexStructType`,
+  - `IndexDict`,
+  - `jsonToIndexStruct`,
+  - `IndexList`,
+  - `IndexStruct`
+  - Fix `IndexDict.toJson()` method
+- d766bd0: Add streaming to agents
+- dd95927: add Claude Haiku support and update anthropic SDK
+
 ## 0.1.21
 
 ### Patch Changes

packages/core/package.json

@@ -1,12 +1,14 @@
 {
   "name": "llamaindex",
-  "version": "0.1.21",
+  "version": "0.2.0",
+  "expectedMinorVersion": "2",
   "license": "MIT",
   "type": "module",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.15.0",
+    "@anthropic-ai/sdk": "^0.18.0",
     "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.4",
+    "@grpc/grpc-js": "^1.10.2",
     "@llamaindex/cloud": "0.0.4",
     "@llamaindex/env": "workspace:*",
     "@mistralai/mistralai": "^0.0.10",
@@ -18,12 +20,13 @@
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.0",
     "@xenova/transformers": "^2.15.0",
+    "@zilliz/milvus2-sdk-node": "^2.3.5",
     "assemblyai": "^4.2.2",
     "chromadb": "~1.7.3",
     "cohere-ai": "^7.7.5",
-    "file-type": "^18.7.0",
     "js-tiktoken": "^1.0.10",
     "lodash": "^4.17.21",
+    "magic-bytes.js": "^1.10.0",
     "mammoth": "^1.6.0",
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.3.0",
@@ -59,10 +62,6 @@
         "types": "./dist/type/index.d.ts",
         "default": "./dist/index.js"
       },
-      "edge-light": {
-        "types": "./dist/type/index.d.ts",
-        "default": "./dist/index.edge-light.js"
-      },
       "require": {
         "types": "./dist/type/index.d.ts",
         "default": "./dist/cjs/index.js"

packages/core/src/agent/react/worker.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from "crypto";
+import { randomUUID } from "@llamaindex/env";
 import { CallbackManager } from "../../callbacks/CallbackManager.js";
 import { AgentChatResponse } from "../../engines/chat/index.js";
 import type { ChatResponse, LLM } from "../../llm/index.js";
@@ -17,7 +17,6 @@ import {
   ObservationReasoningStep,
   ResponseReasoningStep,
 } from "./types.js";
-
 type ReActAgentWorkerParams = {
   tools: BaseTool[];
   llm?: LLM;

packages/core/src/agent/runner/base.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from "crypto";
+import { randomUUID } from "@llamaindex/env";
 import { CallbackManager } from "../../callbacks/CallbackManager.js";
 import type { ChatEngineAgentParams } from "../../engines/chat/index.js";
 import {
@@ -12,7 +12,6 @@ import type { BaseMemory } from "../../memory/types.js";
 import type { AgentWorker, TaskStepOutput } from "../types.js";
 import { Task, TaskStep } from "../types.js";
 import { AgentState, BaseAgentRunner, TaskState } from "./types.js";
-
 const validateStepFromArgs = (
   taskId: string,
   input?: string | null,

packages/core/src/embeddings/utils.ts

@@ -1,5 +1,6 @@
 import { defaultFS } from "@llamaindex/env";
 import _ from "lodash";
+import { filetypemime } from "magic-bytes.js";
 import type { ImageType } from "../Node.js";
 import { DEFAULT_SIMILARITY_TOP_K } from "../constants.js";
 import { VectorStoreQueryMode } from "../storage/vectorStore/types.js";
@@ -199,13 +200,12 @@ export function getTopKMMREmbeddings(
 }
 
 async function blobToDataUrl(input: Blob) {
-  const { fileTypeFromBuffer } = await import("file-type");
   const buffer = Buffer.from(await input.arrayBuffer());
-  const type = await fileTypeFromBuffer(buffer);
-  if (!type) {
+  const mimes = filetypemime(buffer);
+  if (mimes.length < 1) {
     throw new Error("Unsupported image type");
   }
-  return "data:" + type.mime + ";base64," + buffer.toString("base64");
+  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
 }
 
 export async function readImage(input: ImageType) {

packages/core/src/index.edge.ts

@@ -0,0 +1,31 @@
+export * from "./ChatHistory.js";
+export * from "./GlobalsHelper.js";
+export * from "./Node.js";
+export * from "./OutputParser.js";
+export * from "./Prompt.js";
+export * from "./PromptHelper.js";
+export * from "./QuestionGenerator.js";
+export * from "./Response.js";
+export * from "./Retriever.js";
+export * from "./ServiceContext.js";
+export * from "./TextSplitter.js";
+export * from "./agent/index.js";
+export * from "./callbacks/CallbackManager.js";
+export * from "./cloud/index.js";
+export * from "./constants.js";
+export * from "./embeddings/index.js";
+export * from "./engines/chat/index.js";
+export * from "./engines/query/index.js";
+export * from "./evaluation/index.js";
+export * from "./extractors/index.js";
+export * from "./indices/index.js";
+export * from "./ingestion/index.js";
+export * from "./llm/index.js";
+export * from "./nodeParsers/index.js";
+export * from "./objects/index.js";
+export * from "./postprocessors/index.js";
+export * from "./prompts/index.js";
+export * from "./selectors/index.js";
+export * from "./synthesizers/index.js";
+export * from "./tools/index.js";
+export * from "./types.js";

packages/core/src/index.ts

@@ -1,33 +1,3 @@
-export * from "./ChatHistory.js";
-export * from "./GlobalsHelper.js";
-export * from "./Node.js";
-export * from "./OutputParser.js";
-export * from "./Prompt.js";
-export * from "./PromptHelper.js";
-export * from "./QuestionGenerator.js";
-export * from "./Response.js";
-export * from "./Retriever.js";
-export * from "./ServiceContext.js";
-export * from "./TextSplitter.js";
-export * from "./agent/index.js";
-export * from "./callbacks/CallbackManager.js";
-export * from "./cloud/index.js";
-export * from "./constants.js";
-export * from "./embeddings/index.js";
-export * from "./engines/chat/index.js";
-export * from "./engines/query/index.js";
-export * from "./evaluation/index.js";
-export * from "./extractors/index.js";
-export * from "./indices/index.js";
-export * from "./ingestion/index.js";
-export * from "./llm/index.js";
-export * from "./nodeParsers/index.js";
-export * from "./objects/index.js";
-export * from "./postprocessors/index.js";
-export * from "./prompts/index.js";
+export * from "./index.edge.js";
 export * from "./readers/index.js";
-export * from "./selectors/index.js";
 export * from "./storage/index.js";
-export * from "./synthesizers/index.js";
-export * from "./tools/index.js";
-export * from "./types.js";

packages/core/src/indices/keyword/index.ts

@@ -13,8 +13,9 @@ import type { ServiceContext } from "../../ServiceContext.js";
 import { serviceContextFromDefaults } from "../../ServiceContext.js";
 import { RetrieverQueryEngine } from "../../engines/query/index.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
-import type { BaseDocumentStore, StorageContext } from "../../storage/index.js";
-import { storageContextFromDefaults } from "../../storage/index.js";
+import type { StorageContext } from "../../storage/StorageContext.js";
+import { storageContextFromDefaults } from "../../storage/StorageContext.js";
+import type { BaseDocumentStore } from "../../storage/docStore/types.js";
 import type { BaseSynthesizer } from "../../synthesizers/index.js";
 import type { BaseQueryEngine } from "../../types.js";
 import type { BaseIndexInit } from "../BaseIndex.js";

packages/core/src/indices/summary/index.ts

@@ -8,12 +8,12 @@ import type { ServiceContext } from "../../ServiceContext.js";
 import { serviceContextFromDefaults } from "../../ServiceContext.js";
 import { RetrieverQueryEngine } from "../../engines/query/index.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
+import type { StorageContext } from "../../storage/StorageContext.js";
+import { storageContextFromDefaults } from "../../storage/StorageContext.js";
 import type {
   BaseDocumentStore,
   RefDocInfo,
-  StorageContext,
-} from "../../storage/index.js";
-import { storageContextFromDefaults } from "../../storage/index.js";
+} from "../../storage/docStore/types.js";
 import type { BaseSynthesizer } from "../../synthesizers/index.js";
 import {
   CompactAndRefine,

packages/core/src/indices/vectorStore/index.ts

@@ -24,15 +24,15 @@ import { ClipEmbedding } from "../../embeddings/index.js";
 import { RetrieverQueryEngine } from "../../engines/query/RetrieverQueryEngine.js";
 import { runTransformations } from "../../ingestion/index.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/types.js";
+import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";
+import type { BaseIndexStore } from "../../storage/indexStore/types.js";
 import type {
-  BaseIndexStore,
   MetadataFilters,
-  StorageContext,
   VectorStore,
   VectorStoreQuery,
   VectorStoreQueryResult,
-} from "../../storage/index.js";
+} from "../../storage/vectorStore/types.js";
 import { VectorStoreQueryMode } from "../../storage/vectorStore/types.js";
 import type { BaseSynthesizer } from "../../synthesizers/types.js";
 import type { BaseQueryEngine } from "../../types.js";

packages/core/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts

@@ -1,5 +1,6 @@
 import type { BaseNode } from "../../Node.js";
-import type { BaseDocumentStore, VectorStore } from "../../storage/index.js";
+import type { BaseDocumentStore } from "../../storage/docStore/types.js";
+import type { VectorStore } from "../../storage/vectorStore/types.js";
 import { classify } from "./classify.js";
 
 /**

packages/core/src/llm/LLM.ts

@@ -620,6 +620,7 @@ export const ALL_AVAILABLE_ANTHROPIC_LEGACY_MODELS = {
 export const ALL_AVAILABLE_V3_MODELS = {
   "claude-3-opus": { contextWindow: 200000 },
   "claude-3-sonnet": { contextWindow: 200000 },
+  "claude-3-haiku": { contextWindow: 200000 },
 };
 
 export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
@@ -630,6 +631,7 @@ export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
 const AVAILABLE_ANTHROPIC_MODELS_WITHOUT_DATE: { [key: string]: string } = {
   "claude-3-opus": "claude-3-opus-20240229",
   "claude-3-sonnet": "claude-3-sonnet-20240229",
+  "claude-3-haiku": "claude-3-haiku-20240307",
 } as { [key in keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS]: string };
 
 /**

packages/core/src/readers/LlamaParseReader.ts

@@ -1,4 +1,5 @@
 import { defaultFS, getEnv, type GenericFileSystem } from "@llamaindex/env";
+import { filetypemime } from "magic-bytes.js";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";
 
@@ -109,11 +110,10 @@ export class LlamaParseReader implements FileReader {
   }
 
   private async getMimeType(data: Buffer): Promise<string> {
-    const { fileTypeFromBuffer } = await import("file-type");
-    const type = await fileTypeFromBuffer(data);
-    if (type?.mime !== "application/pdf") {
+    const mimes = filetypemime(data);
+    if (!mimes.includes("application/pdf")) {
       throw new Error("Currently, only PDF files are supported.");
     }
-    return type.mime;
+    return "application/pdf";
   }
 }

packages/core/src/readers/SimpleDirectoryReader.edge.ts

@@ -0,0 +1,126 @@
+import type { CompleteFileSystem } from "@llamaindex/env";
+import { defaultFS, path } from "@llamaindex/env";
+import { Document } from "../Node.js";
+import { walk } from "../storage/FileSystem.js";
+import { TextFileReader } from "./TextFileReader.js";
+import type { BaseReader } from "./type.js";
+
+type ReaderCallback = (
+  category: "file" | "directory",
+  name: string,
+  status: ReaderStatus,
+  message?: string,
+) => boolean;
+enum ReaderStatus {
+  STARTED = 0,
+  COMPLETE,
+  ERROR,
+}
+
+export type SimpleDirectoryReaderLoadDataParams = {
+  directoryPath: string;
+  fs?: CompleteFileSystem;
+  defaultReader?: BaseReader | null;
+  fileExtToReader?: Record<string, BaseReader>;
+};
+
+/**
+ * Read all the documents in a directory.
+ * By default, supports the list of file types
+ * in the FILE_EXT_TO_READER map.
+ */
+export class SimpleDirectoryReader implements BaseReader {
+  constructor(private observer?: ReaderCallback) {}
+
+  async loadData(
+    params: SimpleDirectoryReaderLoadDataParams,
+  ): Promise<Document[]>;
+  async loadData(directoryPath: string): Promise<Document[]>;
+  async loadData(
+    params: SimpleDirectoryReaderLoadDataParams | string,
+  ): Promise<Document[]> {
+    if (typeof params === "string") {
+      params = { directoryPath: params };
+    }
+
+    const {
+      directoryPath,
+      fs = defaultFS,
+      defaultReader = new TextFileReader(),
+      fileExtToReader,
+    } = params;
+
+    // Observer can decide to skip the directory
+    if (
+      !this.doObserverCheck("directory", directoryPath, ReaderStatus.STARTED)
+    ) {
+      return [];
+    }
+
+    const docs: Document[] = [];
+    for await (const filePath of walk(fs, directoryPath)) {
+      try {
+        const fileExt = path.extname(filePath).slice(1).toLowerCase();
+
+        // Observer can decide to skip each file
+        if (!this.doObserverCheck("file", filePath, ReaderStatus.STARTED)) {
+          // Skip this file
+          continue;
+        }
+
+        let reader: BaseReader;
+
+        if (fileExtToReader && fileExt in fileExtToReader) {
+          reader = fileExtToReader[fileExt];
+        } else if (defaultReader != null) {
+          reader = defaultReader;
+        } else {
+          const msg = `No reader for file extension of ${filePath}`;
+          console.warn(msg);
+
+          // In an error condition, observer's false cancels the whole process.
+          if (
+            !this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)
+          ) {
+            return [];
+          }
+
+          continue;
+        }
+
+        const fileDocs = await reader.loadData(filePath, fs);
+
+        // Observer can still cancel addition of the resulting docs from this file
+        if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) {
+          docs.push(...fileDocs);
+        }
+      } catch (e) {
+        const msg = `Error reading file ${filePath}: ${e}`;
+        console.error(msg);
+
+        // In an error condition, observer's false cancels the whole process.
+        if (!this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)) {
+          return [];
+        }
+      }
+    }
+
+    // After successful import of all files, directory completion
+    // is only a notification for observer, cannot be cancelled.
+    this.doObserverCheck("directory", directoryPath, ReaderStatus.COMPLETE);
+
+    return docs;
+  }
+
+  private doObserverCheck(
+    category: "file" | "directory",
+    name: string,
+    status: ReaderStatus,
+    message?: string,
+  ): boolean {
+    if (this.observer) {
+      return this.observer(category, name, status, message);
+    }
+    return true;
+  }
+}

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -1,40 +1,17 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
 import { Document } from "../Node.js";
-import { walk } from "../storage/FileSystem.js";
 import { PapaCSVReader } from "./CSVReader.js";
 import { DocxReader } from "./DocxReader.js";
 import { HTMLReader } from "./HTMLReader.js";
 import { ImageReader } from "./ImageReader.js";
 import { MarkdownReader } from "./MarkdownReader.js";
 import { PDFReader } from "./PDFReader.js";
+import {
+  SimpleDirectoryReader as EdgeSimpleDirectoryReader,
+  type SimpleDirectoryReaderLoadDataParams,
+} from "./SimpleDirectoryReader.edge.js";
+import { TextFileReader } from "./TextFileReader.js";
 import type { BaseReader } from "./type.js";
 
-type ReaderCallback = (
-  category: "file" | "directory",
-  name: string,
-  status: ReaderStatus,
-  message?: string,
-) => boolean;
-enum ReaderStatus {
-  STARTED = 0,
-  COMPLETE,
-  ERROR,
-}
-
-/**
- * Read a .txt file
- */
-export class TextFileReader implements BaseReader {
-  async loadData(
-    file: string,
-    fs: CompleteFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
-    return [new Document({ text: dataBuffer, id_: file })];
-  }
-}
-
 export const FILE_EXT_TO_READER: Record<string, BaseReader> = {
   txt: new TextFileReader(),
   pdf: new PDFReader(),
@@ -49,21 +26,12 @@ export const FILE_EXT_TO_READER: Record<string, BaseReader> = {
   gif: new ImageReader(),
 };
 
-export type SimpleDirectoryReaderLoadDataParams = {
-  directoryPath: string;
-  fs?: CompleteFileSystem;
-  defaultReader?: BaseReader | null;
-  fileExtToReader?: Record<string, BaseReader>;
-};
-
 /**
  * Read all the documents in a directory.
  * By default, supports the list of file types
  * in the FILE_EXT_TO_READER map.
  */
-export class SimpleDirectoryReader implements BaseReader {
-  constructor(private observer?: ReaderCallback) {}
-
+export class SimpleDirectoryReader extends EdgeSimpleDirectoryReader {
   async loadData(
     params: SimpleDirectoryReaderLoadDataParams,
   ): Promise<Document[]>;
@@ -74,85 +42,7 @@ export class SimpleDirectoryReader implements BaseReader {
     if (typeof params === "string") {
       params = { directoryPath: params };
     }
-
-    const {
-      directoryPath,
-      fs = defaultFS,
-      defaultReader = new TextFileReader(),
-      fileExtToReader = FILE_EXT_TO_READER,
-    } = params;
-
-    // Observer can decide to skip the directory
-    if (
-      !this.doObserverCheck("directory", directoryPath, ReaderStatus.STARTED)
-    ) {
-      return [];
-    }
-
-    const docs: Document[] = [];
-    for await (const filePath of walk(fs, directoryPath)) {
-      try {
-        const fileExt = path.extname(filePath).slice(1).toLowerCase();
-
-        // Observer can decide to skip each file
-        if (!this.doObserverCheck("file", filePath, ReaderStatus.STARTED)) {
-          // Skip this file
-          continue;
-        }
-
-        let reader: BaseReader;
-
-        if (fileExt in fileExtToReader) {
-          reader = fileExtToReader[fileExt];
-        } else if (defaultReader != null) {
-          reader = defaultReader;
-        } else {
-          const msg = `No reader for file extension of ${filePath}`;
-          console.warn(msg);
-
-          // In an error condition, observer's false cancels the whole process.
-          if (
-            !this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)
-          ) {
-            return [];
-          }
-
-          continue;
-        }
-
-        const fileDocs = await reader.loadData(filePath, fs);
-
-        // Observer can still cancel addition of the resulting docs from this file
-        if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) {
-          docs.push(...fileDocs);
-        }
-      } catch (e) {
-        const msg = `Error reading file ${filePath}: ${e}`;
-        console.error(msg);
-
-        // In an error condition, observer's false cancels the whole process.
-        if (!this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)) {
-          return [];
-        }
-      }
-    }
-
-    // After successful import of all files, directory completion
-    // is only a notification for observer, cannot be cancelled.
-    this.doObserverCheck("directory", directoryPath, ReaderStatus.COMPLETE);
-
-    return docs;
-  }
-
-  private doObserverCheck(
-    category: "file" | "directory",
-    name: string,
-    status: ReaderStatus,
-    message?: string,
-  ): boolean {
-    if (this.observer) {
-      return this.observer(category, name, status, message);
-    }
-    return true;
+    params.fileExtToReader = params.fileExtToReader ?? FILE_EXT_TO_READER;
+    return super.loadData(params);
   }
 }

packages/core/src/readers/TextFileReader.ts

@@ -0,0 +1,18 @@
+import type { CompleteFileSystem } from "@llamaindex/env";
+import { defaultFS } from "@llamaindex/env";
+import { Document } from "../Node.js";
+import type { BaseReader } from "./type.js";
+
+/**
+ * Read a .txt file
+ */
+
+export class TextFileReader implements BaseReader {
+  async loadData(
+    file: string,
+    fs: CompleteFileSystem = defaultFS,
+  ): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file);
+    return [new Document({ text: dataBuffer, id_: file })];
+  }
+}

packages/core/src/readers/index.ts

@@ -9,4 +9,5 @@ export * from "./NotionReader.js";
 export * from "./PDFReader.js";
 export * from "./SimpleDirectoryReader.js";
 export * from "./SimpleMongoReader.js";
+export * from "./TextFileReader.js";
 export * from "./type.js";

packages/core/src/storage/index.ts

@@ -11,6 +11,7 @@ export { SimpleKVStore } from "./kvStore/SimpleKVStore.js";
 export * from "./kvStore/types.js";
 export { AstraDBVectorStore } from "./vectorStore/AstraDBVectorStore.js";
 export { ChromaVectorStore } from "./vectorStore/ChromaVectorStore.js";
+export { MilvusVectorStore } from "./vectorStore/MilvusVectorStore.js";
 export { MongoDBAtlasVectorSearch } from "./vectorStore/MongoDBAtlasVectorStore.js";
 export { PGVectorStore } from "./vectorStore/PGVectorStore.js";
 export { PineconeVectorStore } from "./vectorStore/PineconeVectorStore.js";

packages/core/src/storage/vectorStore/MilvusVectorStore.ts

@@ -0,0 +1,214 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import type { ChannelOptions } from "@grpc/grpc-js";
+import {
+  DataType,
+  MilvusClient,
+  type ClientConfig,
+  type DeleteReq,
+  type RowData,
+} from "@zilliz/milvus2-sdk-node";
+import { BaseNode, MetadataMode, type Metadata } from "../../Node.js";
+import type {
+  VectorStore,
+  VectorStoreQuery,
+  VectorStoreQueryResult,
+} from "./types.js";
+import { metadataDictToNode, nodeToMetadata } from "./utils.js";
+
+export class MilvusVectorStore implements VectorStore {
+  public storesText: boolean = true;
+  public isEmbeddingQuery?: boolean;
+  private flatMetadata: boolean = true;
+
+  private milvusClient: MilvusClient;
+  private collectionInitialized = false;
+  private collectionName: string;
+
+  private idKey: string;
+  private contentKey: string;
+  private metadataKey: string;
+  private embeddingKey: string;
+
+  constructor(
+    init?: Partial<{ milvusClient: MilvusClient }> & {
+      params?: {
+        configOrAddress: ClientConfig | string;
+        ssl?: boolean;
+        username?: string;
+        password?: string;
+        channelOptions?: ChannelOptions;
+      };
+      collection?: string;
+      idKey?: string;
+      contentKey?: string;
+      metadataKey?: string;
+      embeddingKey?: string;
+    },
+  ) {
+    if (init?.milvusClient) {
+      this.milvusClient = init.milvusClient;
+    } else {
+      const configOrAddress =
+        init?.params?.configOrAddress ?? process.env.MILVUS_ADDRESS;
+      const ssl = init?.params?.ssl ?? process.env.MILVUS_SSL === "true";
+      const username = init?.params?.username ?? process.env.MILVUS_USERNAME;
+      const password = init?.params?.password ?? process.env.MILVUS_PASSWORD;
+
+      if (!configOrAddress) {
+        throw new Error("Must specify MILVUS_ADDRESS via env variable.");
+      }
+      this.milvusClient = new MilvusClient(
+        configOrAddress,
+        ssl,
+        username,
+        password,
+        init?.params?.channelOptions,
+      );
+    }
+
+    this.collectionName = init?.collection ?? "llamacollection";
+    this.idKey = init?.idKey ?? "id";
+    this.contentKey = init?.contentKey ?? "content";
+    this.metadataKey = init?.metadataKey ?? "metadata";
+    this.embeddingKey = init?.embeddingKey ?? "embedding";
+  }
+
+  public client(): MilvusClient {
+    return this.milvusClient;
+  }
+
+  private async createCollection() {
+    await this.milvusClient.createCollection({
+      collection_name: this.collectionName,
+      fields: [
+        {
+          name: this.idKey,
+          data_type: DataType.VarChar,
+          is_primary_key: true,
+          max_length: 200,
+        },
+        {
+          name: this.embeddingKey,
+          data_type: DataType.FloatVector,
+          dim: 1536,
+        },
+        {
+          name: this.contentKey,
+          data_type: DataType.VarChar,
+          max_length: 9000,
+        },
+        {
+          name: this.metadataKey,
+          data_type: DataType.JSON,
+        },
+      ],
+    });
+    await this.milvusClient.createIndex({
+      collection_name: this.collectionName,
+      field_name: this.embeddingKey,
+    });
+  }
+
+  private async ensureCollection(): Promise<void> {
+    if (!this.collectionInitialized) {
+      await this.milvusClient.connectPromise;
+
+      // Check collection exists
+      const isCollectionExist = await this.milvusClient.hasCollection({
+        collection_name: this.collectionName,
+      });
+      if (!isCollectionExist.value) {
+        await this.createCollection();
+      }
+
+      await this.milvusClient.loadCollectionSync({
+        collection_name: this.collectionName,
+      });
+      this.collectionInitialized = true;
+    }
+  }
+
+  public async add(nodes: BaseNode<Metadata>[]): Promise<string[]> {
+    await this.ensureCollection();
+
+    const result = await this.milvusClient.insert({
+      collection_name: this.collectionName,
+      data: nodes.map((node) => {
+        const metadata = nodeToMetadata(
+          node,
+          true,
+          this.contentKey,
+          this.flatMetadata,
+        );
+
+        const entry: RowData = {
+          [this.idKey]: node.id_,
+          [this.embeddingKey]: node.getEmbedding(),
+          [this.contentKey]: node.getContent(MetadataMode.NONE),
+          [this.metadataKey]: metadata,
+        };
+
+        return entry;
+      }),
+    });
+
+    if (!result.IDs) {
+      return [];
+    }
+
+    if ("int_id" in result.IDs) {
+      return result.IDs.int_id.data.map((i) => String(i));
+    }
+
+    return result.IDs.str_id.data.map((s) => String(s));
+  }
+
+  public async delete(
+    refDocId: string,
+    deleteOptions?: Omit<DeleteReq, "ids">,
+  ): Promise<void> {
+    this.ensureCollection();
+
+    await this.milvusClient.delete({
+      ids: [refDocId],
+      collection_name: this.collectionName,
+      ...deleteOptions,
+    });
+  }
+
+  public async query(
+    query: VectorStoreQuery,
+    _options?: any,
+  ): Promise<VectorStoreQueryResult> {
+    await this.ensureCollection();
+
+    const found = await this.milvusClient.search({
+      collection_name: this.collectionName,
+      limit: query.similarityTopK,
+      vector: query.queryEmbedding,
+    });
+
+    const nodes: BaseNode<Metadata>[] = [];
+    const similarities: number[] = [];
+    const ids: string[] = [];
+
+    found.results.forEach((result) => {
+      const node = metadataDictToNode(result.metadata);
+      node.setContent(result.content);
+      nodes.push(node);
+
+      similarities.push(result.score);
+      ids.push(String(result.id));
+    });
+
+    return {
+      nodes,
+      similarities,
+      ids,
+    };
+  }
+
+  public async persist() {
+    // no need to do anything
+  }
+}

packages/core/src/storage/vectorStore/PineconeVectorStore.ts

@@ -73,7 +73,7 @@ export class PineconeVectorStore implements VectorStore {
 
   async index() {
     const db: Pinecone = await this.getDb();
-    return await db.index(this.indexName);
+    return db.index(this.indexName).namespace(this.namespace);
   }
 
   /**
@@ -82,8 +82,8 @@ export class PineconeVectorStore implements VectorStore {
    * @returns The result of the delete query.
    */
   async clearIndex() {
-    const db: Pinecone = await this.getDb();
-    return await db.index(this.indexName).deleteAll();
+    const idx = await this.index();
+    return await idx.deleteAll();
   }
 
   /**
@@ -155,12 +155,10 @@ export class PineconeVectorStore implements VectorStore {
     };
 
     const idx = await this.index();
-    const results = await idx.namespace(this.namespace).query(options);
+    const results = await idx.query(options);
 
     const idList = results.matches.map((row) => row.id);
-    const records: FetchResponse<any> = await idx
-      .namespace(this.namespace)
-      .fetch(idList);
+    const records: FetchResponse<any> = await idx.fetch(idList);
     const rows = Object.values(records.records);
 
     const nodes = rows.map((row) => {

packages/core/src/storage/vectorStore/QdrantVectorStore.ts

@@ -289,7 +289,7 @@ export class QdrantVectorStore implements VectorStore {
    * @param query The VectorStoreQuery to be used
    */
   private async buildQueryFilter(query: VectorStoreQuery) {
-    if (!query.docIds && !query.queryStr) {
+    if (!query.docIds && !query.queryStr && !query.filters) {
       return null;
     }
 

packages/core/tests/CHANGELOG.md

@@ -0,0 +1,13 @@
+# @llamaindex/core-test
+
+## 0.0.2
+
+### Patch Changes
+
+- 484a710: - Add missing exports:
+  - `IndexStructType`,
+  - `IndexDict`,
+  - `jsonToIndexStruct`,
+  - `IndexList`,
+  - `IndexStruct`
+  - Fix `IndexDict.toJson()` method

packages/core/tests/StorageContext.test.ts

@@ -1,21 +1,37 @@
-import { storageContextFromDefaults } from "llamaindex/storage/StorageContext";
+import {
+  storageContextFromDefaults,
+  type StorageContext,
+} from "llamaindex/storage/StorageContext";
 import { existsSync, rmSync } from "node:fs";
-import { describe, expect, test, vi, vitest } from "vitest";
+import {
+  afterAll,
+  beforeAll,
+  describe,
+  expect,
+  test,
+  vi,
+  vitest,
+} from "vitest";
 
 vitest.spyOn(console, "error");
 
 describe("StorageContext", () => {
+  let storageContext: StorageContext;
+
+  beforeAll(async () => {
+    storageContext = await storageContextFromDefaults({
+      persistDir: "/tmp/test_dir",
+    });
+  });
+
   test("initializes", async () => {
     vi.mocked(console.error).mockImplementation(() => {}); // silence console.error
 
-    const storageContext = await storageContextFromDefaults({
-      persistDir: "/tmp/test_dir",
-    });
-
     expect(existsSync("/tmp/test_dir")).toBe(true);
     expect(storageContext).toBeDefined();
+  });
 
-    // cleanup
+  afterAll(() => {
     rmSync("/tmp/test_dir", { recursive: true });
   });
 });

packages/core/tests/indices/SummaryIndex.test.ts

@@ -1,48 +1,34 @@
-import type { ServiceContext } from "llamaindex";
 import {
   Document,
-  OpenAI,
-  OpenAIEmbedding,
   SummaryIndex,
   VectorStoreIndex,
-  serviceContextFromDefaults,
   storageContextFromDefaults,
+  type ServiceContext,
+  type StorageContext,
 } from "llamaindex";
-import { beforeAll, describe, expect, it, vi } from "vitest";
-import {
-  mockEmbeddingModel,
-  mockLlmGeneration,
-} from "../utility/mockOpenAI.js";
+import { rmSync } from "node:fs";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 
-// Mock the OpenAI getOpenAISession function during testing
 vi.mock("llamaindex/llm/open_ai", () => {
   return {
     getOpenAISession: vi.fn().mockImplementation(() => null),
   };
 });
 
+import { mockServiceContext } from "../utility/mockServiceContext.js";
+
 describe("SummaryIndex", () => {
   let serviceContext: ServiceContext;
+  let storageContext: StorageContext;
 
-  beforeAll(() => {
-    const embeddingModel = new OpenAIEmbedding();
-    const llm = new OpenAI();
-
-    mockEmbeddingModel(embeddingModel);
-    mockLlmGeneration({ languageModel: llm });
-
-    const ctx = serviceContextFromDefaults({
-      embedModel: embeddingModel,
-      llm,
+  beforeAll(async () => {
+    serviceContext = mockServiceContext();
+    storageContext = await storageContextFromDefaults({
+      persistDir: "/tmp/test_dir",
     });
-
-    serviceContext = ctx;
   });
 
   it("SummaryIndex and VectorStoreIndex must be able to share the same storage context", async () => {
-    const storageContext = await storageContextFromDefaults({
-      persistDir: "/tmp/test_dir",
-    });
     const documents = [new Document({ text: "lorem ipsem", id_: "1" })];
     const vectorIndex = await VectorStoreIndex.fromDocuments(documents, {
       serviceContext,
@@ -55,4 +41,8 @@ describe("SummaryIndex", () => {
     });
     expect(summaryIndex).toBeDefined();
   });
+
+  afterAll(() => {
+    rmSync("/tmp/test_dir", { recursive: true });
+  });
 });

packages/core/tests/indices/VectorStoreIndex.test.ts

@@ -0,0 +1,61 @@
+import type { ServiceContext, StorageContext } from "llamaindex";
+import {
+  Document,
+  VectorStoreIndex,
+  storageContextFromDefaults,
+} from "llamaindex";
+import { beforeAll, describe, expect, test, vi } from "vitest";
+import { mockServiceContext } from "../utility/mockServiceContext.js";
+
+vi.mock("llamaindex/llm/open_ai", () => {
+  return {
+    getOpenAISession: vi.fn().mockImplementation(() => null),
+  };
+});
+
+describe.sequential("VectorStoreIndex", () => {
+  let serviceContext: ServiceContext;
+  let storageContext: StorageContext;
+  let testStrategy: (
+    // strategy?: DocStoreStrategy,
+    runs?: number,
+  ) => Promise<Array<number>>;
+
+  beforeAll(async () => {
+    serviceContext = mockServiceContext();
+    storageContext = await storageContextFromDefaults({
+      persistDir: "/tmp/test_dir",
+    });
+    testStrategy = async (
+      // strategy?: DocStoreStrategy,
+      runs: number = 2,
+    ): Promise<Array<number>> => {
+      const documents = [new Document({ text: "lorem ipsem", id_: "1" })];
+      const entries = [];
+      for (let i = 0; i < runs; i++) {
+        await VectorStoreIndex.fromDocuments(documents, {
+          serviceContext,
+          storageContext,
+          // docStoreStrategy: strategy,
+        });
+        const docs = await storageContext.docStore.docs();
+        entries.push(Object.keys(docs).length);
+      }
+      return entries;
+    };
+  });
+
+  test("fromDocuments does not stores duplicates per default", async () => {
+    const entries = await testStrategy();
+    expect(entries[0]).toBe(entries[1]);
+  });
+
+  // test("fromDocuments ignores duplicates in upserts", async () => {
+  //   const entries = await testStrategy(DocStoreStrategy.DUPLICATES_ONLY);
+  //   expect(entries[0]).toBe(entries[1]);
+  // });
+
+  // afterAll(() => {
+  //   rmSync("/tmp/test_dir", { recursive: true });
+  // });
+});

packages/core/tests/objects/ObjectIndex.test.ts

@@ -2,17 +2,10 @@ import type { ServiceContext } from "llamaindex";
 import {
   FunctionTool,
   ObjectIndex,
-  OpenAI,
-  OpenAIEmbedding,
   SimpleToolNodeMapping,
   VectorStoreIndex,
-  serviceContextFromDefaults,
 } from "llamaindex";
 import { beforeAll, describe, expect, test, vi } from "vitest";
-import {
-  mockEmbeddingModel,
-  mockLlmGeneration,
-} from "../utility/mockOpenAI.js";
 
 vi.mock("llamaindex/llm/open_ai", () => {
   return {
@@ -20,22 +13,13 @@ vi.mock("llamaindex/llm/open_ai", () => {
   };
 });
 
+import { mockServiceContext } from "../utility/mockServiceContext.js";
+
 describe("ObjectIndex", () => {
   let serviceContext: ServiceContext;
 
   beforeAll(() => {
-    const embeddingModel = new OpenAIEmbedding();
-    const llm = new OpenAI();
-
-    mockEmbeddingModel(embeddingModel);
-    mockLlmGeneration({ languageModel: llm });
-
-    const ctx = serviceContextFromDefaults({
-      embedModel: embeddingModel,
-      llm,
-    });
-
-    serviceContext = ctx;
+    serviceContext = mockServiceContext();
   });
 
   test("test_object_with_tools", async () => {

packages/core/tests/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core-test",
   "private": true,
-  "version": "0.0.1",
+  "version": "0.0.2",
   "type": "module",
   "scripts": {
     "test": "vitest run"

packages/core/tests/utility/mockServiceContext.ts

@@ -0,0 +1,23 @@
+import {
+  OpenAI,
+  OpenAIEmbedding,
+  serviceContextFromDefaults,
+} from "llamaindex";
+
+import {
+  mockEmbeddingModel,
+  mockLlmGeneration,
+} from "../utility/mockOpenAI.js";
+
+export function mockServiceContext() {
+  const embeddingModel = new OpenAIEmbedding();
+  const llm = new OpenAI();
+
+  mockEmbeddingModel(embeddingModel);
+  mockLlmGeneration({ languageModel: llm });
+
+  return serviceContextFromDefaults({
+    embedModel: embeddingModel,
+    llm,
+  });
+}

packages/create-llama/.eslintrc

@@ -1,20 +0,0 @@
-{
-  "root": false,
-  "rules": {
-    "turbo/no-undeclared-env-vars": [
-      "error",
-      {
-        "allowList": [
-          "OPENAI_API_KEY",
-          "LLAMA_CLOUD_API_KEY",
-          "npm_config_user_agent",
-          "http_proxy",
-          "https_proxy",
-          "MODEL",
-          "NEXT_PUBLIC_CHAT_API",
-          "NEXT_PUBLIC_MODEL"
-        ]
-      }
-    ]
-  }
-}

packages/create-llama/CHANGELOG.md

@@ -1,167 +0,0 @@
-# create-llama
-
-## 0.0.27
-
-### Patch Changes
-
-- 2d29350: Add LlamaParse option when selecting a pdf file or a folder (FastAPI only)
-- b354f23: Add embedding model option to create-llama (FastAPI only)
-
-## 0.0.26
-
-### Patch Changes
-
-- 09d532e: feat: generate llama pack project from llama index
-- cfdd6db: feat: add pinecone support to create llama
-- ef25d69: upgrade llama-index package to version v0.10.7 for create-llama app
-- 50dfd7b: update fastapi for CVE-2024-24762
-
-## 0.0.25
-
-### Patch Changes
-
-- d06a85b: Add option to create an agent by selecting tools (Google, Wikipedia)
-- 7b7329b: Added latest turbo models for GPT-3.5 and GPT 4
-
-## 0.0.24
-
-### Patch Changes
-
-- ba95ca3: Use condense plus context chat engine for FastAPI as default
-
-## 0.0.23
-
-### Patch Changes
-
-- c680af6: Fixed issues with locating templates path
-
-## 0.0.22
-
-### Patch Changes
-
-- 6dd401e: Add an option to provide an URL and chat with the website data (FastAPI only)
-- e9b87ef: Select a folder as data source and support more file types (.pdf, .doc, .docx, .xls, .xlsx, .csv)
-
-## 0.0.20
-
-### Patch Changes
-
-- 27d55fd: Add an option to provide an URL and chat with the website data
-
-## 0.0.19
-
-### Patch Changes
-
-- 3a29a80: Add node_modules to gitignore in Express backends
-- fe03aaa: feat: generate llama pack example
-
-## 0.0.18
-
-### Patch Changes
-
-- 88d3b41: fix packaging
-
-## 0.0.17
-
-### Patch Changes
-
-- fa17f7e: Add an option that allows the user to run the generated app
-- 9e5d8e1: Add an option to select a local PDF file as data source
-
-## 0.0.16
-
-### Patch Changes
-
-- a73942d: Fix: Bundle mongo dependency with NextJS
-- 9492cc6: Feat: Added option to automatically install dependencies (for Python and TS)
-- f74dea5: Feat: Show images in chat messages using GPT4 Vision (Express and NextJS only)
-
-## 0.0.15
-
-### Patch Changes
-
-- 8e124e5: feat: support showing image on chat message
-
-## 0.0.14
-
-### Patch Changes
-
-- 2e6b36e: fix: re-organize file structure
-- 2b356c8: fix: relative path incorrect
-
-## 0.0.13
-
-### Patch Changes
-
-- Added PostgreSQL vector store (for Typescript and Python)
-- Improved async handling in FastAPI
-
-## 0.0.12
-
-### Patch Changes
-
-- 9c5e22a: Added cross-env so frontends with Express/FastAPI backends are working under Windows
-- 5ab65eb: Bring Python templates with TS templates to feature parity
-- 9c5e22a: Added vector DB selector to create-llama (starting with MongoDB support)
-
-## 0.0.11
-
-### Patch Changes
-
-- 2aeb341: - Added option to create a new project based on community templates
-  - Added OpenAI model selector for NextJS projects
-  - Added GPT4 Vision support (and file upload)
-
-## 0.0.10
-
-### Patch Changes
-
-- Bugfixes (thanks @marcusschiesser)
-
-## 0.0.9
-
-### Patch Changes
-
-- acfe232: Deployment fixes (thanks @seldo)
-
-## 0.0.8
-
-### Patch Changes
-
-- 8cdb07f: Fix Next deployment (thanks @seldo and @marcusschiesser)
-
-## 0.0.7
-
-### Patch Changes
-
-- 9f9f293: Added more to README and made it easier to switch models (thanks @seldo)
-
-## 0.0.6
-
-### Patch Changes
-
-- 4431ec7: Label bug fix (thanks @marcusschiesser)
-
-## 0.0.5
-
-### Patch Changes
-
-- 25257f4: Fix issue where it doesn't find OpenAI Key when running npm run generate (#182) (thanks @RayFernando1337)
-
-## 0.0.4
-
-### Patch Changes
-
-- 031e926: Update create-llama readme (thanks @logan-markewich)
-
-## 0.0.3
-
-### Patch Changes
-
-- 91b42a3: change version (thanks @marcusschiesser)
-
-## 0.0.2
-
-### Patch Changes
-
-- e2a6805: Hello Create Llama (thanks @marcusschiesser)

packages/create-llama/LICENSE.md

@@ -1,9 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2023 LlamaIndex, Vercel, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

packages/create-llama/README.md

@@ -1,126 +0,0 @@
-# Create LlamaIndex App
-
-The easiest way to get started with [LlamaIndex](https://www.llamaindex.ai/) is by using `create-llama`. This CLI tool enables you to quickly start building a new LlamaIndex application, with everything set up for you.
-
-Just run
-
-```bash
-npx create-llama@latest
-```
-
-to get started, or see below for more options. Once your app is generated, run
-
-```bash
-npm run dev
-```
-
-to start the development server. You can then visit [http://localhost:3000](http://localhost:3000) to see your app.
-
-## What you'll get
-
-- A Next.js-powered front-end. The app is set up as a chat interface that can answer questions about your data (see below)
-  - You can style it with HTML and CSS, or you can optionally use components from [shadcn/ui](https://ui.shadcn.com/)
-- Your choice of 3 back-ends:
-  - **Next.js**: if you select this option, you’ll have a full stack Next.js application that you can deploy to a host like [Vercel](https://vercel.com/) in just a few clicks. This uses [LlamaIndex.TS](https://www.npmjs.com/package/llamaindex), our TypeScript library.
-  - **Express**: if you want a more traditional Node.js application you can generate an Express backend. This also uses LlamaIndex.TS.
-  - **Python FastAPI**: if you select this option you’ll get a backend powered by the [llama-index python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
-- The back-end has a single endpoint that allows you to send the state of your chat and receive additional responses
-- You can choose whether you want a streaming or non-streaming back-end (if you're not sure, we recommend streaming)
-- You can choose whether you want to use `ContextChatEngine` or `SimpleChatEngine`
-  - `SimpleChatEngine` will just talk to the LLM directly without using your data
-  - `ContextChatEngine` will use your data to answer questions (see below).
-- The app uses OpenAI by default, so you'll need an OpenAI API key, or you can customize it to use any of the dozens of LLMs we support.
-
-## Using your data
-
-If you've enabled `ContextChatEngine`, you can supply your own data and the app will index it and answer questions. Your generated app will have a folder called `data`:
-
-- With the Next.js backend this is `./data`
-- With the Express or Python backend this is in `./backend/data`
-
-The app will ingest any supported files you put in this directory. Your Next.js and Express apps use LlamaIndex.TS so they will be able to ingest any PDF, text, CSV, Markdown, Word and HTML files. The Python backend can read even more types, including video and audio files.
-
-Before you can use your data, you need to index it. If you're using the Next.js or Express apps, run:
-
-```bash
-npm run generate
-```
-
-Then re-start your app. Remember you'll need to re-run `generate` if you add new files to your `data` folder. If you're using the Python backend, you can trigger indexing of your data by deleting the `./storage` folder and re-starting the app.
-
-## Don't want a front-end?
-
-It's optional! If you've selected the Python or Express back-ends, just delete the `frontend` folder and you'll get an API without any front-end code.
-
-## Customizing the LLM
-
-By default the app will use OpenAI's gpt-3.5-turbo model. If you want to use GPT-4, you can modify this by editing a file:
-
-- In the Next.js backend, edit `./app/api/chat/route.ts` and replace `gpt-3.5-turbo` with `gpt-4`
-- In the Express backend, edit `./backend/src/controllers/chat.controller.ts` and likewise replace `gpt-3.5-turbo` with `gpt-4`
-- In the Python backend, edit `./backend/app/utils/index.py` and once again replace `gpt-3.5-turbo` with `gpt-4`
-
-You can also replace OpenAI with one of our [dozens of other supported LLMs](https://docs.llamaindex.ai/en/stable/module_guides/models/llms/modules.html).
-
-## Example
-
-The simplest thing to do is run `create-llama` in interactive mode:
-
-```bash
-npx create-llama@latest
-# or
-npm create llama@latest
-# or
-yarn create llama
-# or
-pnpm create llama@latest
-```
-
-You will be asked for the name of your project, along with other configuration options, something like this:
-
-```bash
->> npm create llama@latest
-Need to install the following packages:
-  create-llama@latest
-Ok to proceed? (y) y
-✔ What is your project named? … my-app
-✔ Which template would you like to use? › Chat with streaming
-✔ Which framework would you like to use? › NextJS
-✔ Which UI would you like to use? › Just HTML
-✔ Which chat engine would you like to use? › ContextChatEngine
-✔ Please provide your OpenAI API key (leave blank to skip): …
-✔ Would you like to use ESLint? … No / Yes
-Creating a new LlamaIndex app in /home/my-app.
-```
-
-### Running non-interactively
-
-You can also pass command line arguments to set up a new project
-non-interactively. See `create-llama --help`:
-
-```bash
-create-llama <project-directory> [options]
-
-Options:
-  -V, --version                      output the version number
-
-  --use-npm
-
-    Explicitly tell the CLI to bootstrap the app using npm
-
-  --use-pnpm
-
-    Explicitly tell the CLI to bootstrap the app using pnpm
-
-  --use-yarn
-
-    Explicitly tell the CLI to bootstrap the app using Yarn
-
-```
-
-## LlamaIndex Documentation
-
-- [TS/JS docs](https://ts.llamaindex.ai/)
-- [Python docs](https://docs.llamaindex.ai/en/stable/)
-
-Inspired by and adapted from [create-next-app](https://github.com/vercel/next.js/tree/canary/packages/create-next-app)

packages/create-llama/create-app.ts

@@ -1,147 +0,0 @@
-/* eslint-disable import/no-extraneous-dependencies */
-import path from "path";
-import { green, yellow } from "picocolors";
-import { tryGitInit } from "./helpers/git";
-import { isFolderEmpty } from "./helpers/is-folder-empty";
-import { getOnline } from "./helpers/is-online";
-import { isWriteable } from "./helpers/is-writeable";
-import { makeDir } from "./helpers/make-dir";
-
-import fs from "fs";
-import terminalLink from "terminal-link";
-import type { InstallTemplateArgs } from "./helpers";
-import { installTemplate } from "./helpers";
-import { writeDevcontainer } from "./helpers/devcontainer";
-import { templatesDir } from "./helpers/dir";
-import { toolsRequireConfig } from "./helpers/tools";
-
-export type InstallAppArgs = Omit<
-  InstallTemplateArgs,
-  "appName" | "root" | "isOnline" | "customApiPath"
-> & {
-  appPath: string;
-  frontend: boolean;
-};
-
-export async function createApp({
-  template,
-  framework,
-  engine,
-  ui,
-  appPath,
-  packageManager,
-  eslint,
-  frontend,
-  openAiKey,
-  llamaCloudKey,
-  model,
-  embeddingModel,
-  communityProjectPath,
-  llamapack,
-  vectorDb,
-  externalPort,
-  postInstallAction,
-  dataSource,
-  tools,
-}: InstallAppArgs): Promise<void> {
-  const root = path.resolve(appPath);
-
-  if (!(await isWriteable(path.dirname(root)))) {
-    console.error(
-      "The application path is not writable, please check folder permissions and try again.",
-    );
-    console.error(
-      "It is likely you do not have write permissions for this folder.",
-    );
-    process.exit(1);
-  }
-
-  const appName = path.basename(root);
-
-  await makeDir(root);
-  if (!isFolderEmpty(root, appName)) {
-    process.exit(1);
-  }
-
-  const useYarn = packageManager === "yarn";
-  const isOnline = !useYarn || (await getOnline());
-
-  console.log(`Creating a new LlamaIndex app in ${green(root)}.`);
-  console.log();
-
-  const args = {
-    appName,
-    root,
-    template,
-    framework,
-    engine,
-    ui,
-    packageManager,
-    isOnline,
-    eslint,
-    openAiKey,
-    llamaCloudKey,
-    model,
-    embeddingModel,
-    communityProjectPath,
-    llamapack,
-    vectorDb,
-    externalPort,
-    postInstallAction,
-    dataSource,
-    tools,
-  };
-
-  if (frontend) {
-    // install backend
-    const backendRoot = path.join(root, "backend");
-    await makeDir(backendRoot);
-    await installTemplate({ ...args, root: backendRoot, backend: true });
-    // install frontend
-    const frontendRoot = path.join(root, "frontend");
-    await makeDir(frontendRoot);
-    await installTemplate({
-      ...args,
-      root: frontendRoot,
-      framework: "nextjs",
-      customApiPath: `http://localhost:${externalPort ?? 8000}/api/chat`,
-      backend: false,
-    });
-    // copy readme for fullstack
-    await fs.promises.copyFile(
-      path.join(templatesDir, "README-fullstack.md"),
-      path.join(root, "README.md"),
-    );
-  } else {
-    await installTemplate({ ...args, backend: true, forBackend: framework });
-  }
-
-  process.chdir(root);
-  if (tryGitInit(root)) {
-    console.log("Initialized a git repository.");
-    console.log();
-  }
-
-  await writeDevcontainer(root, templatesDir, framework, frontend);
-
-  if (toolsRequireConfig(tools)) {
-    console.log(
-      yellow(
-        `You have selected tools that require configuration. Please configure them in the ${terminalLink(
-          "tools_config.json",
-          `file://${root}/tools_config.json`,
-        )} file.`,
-      ),
-    );
-  }
-  console.log("");
-  console.log(`${green("Success!")} Created ${appName} at ${appPath}`);
-
-  console.log(
-    `Now have a look at the ${terminalLink(
-      "README.md",
-      `file://${root}/README.md`,
-    )} and learn how to get started.`,
-  );
-  console.log();
-}

packages/create-llama/e2e/basic.spec.ts

@@ -1,145 +0,0 @@
-/* eslint-disable turbo/no-undeclared-env-vars */
-import { expect, test } from "@playwright/test";
-import { ChildProcess } from "child_process";
-import fs from "fs";
-import path from "path";
-import type {
-  TemplateEngine,
-  TemplateFramework,
-  TemplatePostInstallAction,
-  TemplateType,
-  TemplateUI,
-} from "../helpers";
-import { createTestDir, runCreateLlama, type AppType } from "./utils";
-
-const templateTypes: TemplateType[] = ["streaming", "simple"];
-const templateFrameworks: TemplateFramework[] = [
-  "nextjs",
-  "express",
-  "fastapi",
-];
-const templateEngines: TemplateEngine[] = ["simple", "context"];
-const templateUIs: TemplateUI[] = ["shadcn", "html"];
-const templatePostInstallActions: TemplatePostInstallAction[] = [
-  "none",
-  "runApp",
-];
-
-for (const templateType of templateTypes) {
-  for (const templateFramework of templateFrameworks) {
-    for (const templateEngine of templateEngines) {
-      for (const templateUI of templateUIs) {
-        for (const templatePostInstallAction of templatePostInstallActions) {
-          if (templateFramework === "nextjs" && templateType === "simple") {
-            // nextjs doesn't support simple templates - skip tests
-            continue;
-          }
-          const appType: AppType =
-            templateFramework === "express" || templateFramework === "fastapi"
-              ? templateType === "simple"
-                ? "--no-frontend" // simple templates don't have frontends
-                : "--frontend"
-              : "";
-          if (appType === "--no-frontend" && templateUI !== "html") {
-            // if there's no frontend, don't iterate over UIs
-            continue;
-          }
-          test.describe(`try create-llama ${templateType} ${templateFramework} ${templateEngine} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
-            let port: number;
-            let externalPort: number;
-            let cwd: string;
-            let name: string;
-            let appProcess: ChildProcess;
-            // Only test without using vector db for now
-            const vectorDb = "none";
-
-            test.beforeAll(async () => {
-              port = Math.floor(Math.random() * 10000) + 10000;
-              externalPort = port + 1;
-              cwd = await createTestDir();
-              const result = await runCreateLlama(
-                cwd,
-                templateType,
-                templateFramework,
-                templateEngine,
-                templateUI,
-                vectorDb,
-                appType,
-                port,
-                externalPort,
-                templatePostInstallAction,
-              );
-              name = result.projectName;
-              appProcess = result.appProcess;
-            });
-
-            test("App folder should exist", async () => {
-              const dirExists = fs.existsSync(path.join(cwd, name));
-              expect(dirExists).toBeTruthy();
-            });
-            test("Frontend should have a title", async ({ page }) => {
-              test.skip(templatePostInstallAction !== "runApp");
-              test.skip(appType === "--no-frontend");
-              await page.goto(`http://localhost:${port}`);
-              await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
-            });
-
-            test("Frontend should be able to submit a message and receive a response", async ({
-              page,
-            }) => {
-              test.skip(templatePostInstallAction !== "runApp");
-              test.skip(appType === "--no-frontend");
-              await page.goto(`http://localhost:${port}`);
-              await page.fill("form input", "hello");
-              const [response] = await Promise.all([
-                page.waitForResponse(
-                  (res) => {
-                    return (
-                      res.url().includes("/api/chat") && res.status() === 200
-                    );
-                  },
-                  {
-                    timeout: 1000 * 60,
-                  },
-                ),
-                page.click("form button[type=submit]"),
-              ]);
-              const text = await response.text();
-              console.log("AI response when submitting message: ", text);
-              expect(response.ok()).toBeTruthy();
-            });
-
-            test("Backend should response when calling API", async ({
-              request,
-            }) => {
-              test.skip(templatePostInstallAction !== "runApp");
-              test.skip(appType !== "--no-frontend");
-              const backendPort = appType === "" ? port : externalPort;
-              const response = await request.post(
-                `http://localhost:${backendPort}/api/chat`,
-                {
-                  data: {
-                    messages: [
-                      {
-                        role: "user",
-                        content: "Hello",
-                      },
-                    ],
-                  },
-                },
-              );
-              const text = await response.text();
-              console.log("AI response when calling API: ", text);
-              expect(response.ok()).toBeTruthy();
-            });
-
-            // clean processes
-            test.afterAll(async () => {
-              appProcess?.kill();
-            });
-          });
-        }
-      }
-    }
-  }
-}

packages/create-llama/e2e/tsconfig.json

@@ -1,16 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "es2019",
-    "module": "esnext",
-    "moduleResolution": "node",
-    "strict": true,
-    "resolveJsonModule": true,
-    "skipLibCheck": true,
-    "declaration": false,
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "incremental": true,
-    "tsBuildInfoFile": "./lib/.tsbuildinfo"
-  },
-  "include": ["./**/*.ts"]
-}

packages/create-llama/e2e/utils.ts

@@ -1,181 +0,0 @@
-import { ChildProcess, exec } from "child_process";
-import crypto from "node:crypto";
-import { mkdir } from "node:fs/promises";
-import * as path from "path";
-import waitPort from "wait-port";
-import {
-  TemplateEngine,
-  TemplateFramework,
-  TemplatePostInstallAction,
-  TemplateType,
-  TemplateUI,
-  TemplateVectorDB,
-} from "../helpers";
-
-export type AppType = "--frontend" | "--no-frontend" | "";
-const MODEL = "gpt-3.5-turbo";
-const EMBEDDING_MODEL = "text-embedding-ada-002";
-export type CreateLlamaResult = {
-  projectName: string;
-  appProcess: ChildProcess;
-};
-
-// eslint-disable-next-line max-params
-export async function checkAppHasStarted(
-  frontend: boolean,
-  framework: TemplateFramework,
-  port: number,
-  externalPort: number,
-  timeout: number,
-) {
-  if (frontend) {
-    await Promise.all([
-      waitPort({
-        host: "localhost",
-        port: port,
-        timeout,
-      }),
-      waitPort({
-        host: "localhost",
-        port: externalPort,
-        timeout,
-      }),
-    ]).catch((err) => {
-      console.error(err);
-      throw err;
-    });
-  } else {
-    let wPort: number;
-    if (framework === "nextjs") {
-      wPort = port;
-    } else {
-      wPort = externalPort;
-    }
-    await waitPort({
-      host: "localhost",
-      port: wPort,
-      timeout,
-    }).catch((err) => {
-      console.error(err);
-      throw err;
-    });
-  }
-}
-
-// eslint-disable-next-line max-params
-export async function runCreateLlama(
-  cwd: string,
-  templateType: TemplateType,
-  templateFramework: TemplateFramework,
-  templateEngine: TemplateEngine,
-  templateUI: TemplateUI,
-  vectorDb: TemplateVectorDB,
-  appType: AppType,
-  port: number,
-  externalPort: number,
-  postInstallAction: TemplatePostInstallAction,
-): Promise<CreateLlamaResult> {
-  const createLlama = path.join(
-    __dirname,
-    "..",
-    "output",
-    "package",
-    "dist",
-    "index.js",
-  );
-
-  const name = [
-    templateType,
-    templateFramework,
-    templateEngine,
-    templateUI,
-    appType,
-  ].join("-");
-  const command = [
-    "node",
-    createLlama,
-    name,
-    "--template",
-    templateType,
-    "--framework",
-    templateFramework,
-    "--engine",
-    templateEngine,
-    "--ui",
-    templateUI,
-    "--vector-db",
-    vectorDb,
-    "--model",
-    MODEL,
-    "--embedding-model",
-    EMBEDDING_MODEL,
-    "--open-ai-key",
-    process.env.OPENAI_API_KEY || "testKey",
-    appType,
-    "--eslint",
-    "--use-npm",
-    "--port",
-    port,
-    "--external-port",
-    externalPort,
-    "--post-install-action",
-    postInstallAction,
-    "--tools",
-    "none",
-    "--no-llama-parse",
-  ].join(" ");
-  console.log(`running command '${command}' in ${cwd}`);
-  const appProcess = exec(command, {
-    cwd,
-    env: {
-      ...process.env,
-    },
-  });
-  appProcess.stderr?.on("data", (data) => {
-    console.log(data.toString());
-  });
-  appProcess.on("exit", (code) => {
-    if (code !== 0 && code !== null) {
-      throw new Error(`create-llama command was failed!`);
-    }
-  });
-
-  // Wait for app to start
-  if (postInstallAction === "runApp") {
-    await checkAppHasStarted(
-      appType === "--frontend",
-      templateFramework,
-      port,
-      externalPort,
-      1000 * 60 * 5,
-    );
-  } else {
-    // wait create-llama to exit
-    // we don't test install dependencies for now, so just set timeout for 10 seconds
-    await new Promise((resolve, reject) => {
-      const timeout = setTimeout(() => {
-        reject(new Error("create-llama timeout error"));
-      }, 1000 * 10);
-      appProcess.on("exit", (code) => {
-        if (code !== 0 && code !== null) {
-          clearTimeout(timeout);
-          reject(new Error("create-llama command was failed!"));
-        } else {
-          clearTimeout(timeout);
-          resolve(undefined);
-        }
-      });
-    });
-  }
-
-  return {
-    projectName: name,
-    appProcess,
-  };
-}
-
-export async function createTestDir() {
-  const cwd = path.join(__dirname, ".cache", crypto.randomUUID());
-  await mkdir(cwd, { recursive: true });
-  return cwd;
-}

packages/create-llama/helpers/constant.ts

@@ -1,6 +0,0 @@
-export const COMMUNITY_OWNER = "run-llama";
-export const COMMUNITY_REPO = "create_llama_projects";
-export const LLAMA_PACK_OWNER = "run-llama";
-export const LLAMA_PACK_REPO = "llama_index";
-export const LLAMA_PACK_FOLDER = "llama-index-packs";
-export const LLAMA_PACK_FOLDER_PATH = `${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/main/${LLAMA_PACK_FOLDER}`;

packages/create-llama/helpers/copy.ts

@@ -1,50 +0,0 @@
-/* eslint-disable import/no-extraneous-dependencies */
-import { async as glob } from "fast-glob";
-import fs from "fs";
-import path from "path";
-
-interface CopyOption {
-  cwd?: string;
-  rename?: (basename: string) => string;
-  parents?: boolean;
-}
-
-const identity = (x: string) => x;
-
-export const copy = async (
-  src: string | string[],
-  dest: string,
-  { cwd, rename = identity, parents = true }: CopyOption = {},
-) => {
-  const source = typeof src === "string" ? [src] : src;
-
-  if (source.length === 0 || !dest) {
-    throw new TypeError("`src` and `dest` are required");
-  }
-
-  const sourceFiles = await glob(source, {
-    cwd,
-    dot: true,
-    absolute: false,
-    stats: false,
-  });
-
-  const destRelativeToCwd = cwd ? path.resolve(cwd, dest) : dest;
-
-  return Promise.all(
-    sourceFiles.map(async (p) => {
-      const dirname = path.dirname(p);
-      const basename = rename(path.basename(p));
-
-      const from = cwd ? path.resolve(cwd, p) : p;
-      const to = parents
-        ? path.join(destRelativeToCwd, dirname, basename)
-        : path.join(destRelativeToCwd, basename);
-
-      // Ensure the destination directory exists
-      await fs.promises.mkdir(path.dirname(to), { recursive: true });
-
-      return fs.promises.copyFile(from, to);
-    }),
-  );
-};

packages/create-llama/helpers/devcontainer.ts

@@ -1,61 +0,0 @@
-import fs from "fs";
-import path from "path";
-import { TemplateFramework } from "./types";
-
-function renderDevcontainerContent(
-  templatesDir: string,
-  framework: TemplateFramework,
-  frontend: boolean,
-) {
-  const devcontainerJson: any = JSON.parse(
-    fs.readFileSync(path.join(templatesDir, "devcontainer.json"), "utf8"),
-  );
-
-  // Modify postCreateCommand
-  if (frontend) {
-    devcontainerJson.postCreateCommand =
-      framework === "fastapi"
-        ? "cd backend && poetry install && cd ../frontend && npm install"
-        : "cd backend && npm install && cd ../frontend && npm install";
-  } else {
-    devcontainerJson.postCreateCommand =
-      framework === "fastapi" ? "poetry install" : "npm install";
-  }
-
-  // Modify containerEnv
-  if (framework === "fastapi") {
-    if (frontend) {
-      devcontainerJson.containerEnv = {
-        ...devcontainerJson.containerEnv,
-        PYTHONPATH: "${PYTHONPATH}:${workspaceFolder}/backend",
-      };
-    } else {
-      devcontainerJson.containerEnv = {
-        ...devcontainerJson.containerEnv,
-        PYTHONPATH: "${PYTHONPATH}:${workspaceFolder}",
-      };
-    }
-  }
-
-  return JSON.stringify(devcontainerJson, null, 2);
-}
-
-export const writeDevcontainer = async (
-  root: string,
-  templatesDir: string,
-  framework: TemplateFramework,
-  frontend: boolean,
-) => {
-  console.log("Adding .devcontainer");
-  const devcontainerContent = renderDevcontainerContent(
-    templatesDir,
-    framework,
-    frontend,
-  );
-  const devcontainerDir = path.join(root, ".devcontainer");
-  fs.mkdirSync(devcontainerDir);
-  await fs.promises.writeFile(
-    path.join(devcontainerDir, "devcontainer.json"),
-    devcontainerContent,
-  );
-};

packages/create-llama/helpers/dir.ts

@@ -1,3 +0,0 @@
-import path from "path";
-
-export const templatesDir = path.join(__dirname, "..", "templates");

packages/create-llama/helpers/get-pkg-manager.ts

@@ -1,15 +0,0 @@
-export type PackageManager = "npm" | "pnpm" | "yarn";
-
-export function getPkgManager(): PackageManager {
-  const userAgent = process.env.npm_config_user_agent || "";
-
-  if (userAgent.startsWith("yarn")) {
-    return "yarn";
-  }
-
-  if (userAgent.startsWith("pnpm")) {
-    return "pnpm";
-  }
-
-  return "npm";
-}

packages/create-llama/helpers/git.ts

@@ -1,58 +0,0 @@
-/* eslint-disable import/no-extraneous-dependencies */
-import { execSync } from "child_process";
-import fs from "fs";
-import path from "path";
-
-function isInGitRepository(): boolean {
-  try {
-    execSync("git rev-parse --is-inside-work-tree", { stdio: "ignore" });
-    return true;
-  } catch (_) {}
-  return false;
-}
-
-function isInMercurialRepository(): boolean {
-  try {
-    execSync("hg --cwd . root", { stdio: "ignore" });
-    return true;
-  } catch (_) {}
-  return false;
-}
-
-function isDefaultBranchSet(): boolean {
-  try {
-    execSync("git config init.defaultBranch", { stdio: "ignore" });
-    return true;
-  } catch (_) {}
-  return false;
-}
-
-export function tryGitInit(root: string): boolean {
-  let didInit = false;
-  try {
-    execSync("git --version", { stdio: "ignore" });
-    if (isInGitRepository() || isInMercurialRepository()) {
-      return false;
-    }
-
-    execSync("git init", { stdio: "ignore" });
-    didInit = true;
-
-    if (!isDefaultBranchSet()) {
-      execSync("git checkout -b main", { stdio: "ignore" });
-    }
-
-    execSync("git add -A", { stdio: "ignore" });
-    execSync('git commit -m "Initial commit from Create Llama"', {
-      stdio: "ignore",
-    });
-    return true;
-  } catch (e) {
-    if (didInit) {
-      try {
-        fs.rmSync(path.join(root, ".git"), { recursive: true, force: true });
-      } catch (_) {}
-    }
-    return false;
-  }
-}

packages/create-llama/helpers/index.ts

@@ -1,268 +0,0 @@
-import { copy } from "./copy";
-import { callPackageManager } from "./install";
-
-import fs from "fs/promises";
-import path from "path";
-import { cyan } from "picocolors";
-
-import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./constant";
-import { templatesDir } from "./dir";
-import { PackageManager } from "./get-pkg-manager";
-import { installLlamapackProject } from "./llama-pack";
-import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry";
-import { installPythonTemplate } from "./python";
-import { downloadAndExtractRepo } from "./repo";
-import {
-  FileSourceConfig,
-  InstallTemplateArgs,
-  TemplateDataSource,
-  TemplateFramework,
-  TemplateVectorDB,
-  WebSourceConfig,
-} from "./types";
-import { installTSTemplate } from "./typescript";
-
-const createEnvLocalFile = async (
-  root: string,
-  opts?: {
-    openAiKey?: string;
-    llamaCloudKey?: string;
-    vectorDb?: TemplateVectorDB;
-    model?: string;
-    embeddingModel?: string;
-    framework?: TemplateFramework;
-    dataSource?: TemplateDataSource;
-  },
-) => {
-  const envFileName = ".env";
-  let content = "";
-
-  const model = opts?.model || "gpt-3.5-turbo";
-  content += `MODEL=${model}\n`;
-  if (opts?.framework === "nextjs") {
-    content += `NEXT_PUBLIC_MODEL=${model}\n`;
-  }
-  console.log("\nUsing OpenAI model: ", model, "\n");
-
-  if (opts?.openAiKey) {
-    content += `OPENAI_API_KEY=${opts?.openAiKey}\n`;
-  }
-
-  if (opts?.embeddingModel) {
-    content += `EMBEDDING_MODEL=${opts?.embeddingModel}\n`;
-  }
-
-  if ((opts?.dataSource?.config as FileSourceConfig).useLlamaParse) {
-    if (opts?.llamaCloudKey) {
-      content += `LLAMA_CLOUD_API_KEY=${opts?.llamaCloudKey}\n`;
-    } else {
-      content += `# Please obtain the Llama Cloud API key from https://cloud.llamaindex.ai/api-key 
-# and set it to the LLAMA_CLOUD_API_KEY variable below.
-# LLAMA_CLOUD_API_KEY=`;
-    }
-  }
-
-  switch (opts?.vectorDb) {
-    case "mongo": {
-      content += `# For generating a connection URI, see https://www.mongodb.com/docs/guides/atlas/connection-string\n`;
-      content += `MONGO_URI=\n`;
-      content += `MONGODB_DATABASE=\n`;
-      content += `MONGODB_VECTORS=\n`;
-      content += `MONGODB_VECTOR_INDEX=\n`;
-      break;
-    }
-    case "pg": {
-      content += `# For generating a connection URI, see https://docs.timescale.com/use-timescale/latest/services/create-a-service\n`;
-      content += `PG_CONNECTION_STRING=\n`;
-      break;
-    }
-    case "pinecone": {
-      content += `PINECONE_API_KEY=\n`;
-      content += `PINECONE_ENVIRONMENT=\n`;
-      content += `PINECONE_INDEX_NAME=\n`;
-      break;
-    }
-  }
-
-  switch (opts?.dataSource?.type) {
-    case "web": {
-      const webConfig = opts?.dataSource.config as WebSourceConfig;
-      content += `# web loader config\n`;
-      content += `BASE_URL=${webConfig.baseUrl}\n`;
-      content += `URL_PREFIX=${webConfig.baseUrl}\n`;
-      content += `MAX_DEPTH=${webConfig.depth}\n`;
-      break;
-    }
-  }
-
-  if (content) {
-    await fs.writeFile(path.join(root, envFileName), content);
-    console.log(`Created '${envFileName}' file. Please check the settings.`);
-  }
-};
-
-// eslint-disable-next-line max-params
-async function generateContextData(
-  framework: TemplateFramework,
-  packageManager?: PackageManager,
-  openAiKey?: string,
-  vectorDb?: TemplateVectorDB,
-  dataSource?: TemplateDataSource,
-  llamaCloudKey?: string,
-) {
-  if (packageManager) {
-    const runGenerate = `${cyan(
-      framework === "fastapi"
-        ? "poetry run python app/engine/generate.py"
-        : `${packageManager} run generate`,
-    )}`;
-    const openAiKeyConfigured = openAiKey || process.env["OPENAI_API_KEY"];
-    const llamaCloudKeyConfigured = (dataSource?.config as FileSourceConfig)
-      ?.useLlamaParse
-      ? llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
-      : true;
-    const hasVectorDb = vectorDb && vectorDb !== "none";
-    if (framework === "fastapi") {
-      if (
-        openAiKeyConfigured &&
-        llamaCloudKeyConfigured &&
-        !hasVectorDb &&
-        isHavingPoetryLockFile()
-      ) {
-        console.log(`Running ${runGenerate} to generate the context data.`);
-        const result = tryPoetryRun("python app/engine/generate.py");
-        if (!result) {
-          console.log(`Failed to run ${runGenerate}.`);
-          process.exit(1);
-        }
-        console.log(`Generated context data`);
-        return;
-      }
-    } else {
-      if (openAiKeyConfigured && vectorDb === "none") {
-        console.log(`Running ${runGenerate} to generate the context data.`);
-        await callPackageManager(packageManager, true, ["run", "generate"]);
-        return;
-      }
-    }
-
-    const settings = [];
-    if (!openAiKeyConfigured) settings.push("your OpenAI key");
-    if (!llamaCloudKeyConfigured) settings.push("your Llama Cloud key");
-    if (hasVectorDb) settings.push("your Vector DB environment variables");
-    const settingsMessage =
-      settings.length > 0 ? `After setting ${settings.join(" and ")}, ` : "";
-    const generateMessage = `run ${runGenerate} to generate the context data.`;
-    console.log(`\n${settingsMessage}${generateMessage}\n\n`);
-  }
-}
-
-const copyContextData = async (
-  root: string,
-  dataSource?: TemplateDataSource,
-) => {
-  const destPath = path.join(root, "data");
-
-  const dataSourceConfig = dataSource?.config as FileSourceConfig;
-
-  // Copy file
-  if (dataSource?.type === "file") {
-    if (dataSourceConfig.path) {
-      console.log(`\nCopying file to ${cyan(destPath)}\n`);
-      await fs.mkdir(destPath, { recursive: true });
-      await fs.copyFile(
-        dataSourceConfig.path,
-        path.join(destPath, path.basename(dataSourceConfig.path)),
-      );
-    } else {
-      console.log("Missing file path in config");
-      process.exit(1);
-    }
-    return;
-  }
-
-  // Copy folder
-  if (dataSource?.type === "folder") {
-    const srcPath =
-      dataSourceConfig.path ?? path.join(templatesDir, "components", "data");
-    console.log(`\nCopying data to ${cyan(destPath)}\n`);
-    await copy("**", destPath, {
-      parents: true,
-      cwd: srcPath,
-    });
-    return;
-  }
-};
-
-const installCommunityProject = async ({
-  root,
-  communityProjectPath,
-}: Pick<InstallTemplateArgs, "root" | "communityProjectPath">) => {
-  console.log("\nInstalling community project:", communityProjectPath!);
-  await downloadAndExtractRepo(root, {
-    username: COMMUNITY_OWNER,
-    name: COMMUNITY_REPO,
-    branch: "main",
-    filePath: communityProjectPath!,
-  });
-};
-
-export const installTemplate = async (
-  props: InstallTemplateArgs & { backend: boolean },
-) => {
-  process.chdir(props.root);
-
-  if (props.template === "community" && props.communityProjectPath) {
-    await installCommunityProject(props);
-    return;
-  }
-
-  if (props.template === "llamapack" && props.llamapack) {
-    await installLlamapackProject(props);
-    return;
-  }
-
-  if (props.framework === "fastapi") {
-    await installPythonTemplate(props);
-  } else {
-    await installTSTemplate(props);
-  }
-
-  if (props.backend) {
-    // This is a backend, so we need to copy the test data and create the env file.
-
-    // Copy the environment file to the target directory.
-    await createEnvLocalFile(props.root, {
-      openAiKey: props.openAiKey,
-      llamaCloudKey: props.llamaCloudKey,
-      vectorDb: props.vectorDb,
-      model: props.model,
-      embeddingModel: props.embeddingModel,
-      framework: props.framework,
-      dataSource: props.dataSource,
-    });
-
-    if (props.engine === "context") {
-      await copyContextData(props.root, props.dataSource);
-      if (
-        props.postInstallAction === "runApp" ||
-        props.postInstallAction === "dependencies"
-      ) {
-        await generateContextData(
-          props.framework,
-          props.packageManager,
-          props.openAiKey,
-          props.vectorDb,
-          props.dataSource,
-          props.llamaCloudKey,
-        );
-      }
-    }
-  } else {
-    // this is a frontend for a full-stack app, create .env file with model information
-    const content = `MODEL=${props.model}\nNEXT_PUBLIC_MODEL=${props.model}\n`;
-    await fs.writeFile(path.join(props.root, ".env"), content);
-  }
-};
-
-export * from "./types";

packages/create-llama/helpers/install.ts

@@ -1,50 +0,0 @@
-/* eslint-disable import/no-extraneous-dependencies */
-import spawn from "cross-spawn";
-import { yellow } from "picocolors";
-import type { PackageManager } from "./get-pkg-manager";
-
-/**
- * Spawn a package manager installation based on user preference.
- *
- * @returns A Promise that resolves once the installation is finished.
- */
-export async function callPackageManager(
-  /** Indicate which package manager to use. */
-  packageManager: PackageManager,
-  /** Indicate whether there is an active Internet connection.*/
-  isOnline: boolean,
-  args: string[] = ["install"],
-): Promise<void> {
-  if (!isOnline) {
-    console.log(
-      yellow("You appear to be offline.\nFalling back to the local cache."),
-    );
-    args.push("--offline");
-  }
-  /**
-   * Return a Promise that resolves once the installation is finished.
-   */
-  return new Promise((resolve, reject) => {
-    /**
-     * Spawn the installation process.
-     */
-    const child = spawn(packageManager, args, {
-      stdio: "inherit",
-      env: {
-        ...process.env,
-        ADBLOCK: "1",
-        // we set NODE_ENV to development as pnpm skips dev
-        // dependencies when production
-        NODE_ENV: "development",
-        DISABLE_OPENCOLLECTIVE: "1",
-      },
-    });
-    child.on("close", (code) => {
-      if (code !== 0) {
-        reject({ command: `${packageManager} ${args.join(" ")}` });
-        return;
-      }
-      resolve();
-    });
-  });
-}

packages/create-llama/helpers/is-folder-empty.ts

@@ -1,62 +0,0 @@
-/* eslint-disable import/no-extraneous-dependencies */
-import fs from "fs";
-import path from "path";
-import { blue, green } from "picocolors";
-
-export function isFolderEmpty(root: string, name: string): boolean {
-  const validFiles = [
-    ".DS_Store",
-    ".git",
-    ".gitattributes",
-    ".gitignore",
-    ".gitlab-ci.yml",
-    ".hg",
-    ".hgcheck",
-    ".hgignore",
-    ".idea",
-    ".npmignore",
-    ".travis.yml",
-    "LICENSE",
-    "Thumbs.db",
-    "docs",
-    "mkdocs.yml",
-    "npm-debug.log",
-    "yarn-debug.log",
-    "yarn-error.log",
-    "yarnrc.yml",
-    ".yarn",
-  ];
-
-  const conflicts = fs
-    .readdirSync(root)
-    .filter((file) => !validFiles.includes(file))
-    // Support IntelliJ IDEA-based editors
-    .filter((file) => !/\.iml$/.test(file));
-
-  if (conflicts.length > 0) {
-    console.log(
-      `The directory ${green(name)} contains files that could conflict:`,
-    );
-    console.log();
-    for (const file of conflicts) {
-      try {
-        const stats = fs.lstatSync(path.join(root, file));
-        if (stats.isDirectory()) {
-          console.log(`  ${blue(file)}/`);
-        } else {
-          console.log(`  ${file}`);
-        }
-      } catch {
-        console.log(`  ${file}`);
-      }
-    }
-    console.log();
-    console.log(
-      "Either try using a new directory name, or remove the files listed above.",
-    );
-    console.log();
-    return false;
-  }
-
-  return true;
-}

packages/create-llama/helpers/is-online.ts

@@ -1,40 +0,0 @@
-import { execSync } from "child_process";
-import dns from "dns";
-import url from "url";
-
-function getProxy(): string | undefined {
-  if (process.env.https_proxy) {
-    return process.env.https_proxy;
-  }
-
-  try {
-    const httpsProxy = execSync("npm config get https-proxy").toString().trim();
-    return httpsProxy !== "null" ? httpsProxy : undefined;
-  } catch (e) {
-    return;
-  }
-}
-
-export function getOnline(): Promise<boolean> {
-  return new Promise((resolve) => {
-    dns.lookup("registry.yarnpkg.com", (registryErr) => {
-      if (!registryErr) {
-        return resolve(true);
-      }
-
-      const proxy = getProxy();
-      if (!proxy) {
-        return resolve(false);
-      }
-
-      const { hostname } = url.parse(proxy);
-      if (!hostname) {
-        return resolve(false);
-      }
-
-      dns.lookup(hostname, (proxyErr) => {
-        resolve(proxyErr == null);
-      });
-    });
-  });
-}

packages/create-llama/helpers/is-url.ts

@@ -1,8 +0,0 @@
-export function isUrl(url: string): boolean {
-  try {
-    new URL(url);
-    return true;
-  } catch (error) {
-    return false;
-  }
-}

packages/create-llama/helpers/is-writeable.ts

@@ -1,10 +0,0 @@
-import fs from "fs";
-
-export async function isWriteable(directory: string): Promise<boolean> {
-  try {
-    await fs.promises.access(directory, (fs.constants || fs).W_OK);
-    return true;
-  } catch (err) {
-    return false;
-  }
-}

packages/create-llama/helpers/llama-pack.ts

@@ -1,148 +0,0 @@
-import fs from "fs/promises";
-import got from "got";
-import path from "path";
-import { parse } from "smol-toml";
-import {
-  LLAMA_PACK_FOLDER,
-  LLAMA_PACK_FOLDER_PATH,
-  LLAMA_PACK_OWNER,
-  LLAMA_PACK_REPO,
-} from "./constant";
-import { copy } from "./copy";
-import { templatesDir } from "./dir";
-import { addDependencies, installPythonDependencies } from "./python";
-import { getRepoRawContent } from "./repo";
-import { InstallTemplateArgs } from "./types";
-
-const getLlamaPackFolderSHA = async () => {
-  const url = `https://api.github.com/repos/${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/contents`;
-  const response = await got(url, {
-    responseType: "json",
-  });
-  const data = response.body as any[];
-  const llamaPackFolder = data.find((item) => item.name === LLAMA_PACK_FOLDER);
-  return llamaPackFolder.sha;
-};
-
-const getLLamaPackFolderTree = async (
-  sha: string,
-): Promise<
-  Array<{
-    path: string;
-  }>
-> => {
-  const url = `https://api.github.com/repos/${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/git/trees/${sha}?recursive=1`;
-  const response = await got(url, {
-    responseType: "json",
-  });
-  return (response.body as any).tree;
-};
-
-export async function getAvailableLlamapackOptions(): Promise<
-  {
-    name: string;
-    folderPath: string;
-  }[]
-> {
-  const EXAMPLE_RELATIVE_PATH = "/examples/example.py";
-  const PACK_FOLDER_SUBFIX = "llama-index-packs";
-
-  const llamaPackFolderSHA = await getLlamaPackFolderSHA();
-  const llamaPackTree = await getLLamaPackFolderTree(llamaPackFolderSHA);
-
-  // Return options that have example files
-  const exampleFiles = llamaPackTree.filter((item) =>
-    item.path.endsWith(EXAMPLE_RELATIVE_PATH),
-  );
-  const options = exampleFiles.map((file) => {
-    const packFolder = file.path.substring(
-      0,
-      file.path.indexOf(EXAMPLE_RELATIVE_PATH),
-    );
-    const packName = packFolder.substring(PACK_FOLDER_SUBFIX.length + 1);
-    return {
-      name: packName,
-      folderPath: packFolder,
-    };
-  });
-  return options;
-}
-
-const copyLlamapackEmptyProject = async ({
-  root,
-}: Pick<InstallTemplateArgs, "root">) => {
-  const templatePath = path.join(
-    templatesDir,
-    "components/sample-projects/llamapack",
-  );
-  await copy("**", root, {
-    parents: true,
-    cwd: templatePath,
-  });
-};
-
-const copyData = async ({
-  root,
-}: Pick<InstallTemplateArgs, "root" | "llamapack">) => {
-  const dataPath = path.join(templatesDir, "components/data");
-  await copy("**", path.join(root, "data"), {
-    parents: true,
-    cwd: dataPath,
-  });
-};
-
-const installLlamapackExample = async ({
-  root,
-  llamapack,
-}: Pick<InstallTemplateArgs, "root" | "llamapack">) => {
-  const exampleFileName = "example.py";
-  const readmeFileName = "README.md";
-  const projectTomlFileName = "pyproject.toml";
-  const exampleFilePath = `${LLAMA_PACK_FOLDER_PATH}/${llamapack}/examples/${exampleFileName}`;
-  const readmeFilePath = `${LLAMA_PACK_FOLDER_PATH}/${llamapack}/${readmeFileName}`;
-  const projectTomlFilePath = `${LLAMA_PACK_FOLDER_PATH}/${llamapack}/${projectTomlFileName}`;
-
-  // Download example.py from llamapack and save to root
-  const exampleContent = await getRepoRawContent(exampleFilePath);
-  await fs.writeFile(path.join(root, exampleFileName), exampleContent);
-
-  // Download README.md from llamapack and combine with README-template.md,
-  // save to root and then delete template file
-  const readmeContent = await getRepoRawContent(readmeFilePath);
-  const readmeTemplateContent = await fs.readFile(
-    path.join(root, "README-template.md"),
-    "utf-8",
-  );
-  await fs.writeFile(
-    path.join(root, readmeFileName),
-    `${readmeContent}\n${readmeTemplateContent}`,
-  );
-  await fs.unlink(path.join(root, "README-template.md"));
-
-  // Download pyproject.toml from llamapack, parse it to get package name and version,
-  // then add it as a dependency to current toml file in the project
-  const projectTomlContent = await getRepoRawContent(projectTomlFilePath);
-  const fileParsed = parse(projectTomlContent) as any;
-  const packageName = fileParsed.tool.poetry.name;
-  const packageVersion = fileParsed.tool.poetry.version;
-  await addDependencies(root, [
-    {
-      name: packageName,
-      version: packageVersion,
-    },
-  ]);
-};
-
-export const installLlamapackProject = async ({
-  root,
-  llamapack,
-  postInstallAction,
-}: Pick<InstallTemplateArgs, "root" | "llamapack" | "postInstallAction">) => {
-  console.log("\nInstalling Llamapack project:", llamapack!);
-  await copyLlamapackEmptyProject({ root });
-  await copyData({ root });
-  await installLlamapackExample({ root, llamapack });
-  if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
-    installPythonDependencies({ noRoot: true });
-  }
-};

packages/create-llama/helpers/make-dir.ts

@@ -1,8 +0,0 @@
-import fs from "fs";
-
-export function makeDir(
-  root: string,
-  options = { recursive: true },
-): Promise<string | undefined> {
-  return fs.promises.mkdir(root, options);
-}

packages/create-llama/helpers/poetry.ts

@@ -1,36 +0,0 @@
-/* eslint-disable import/no-extraneous-dependencies */
-import { execSync } from "child_process";
-import fs from "fs";
-
-export function isPoetryAvailable(): boolean {
-  try {
-    execSync("poetry --version", { stdio: "ignore" });
-    return true;
-  } catch (_) {}
-  return false;
-}
-
-export function tryPoetryInstall(noRoot: boolean): boolean {
-  try {
-    execSync(`poetry install${noRoot ? " --no-root" : ""}`, {
-      stdio: "inherit",
-    });
-    return true;
-  } catch (_) {}
-  return false;
-}
-
-export function tryPoetryRun(command: string): boolean {
-  try {
-    execSync(`poetry run ${command}`, { stdio: "inherit" });
-    return true;
-  } catch (_) {}
-  return false;
-}
-
-export function isHavingPoetryLockFile(): boolean {
-  try {
-    return fs.existsSync("poetry.lock");
-  } catch (_) {}
-  return false;
-}

packages/create-llama/helpers/python.ts

@@ -1,272 +0,0 @@
-import fs from "fs/promises";
-import path from "path";
-import { cyan, red } from "picocolors";
-import { parse, stringify } from "smol-toml";
-import terminalLink from "terminal-link";
-import { copy } from "./copy";
-import { templatesDir } from "./dir";
-import { isPoetryAvailable, tryPoetryInstall } from "./poetry";
-import { Tool } from "./tools";
-import {
-  FileSourceConfig,
-  InstallTemplateArgs,
-  TemplateDataSource,
-  TemplateVectorDB,
-} from "./types";
-
-interface Dependency {
-  name: string;
-  version?: string;
-  extras?: string[];
-}
-
-const getAdditionalDependencies = (
-  vectorDb?: TemplateVectorDB,
-  dataSource?: TemplateDataSource,
-  tools?: Tool[],
-) => {
-  const dependencies: Dependency[] = [];
-
-  // Add vector db dependencies
-  switch (vectorDb) {
-    case "mongo": {
-      dependencies.push({
-        name: "llama-index-vector-stores-mongodb",
-        version: "^0.1.3",
-      });
-      break;
-    }
-    case "pg": {
-      dependencies.push({
-        name: "llama-index-vector-stores-postgres",
-        version: "^0.1.1",
-      });
-    }
-    case "pinecone": {
-      dependencies.push({
-        name: "llama-index-vector-stores-pinecone",
-        version: "^0.1.3",
-      });
-      break;
-    }
-  }
-
-  // Add data source dependencies
-  const dataSourceType = dataSource?.type;
-  if (dataSourceType === "file" || dataSourceType === "folder") {
-    // llama-index-readers-file (pdf, excel, csv) is already included in llama_index package
-    dependencies.push({
-      name: "docx2txt",
-      version: "^0.8",
-    });
-  } else if (dataSourceType === "web") {
-    dependencies.push({
-      name: "llama-index-readers-web",
-      version: "^0.1.6",
-    });
-  }
-
-  // Add tools dependencies
-  tools?.forEach((tool) => {
-    tool.dependencies?.forEach((dep) => {
-      dependencies.push(dep);
-    });
-  });
-
-  return dependencies;
-};
-
-const mergePoetryDependencies = (
-  dependencies: Dependency[],
-  existingDependencies: Record<string, Omit<Dependency, "name">>,
-) => {
-  for (const dependency of dependencies) {
-    let value = existingDependencies[dependency.name] ?? {};
-
-    // default string value is equal to attribute "version"
-    if (typeof value === "string") {
-      value = { version: value };
-    }
-
-    value.version = dependency.version ?? value.version;
-    value.extras = dependency.extras ?? value.extras;
-
-    if (value.version === undefined) {
-      throw new Error(
-        `Dependency "${dependency.name}" is missing attribute "version"!`,
-      );
-    }
-
-    existingDependencies[dependency.name] = value;
-  }
-};
-
-export const addDependencies = async (
-  projectDir: string,
-  dependencies: Dependency[],
-) => {
-  if (dependencies.length === 0) return;
-
-  const FILENAME = "pyproject.toml";
-  try {
-    // Parse toml file
-    const file = path.join(projectDir, FILENAME);
-    const fileContent = await fs.readFile(file, "utf8");
-    const fileParsed = parse(fileContent);
-
-    // Modify toml dependencies
-    const tool = fileParsed.tool as any;
-    const existingDependencies = tool.poetry.dependencies;
-    mergePoetryDependencies(dependencies, existingDependencies);
-
-    // Write toml file
-    const newFileContent = stringify(fileParsed);
-    await fs.writeFile(file, newFileContent);
-
-    const dependenciesString = dependencies.map((d) => d.name).join(", ");
-    console.log(`\nAdded ${dependenciesString} to ${cyan(FILENAME)}\n`);
-  } catch (error) {
-    console.log(
-      `Error while updating dependencies for Poetry project file ${FILENAME}\n`,
-      error,
-    );
-  }
-};
-
-export const installPythonDependencies = (
-  { noRoot }: { noRoot: boolean } = { noRoot: false },
-) => {
-  if (isPoetryAvailable()) {
-    console.log(
-      `Installing python dependencies using poetry. This may take a while...`,
-    );
-    const installSuccessful = tryPoetryInstall(noRoot);
-    if (!installSuccessful) {
-      console.error(
-        red(
-          "Installing dependencies using poetry failed. Please check error log above and try running create-llama again.",
-        ),
-      );
-      process.exit(1);
-    }
-  } else {
-    console.error(
-      red(
-        `Poetry is not available in the current environment. Please check ${terminalLink(
-          "Poetry Installation",
-          `https://python-poetry.org/docs/#installation`,
-        )} to install poetry first, then run create-llama again.`,
-      ),
-    );
-    process.exit(1);
-  }
-};
-
-export const installPythonTemplate = async ({
-  root,
-  template,
-  framework,
-  engine,
-  vectorDb,
-  dataSource,
-  tools,
-  postInstallAction,
-}: Pick<
-  InstallTemplateArgs,
-  | "root"
-  | "framework"
-  | "template"
-  | "engine"
-  | "vectorDb"
-  | "dataSource"
-  | "tools"
-  | "postInstallAction"
->) => {
-  console.log("\nInitializing Python project with template:", template, "\n");
-  const templatePath = path.join(templatesDir, "types", template, framework);
-  await copy("**", root, {
-    parents: true,
-    cwd: templatePath,
-    rename(name) {
-      switch (name) {
-        case "gitignore": {
-          return `.${name}`;
-        }
-        // README.md is ignored by webpack-asset-relocator-loader used by ncc:
-        // https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
-        case "README-template.md": {
-          return "README.md";
-        }
-        default: {
-          return name;
-        }
-      }
-    },
-  });
-
-  if (engine === "context") {
-    const enginePath = path.join(root, "app", "engine");
-    const compPath = path.join(templatesDir, "components");
-
-    const vectorDbDirName = vectorDb ?? "none";
-    const VectorDBPath = path.join(
-      compPath,
-      "vectordbs",
-      "python",
-      vectorDbDirName,
-    );
-    await copy("**", enginePath, {
-      parents: true,
-      cwd: VectorDBPath,
-    });
-
-    // Copy engine code
-    if (tools !== undefined && tools.length > 0) {
-      await copy("**", enginePath, {
-        parents: true,
-        cwd: path.join(compPath, "engines", "python", "agent"),
-      });
-      // Write tools_config.json
-      const configContent: Record<string, any> = {};
-      tools.forEach((tool) => {
-        configContent[tool.name] = tool.config ?? {};
-      });
-      const configFilePath = path.join(root, "tools_config.json");
-      await fs.writeFile(
-        configFilePath,
-        JSON.stringify(configContent, null, 2),
-      );
-    } else {
-      await copy("**", enginePath, {
-        parents: true,
-        cwd: path.join(compPath, "engines", "python", "chat"),
-      });
-    }
-
-    const dataSourceType = dataSource?.type;
-    if (dataSourceType !== undefined && dataSourceType !== "none") {
-      let loaderFolder: string;
-      if (dataSourceType === "file" || dataSourceType === "folder") {
-        const dataSourceConfig = dataSource?.config as FileSourceConfig;
-        loaderFolder = dataSourceConfig.useLlamaParse ? "llama_parse" : "file";
-      } else {
-        loaderFolder = dataSourceType;
-      }
-      await copy("**", enginePath, {
-        parents: true,
-        cwd: path.join(compPath, "loaders", "python", loaderFolder),
-      });
-    }
-  }
-
-  const addOnDependencies = getAdditionalDependencies(
-    vectorDb,
-    dataSource,
-    tools,
-  );
-  await addDependencies(root, addOnDependencies);
-
-  if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
-    installPythonDependencies();
-  }
-};

packages/create-llama/helpers/repo.ts

@@ -1,71 +0,0 @@
-import { createWriteStream, promises } from "fs";
-import got from "got";
-import { tmpdir } from "os";
-import { join } from "path";
-import { Stream } from "stream";
-import tar from "tar";
-import { promisify } from "util";
-import { makeDir } from "./make-dir";
-
-export type RepoInfo = {
-  username: string;
-  name: string;
-  branch: string;
-  filePath: string;
-};
-
-const pipeline = promisify(Stream.pipeline);
-
-async function downloadTar(url: string) {
-  const tempFile = join(tmpdir(), `next.js-cna-example.temp-${Date.now()}`);
-  await pipeline(got.stream(url), createWriteStream(tempFile));
-  return tempFile;
-}
-
-export async function downloadAndExtractRepo(
-  root: string,
-  { username, name, branch, filePath }: RepoInfo,
-) {
-  await makeDir(root);
-
-  const tempFile = await downloadTar(
-    `https://codeload.github.com/${username}/${name}/tar.gz/${branch}`,
-  );
-
-  await tar.x({
-    file: tempFile,
-    cwd: root,
-    strip: filePath ? filePath.split("/").length + 1 : 1,
-    filter: (p) =>
-      p.startsWith(
-        `${name}-${branch.replace(/\//g, "-")}${
-          filePath ? `/${filePath}/` : "/"
-        }`,
-      ),
-  });
-
-  await promises.unlink(tempFile);
-}
-
-export async function getRepoRootFolders(
-  owner: string,
-  repo: string,
-): Promise<string[]> {
-  const url = `https://api.github.com/repos/${owner}/${repo}/contents`;
-
-  const response = await got(url, {
-    responseType: "json",
-  });
-
-  const data = response.body as any[];
-  const folders = data.filter((item) => item.type === "dir");
-  return folders.map((item) => item.name);
-}
-
-export async function getRepoRawContent(repoFilePath: string) {
-  const url = `https://raw.githubusercontent.com/${repoFilePath}`;
-  const response = await got(url, {
-    responseType: "text",
-  });
-  return response.body;
-}

packages/create-llama/helpers/run-app.ts

@@ -1,88 +0,0 @@
-import { ChildProcess, SpawnOptions, spawn } from "child_process";
-import path from "path";
-import { TemplateFramework } from "./types";
-
-const createProcess = (
-  command: string,
-  args: string[],
-  options: SpawnOptions,
-) => {
-  return spawn(command, args, {
-    ...options,
-    shell: true,
-  })
-    .on("exit", function (code) {
-      if (code !== 0) {
-        console.log(`Child process exited with code=${code}`);
-        process.exit(1);
-      }
-    })
-    .on("error", function (err) {
-      console.log("Error when running chill process: ", err);
-      process.exit(1);
-    });
-};
-
-// eslint-disable-next-line max-params
-export async function runApp(
-  appPath: string,
-  frontend: boolean,
-  framework: TemplateFramework,
-  port?: number,
-  externalPort?: number,
-): Promise<any> {
-  let backendAppProcess: ChildProcess;
-  let frontendAppProcess: ChildProcess | undefined;
-  const frontendPort = port || 3000;
-  let backendPort = externalPort || 8000;
-
-  // Callback to kill app processes
-  process.on("exit", () => {
-    console.log("Killing app processes...");
-    backendAppProcess.kill();
-    frontendAppProcess?.kill();
-  });
-
-  let backendCommand = "";
-  let backendArgs: string[];
-  if (framework === "fastapi") {
-    backendCommand = "poetry";
-    backendArgs = [
-      "run",
-      "uvicorn",
-      "main:app",
-      "--host=0.0.0.0",
-      "--port=" + backendPort,
-    ];
-  } else if (framework === "nextjs") {
-    backendCommand = "npm";
-    backendArgs = ["run", "dev"];
-    backendPort = frontendPort;
-  } else {
-    backendCommand = "npm";
-    backendArgs = ["run", "dev"];
-  }
-
-  if (frontend) {
-    return new Promise((resolve, reject) => {
-      backendAppProcess = createProcess(backendCommand, backendArgs, {
-        stdio: "inherit",
-        cwd: path.join(appPath, "backend"),
-        env: { ...process.env, PORT: `${backendPort}` },
-      });
-      frontendAppProcess = createProcess("npm", ["run", "dev"], {
-        stdio: "inherit",
-        cwd: path.join(appPath, "frontend"),
-        env: { ...process.env, PORT: `${frontendPort}` },
-      });
-    });
-  } else {
-    return new Promise((resolve, reject) => {
-      backendAppProcess = createProcess(backendCommand, backendArgs, {
-        stdio: "inherit",
-        cwd: path.join(appPath),
-        env: { ...process.env, PORT: `${backendPort}` },
-      });
-    });
-  }
-}

packages/create-llama/helpers/tools.ts

@@ -1,71 +0,0 @@
-import { red } from "picocolors";
-
-export type Tool = {
-  display: string;
-  name: string;
-  config?: Record<string, any>;
-  dependencies?: ToolDependencies[];
-};
-export type ToolDependencies = {
-  name: string;
-  version?: string;
-};
-
-export const supportedTools: Tool[] = [
-  {
-    display: "Google Search (configuration required after installation)",
-    name: "google.GoogleSearchToolSpec",
-    config: {
-      engine:
-        "Your search engine id, see https://developers.google.com/custom-search/v1/overview#prerequisites",
-      key: "Your search api key",
-      num: 2,
-    },
-    dependencies: [
-      {
-        name: "llama-index-tools-google",
-        version: "0.1.2",
-      },
-    ],
-  },
-  {
-    display: "Wikipedia",
-    name: "wikipedia.WikipediaToolSpec",
-    dependencies: [
-      {
-        name: "llama-index-tools-wikipedia",
-        version: "0.1.2",
-      },
-    ],
-  },
-];
-
-export const getTool = (toolName: string): Tool | undefined => {
-  return supportedTools.find((tool) => tool.name === toolName);
-};
-
-export const getTools = (toolsName: string[]): Tool[] => {
-  const tools: Tool[] = [];
-  for (const toolName of toolsName) {
-    const tool = getTool(toolName);
-    if (!tool) {
-      console.log(
-        red(
-          `Error: Tool '${toolName}' is not supported. Supported tools are: ${supportedTools
-            .map((t) => t.name)
-            .join(", ")}`,
-        ),
-      );
-      process.exit(1);
-    }
-    tools.push(tool);
-  }
-  return tools;
-};
-
-export const toolsRequireConfig = (tools?: Tool[]): boolean => {
-  if (tools) {
-    return tools?.some((tool) => Object.keys(tool.config || {}).length > 0);
-  }
-  return false;
-};

packages/create-llama/helpers/types.ts

@@ -1,53 +0,0 @@
-import { PackageManager } from "../helpers/get-pkg-manager";
-import { Tool } from "./tools";
-
-export type TemplateType = "simple" | "streaming" | "community" | "llamapack";
-export type TemplateFramework = "nextjs" | "express" | "fastapi";
-export type TemplateEngine = "simple" | "context";
-export type TemplateUI = "html" | "shadcn";
-export type TemplateVectorDB = "none" | "mongo" | "pg" | "pinecone";
-export type TemplatePostInstallAction =
-  | "none"
-  | "VSCode"
-  | "dependencies"
-  | "runApp";
-export type TemplateDataSource = {
-  type: TemplateDataSourceType;
-  config: TemplateDataSourceConfig;
-};
-export type TemplateDataSourceType = "none" | "file" | "folder" | "web";
-// Config for both file and folder
-export type FileSourceConfig = {
-  path?: string;
-  useLlamaParse?: boolean;
-};
-export type WebSourceConfig = {
-  baseUrl?: string;
-  depth?: number;
-};
-export type TemplateDataSourceConfig = FileSourceConfig | WebSourceConfig;
-
-export interface InstallTemplateArgs {
-  appName: string;
-  root: string;
-  packageManager: PackageManager;
-  isOnline: boolean;
-  template: TemplateType;
-  framework: TemplateFramework;
-  engine: TemplateEngine;
-  ui: TemplateUI;
-  dataSource?: TemplateDataSource;
-  eslint: boolean;
-  customApiPath?: string;
-  openAiKey?: string;
-  llamaCloudKey?: string;
-  forBackend?: string;
-  model: string;
-  embeddingModel: string;
-  communityProjectPath?: string;
-  llamapack?: string;
-  vectorDb?: TemplateVectorDB;
-  externalPort?: number;
-  postInstallAction?: TemplatePostInstallAction;
-  tools?: Tool[];
-}

packages/create-llama/helpers/typescript.ts

@@ -1,234 +0,0 @@
-import fs from "fs/promises";
-import os from "os";
-import path from "path";
-import { bold, cyan } from "picocolors";
-import { version } from "../../core/package.json";
-import { copy } from "../helpers/copy";
-import { callPackageManager } from "../helpers/install";
-import { templatesDir } from "./dir";
-import { PackageManager } from "./get-pkg-manager";
-import { InstallTemplateArgs } from "./types";
-
-const rename = (name: string) => {
-  switch (name) {
-    case "gitignore":
-    case "eslintrc.json": {
-      return `.${name}`;
-    }
-    // README.md is ignored by webpack-asset-relocator-loader used by ncc:
-    // https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
-    case "README-template.md": {
-      return "README.md";
-    }
-    default: {
-      return name;
-    }
-  }
-};
-
-export const installTSDependencies = async (
-  packageJson: any,
-  packageManager: PackageManager,
-  isOnline: boolean,
-): Promise<void> => {
-  console.log("\nInstalling dependencies:");
-  for (const dependency in packageJson.dependencies)
-    console.log(`- ${cyan(dependency)}`);
-
-  console.log("\nInstalling devDependencies:");
-  for (const dependency in packageJson.devDependencies)
-    console.log(`- ${cyan(dependency)}`);
-
-  console.log();
-
-  await callPackageManager(packageManager, isOnline).catch((error) => {
-    console.error("Failed to install TS dependencies. Exiting...");
-    process.exit(1);
-  });
-};
-
-/**
- * Install a LlamaIndex internal template to a given `root` directory.
- */
-export const installTSTemplate = async ({
-  appName,
-  root,
-  packageManager,
-  isOnline,
-  template,
-  framework,
-  engine,
-  ui,
-  eslint,
-  customApiPath,
-  forBackend,
-  vectorDb,
-  postInstallAction,
-}: InstallTemplateArgs) => {
-  console.log(bold(`Using ${packageManager}.`));
-
-  /**
-   * Copy the template files to the target directory.
-   */
-  console.log("\nInitializing project with template:", template, "\n");
-  const templatePath = path.join(templatesDir, "types", template, framework);
-  const copySource = ["**"];
-  if (!eslint) copySource.push("!eslintrc.json");
-
-  await copy(copySource, root, {
-    parents: true,
-    cwd: templatePath,
-    rename,
-  });
-
-  /**
-   * If the backend is next.js, rename next.config.app.js to next.config.js
-   * If not, rename next.config.static.js to next.config.js
-   */
-  if (framework == "nextjs" && forBackend === "nextjs") {
-    const nextConfigAppPath = path.join(root, "next.config.app.js");
-    const nextConfigPath = path.join(root, "next.config.js");
-    await fs.rename(nextConfigAppPath, nextConfigPath);
-    // delete next.config.static.js
-    const nextConfigStaticPath = path.join(root, "next.config.static.js");
-    await fs.rm(nextConfigStaticPath);
-  } else if (framework == "nextjs" && typeof forBackend === "undefined") {
-    const nextConfigStaticPath = path.join(root, "next.config.static.js");
-    const nextConfigPath = path.join(root, "next.config.js");
-    await fs.rename(nextConfigStaticPath, nextConfigPath);
-    // delete next.config.app.js
-    const nextConfigAppPath = path.join(root, "next.config.app.js");
-    await fs.rm(nextConfigAppPath);
-  }
-
-  /**
-   * Copy the selected chat engine files to the target directory and reference it.
-   */
-  let relativeEngineDestPath;
-  const compPath = path.join(templatesDir, "components");
-  if (engine && (framework === "express" || framework === "nextjs")) {
-    console.log("\nUsing chat engine:", engine, "\n");
-
-    let vectorDBFolder: string = engine;
-
-    if (engine !== "simple" && vectorDb) {
-      console.log("\nUsing vector DB:", vectorDb, "\n");
-      vectorDBFolder = vectorDb;
-    }
-
-    const VectorDBPath = path.join(
-      compPath,
-      "vectordbs",
-      "typescript",
-      vectorDBFolder,
-    );
-    relativeEngineDestPath =
-      framework === "nextjs"
-        ? path.join("app", "api", "chat")
-        : path.join("src", "controllers");
-    await copy("**", path.join(root, relativeEngineDestPath, "engine"), {
-      parents: true,
-      cwd: VectorDBPath,
-    });
-  }
-
-  /**
-   * Copy the selected UI files to the target directory and reference it.
-   */
-  if (framework === "nextjs" && ui !== "shadcn") {
-    console.log("\nUsing UI:", ui, "\n");
-    const uiPath = path.join(compPath, "ui", ui);
-    const destUiPath = path.join(root, "app", "components", "ui");
-    // remove the default ui folder
-    await fs.rm(destUiPath, { recursive: true });
-    // copy the selected ui folder
-    await copy("**", destUiPath, {
-      parents: true,
-      cwd: uiPath,
-      rename,
-    });
-  }
-
-  /**
-   * Update the package.json scripts.
-   */
-  const packageJsonFile = path.join(root, "package.json");
-  const packageJson: any = JSON.parse(
-    await fs.readFile(packageJsonFile, "utf8"),
-  );
-  packageJson.name = appName;
-  packageJson.version = "0.1.0";
-
-  packageJson.dependencies = {
-    ...packageJson.dependencies,
-    llamaindex: version,
-  };
-
-  if (framework === "nextjs" && customApiPath) {
-    console.log(
-      "\nUsing external API with custom API path:",
-      customApiPath,
-      "\n",
-    );
-    // remove the default api folder
-    const apiPath = path.join(root, "app", "api");
-    await fs.rm(apiPath, { recursive: true });
-    // modify the dev script to use the custom api path
-    packageJson.scripts = {
-      ...packageJson.scripts,
-      dev: `cross-env NEXT_PUBLIC_CHAT_API=${customApiPath} next dev`,
-    };
-  }
-
-  if (engine === "context" && relativeEngineDestPath) {
-    // add generate script if using context engine
-    packageJson.scripts = {
-      ...packageJson.scripts,
-      generate: `node ${path.join(
-        relativeEngineDestPath,
-        "engine",
-        "generate.mjs",
-      )}`,
-    };
-  }
-
-  if (framework === "nextjs" && ui === "html") {
-    // remove shadcn dependencies if html ui is selected
-    packageJson.dependencies = {
-      ...packageJson.dependencies,
-      "tailwind-merge": undefined,
-      "@radix-ui/react-slot": undefined,
-      "class-variance-authority": undefined,
-      clsx: undefined,
-      "lucide-react": undefined,
-      remark: undefined,
-      "remark-code-import": undefined,
-      "remark-gfm": undefined,
-      "remark-math": undefined,
-      "react-markdown": undefined,
-      "react-syntax-highlighter": undefined,
-    };
-
-    packageJson.devDependencies = {
-      ...packageJson.devDependencies,
-      "@types/react-syntax-highlighter": undefined,
-    };
-  }
-
-  if (!eslint) {
-    // Remove packages starting with "eslint" from devDependencies
-    packageJson.devDependencies = Object.fromEntries(
-      Object.entries(packageJson.devDependencies).filter(
-        ([key]) => !key.startsWith("eslint"),
-      ),
-    );
-  }
-  await fs.writeFile(
-    packageJsonFile,
-    JSON.stringify(packageJson, null, 2) + os.EOL,
-  );
-
-  if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
-    await installTSDependencies(packageJson, packageManager, isOnline);
-  }
-};

packages/create-llama/helpers/validate-pkg.ts

@@ -1,20 +0,0 @@
-// eslint-disable-next-line import/no-extraneous-dependencies
-import validateProjectName from "validate-npm-package-name";
-
-export function validateNpmName(name: string): {
-  valid: boolean;
-  problems?: string[];
-} {
-  const nameValidation = validateProjectName(name);
-  if (nameValidation.validForNewPackages) {
-    return { valid: true };
-  }
-
-  return {
-    valid: false,
-    problems: [
-      ...(nameValidation.errors || []),
-      ...(nameValidation.warnings || []),
-    ],
-  };
-}

packages/create-llama/index.ts

@@ -1,383 +0,0 @@
-#!/usr/bin/env node
-/* eslint-disable import/no-extraneous-dependencies */
-import { execSync } from "child_process";
-import Commander from "commander";
-import Conf from "conf";
-import fs from "fs";
-import path from "path";
-import { bold, cyan, green, red, yellow } from "picocolors";
-import prompts from "prompts";
-import terminalLink from "terminal-link";
-import checkForUpdate from "update-check";
-import { createApp } from "./create-app";
-import { getPkgManager } from "./helpers/get-pkg-manager";
-import { isFolderEmpty } from "./helpers/is-folder-empty";
-import { runApp } from "./helpers/run-app";
-import { getTools } from "./helpers/tools";
-import { validateNpmName } from "./helpers/validate-pkg";
-import packageJson from "./package.json";
-import { QuestionArgs, askQuestions, onPromptState } from "./questions";
-
-let projectPath: string = "";
-
-const handleSigTerm = () => process.exit(0);
-
-process.on("SIGINT", handleSigTerm);
-process.on("SIGTERM", handleSigTerm);
-
-const program = new Commander.Command(packageJson.name)
-  .version(packageJson.version)
-  .arguments("<project-directory>")
-  .usage(`${green("<project-directory>")} [options]`)
-  .action((name) => {
-    projectPath = name;
-  })
-  .option(
-    "--eslint",
-    `
-
-  Initialize with eslint config.
-`,
-  )
-  .option(
-    "--use-npm",
-    `
-
-  Explicitly tell the CLI to bootstrap the application using npm
-`,
-  )
-  .option(
-    "--use-pnpm",
-    `
-
-  Explicitly tell the CLI to bootstrap the application using pnpm
-`,
-  )
-  .option(
-    "--use-yarn",
-    `
-
-  Explicitly tell the CLI to bootstrap the application using Yarn
-`,
-  )
-  .option(
-    "--reset-preferences",
-    `
-
-  Explicitly tell the CLI to reset any stored preferences
-`,
-  )
-  .option(
-    "--template <template>",
-    `
-
-  Select a template to bootstrap the application with.
-`,
-  )
-  .option(
-    "--engine <engine>",
-    `
-
-  Select a chat engine to bootstrap the application with.
-`,
-  )
-  .option(
-    "--framework <framework>",
-    `
-
-  Select a framework to bootstrap the application with.
-`,
-  )
-  .option(
-    "--files <path>",
-    `
-  
-    Specify the path to a local file or folder for chatting.
-`,
-  )
-  .option(
-    "--open-ai-key <key>",
-    `
-
-  Provide an OpenAI API key.
-`,
-  )
-  .option(
-    "--ui <ui>",
-    `
-
-  Select a UI to bootstrap the application with.
-`,
-  )
-  .option(
-    "--frontend",
-    `
-
-  Whether to generate a frontend for your backend.
-`,
-  )
-  .option(
-    "--model <model>",
-    `
-
-  Select OpenAI model to use. E.g. gpt-3.5-turbo.
-`,
-  )
-  .option(
-    "--embedding-model <embeddingModel>",
-    `
-  Select OpenAI embedding model to use. E.g. text-embedding-ada-002.
-`,
-  )
-  .option(
-    "--port <port>",
-    `
-
-  Select UI port.
-`,
-  )
-  .option(
-    "--external-port <external>",
-    `
-
-  Select external port.
-`,
-  )
-  .option(
-    "--post-install-action <action>",
-    `
-
-  Choose an action after installation. For example, 'runApp' or 'dependencies'. The default option is just to generate the app.
-`,
-  )
-  .option(
-    "--vector-db <vectorDb>",
-    `
-
-  Select which vector database you would like to use, such as 'none', 'pg' or 'mongo'. The default option is not to use a vector database and use the local filesystem instead ('none').
-`,
-  )
-  .option(
-    "--tools <tools>",
-    `
-
-  Specify the tools you want to use by providing a comma-separated list. For example, 'wikipedia.WikipediaToolSpec,google.GoogleSearchToolSpec'. Use 'none' to not using any tools.
-`,
-  )
-  .option(
-    "--llama-parse",
-    `
-    Enable LlamaParse.
-`,
-  )
-  .option(
-    "--llama-cloud-key <key>",
-    `
-  Provide a LlamaCloud API key.
-`,
-  )
-  .allowUnknownOption()
-  .parse(process.argv);
-if (process.argv.includes("--no-frontend")) {
-  program.frontend = false;
-}
-if (process.argv.includes("--no-eslint")) {
-  program.eslint = false;
-}
-if (process.argv.includes("--tools")) {
-  if (program.tools === "none") {
-    program.tools = [];
-  } else {
-    program.tools = getTools(program.tools.split(","));
-  }
-}
-if (process.argv.includes("--no-llama-parse")) {
-  program.llamaParse = false;
-}
-
-const packageManager = !!program.useNpm
-  ? "npm"
-  : !!program.usePnpm
-    ? "pnpm"
-    : !!program.useYarn
-      ? "yarn"
-      : getPkgManager();
-
-async function run(): Promise<void> {
-  const conf = new Conf({ projectName: "create-llama" });
-
-  if (program.resetPreferences) {
-    conf.clear();
-    console.log(`Preferences reset successfully`);
-    return;
-  }
-
-  if (typeof projectPath === "string") {
-    projectPath = projectPath.trim();
-  }
-
-  if (!projectPath) {
-    const res = await prompts({
-      onState: onPromptState,
-      type: "text",
-      name: "path",
-      message: "What is your project named?",
-      initial: "my-app",
-      validate: (name) => {
-        const validation = validateNpmName(path.basename(path.resolve(name)));
-        if (validation.valid) {
-          return true;
-        }
-        return "Invalid project name: " + validation.problems![0];
-      },
-    });
-
-    if (typeof res.path === "string") {
-      projectPath = res.path.trim();
-    }
-  }
-
-  if (!projectPath) {
-    console.log(
-      "\nPlease specify the project directory:\n" +
-        `  ${cyan(program.name())} ${green("<project-directory>")}\n` +
-        "For example:\n" +
-        `  ${cyan(program.name())} ${green("my-app")}\n\n` +
-        `Run ${cyan(`${program.name()} --help`)} to see all options.`,
-    );
-    process.exit(1);
-  }
-
-  const resolvedProjectPath = path.resolve(projectPath);
-  const projectName = path.basename(resolvedProjectPath);
-
-  const { valid, problems } = validateNpmName(projectName);
-  if (!valid) {
-    console.error(
-      `Could not create a project called ${red(
-        `"${projectName}"`,
-      )} because of npm naming restrictions:`,
-    );
-
-    problems!.forEach((p) => console.error(`    ${red(bold("*"))} ${p}`));
-    process.exit(1);
-  }
-
-  /**
-   * Verify the project dir is empty or doesn't exist
-   */
-  const root = path.resolve(resolvedProjectPath);
-  const appName = path.basename(root);
-  const folderExists = fs.existsSync(root);
-
-  if (folderExists && !isFolderEmpty(root, appName)) {
-    process.exit(1);
-  }
-
-  const preferences = (conf.get("preferences") || {}) as QuestionArgs;
-  await askQuestions(program as unknown as QuestionArgs, preferences);
-
-  await createApp({
-    template: program.template,
-    framework: program.framework,
-    engine: program.engine,
-    ui: program.ui,
-    appPath: resolvedProjectPath,
-    packageManager,
-    eslint: program.eslint,
-    frontend: program.frontend,
-    openAiKey: program.openAiKey,
-    llamaCloudKey: program.llamaCloudKey,
-    model: program.model,
-    embeddingModel: program.embeddingModel,
-    communityProjectPath: program.communityProjectPath,
-    llamapack: program.llamapack,
-    vectorDb: program.vectorDb,
-    externalPort: program.externalPort,
-    postInstallAction: program.postInstallAction,
-    dataSource: program.dataSource,
-    tools: program.tools,
-  });
-  conf.set("preferences", preferences);
-
-  if (program.postInstallAction === "VSCode") {
-    console.log(`Starting VSCode in ${root}...`);
-    try {
-      execSync(`code . --new-window --goto README.md`, {
-        stdio: "inherit",
-        cwd: root,
-      });
-    } catch (error) {
-      console.log(
-        red(
-          `Failed to start VSCode in ${root}. 
-Got error: ${(error as Error).message}.\n`,
-        ),
-      );
-      console.log(
-        `Make sure you have VSCode installed and added to your PATH. 
-Please check ${cyan(
-          terminalLink(
-            "This documentation",
-            `https://code.visualstudio.com/docs/setup/setup-overview`,
-          ),
-        )} for more information.`,
-      );
-    }
-  } else if (program.postInstallAction === "runApp") {
-    console.log(`Running app in ${root}...`);
-    await runApp(
-      root,
-      program.frontend,
-      program.framework,
-      program.port,
-      program.externalPort,
-    );
-  }
-}
-
-const update = checkForUpdate(packageJson).catch(() => null);
-
-async function notifyUpdate(): Promise<void> {
-  try {
-    const res = await update;
-    if (res?.latest) {
-      const updateMessage =
-        packageManager === "yarn"
-          ? "yarn global add create-llama@latest"
-          : packageManager === "pnpm"
-            ? "pnpm add -g create-llama@latest"
-            : "npm i -g create-llama@latest";
-
-      console.log(
-        yellow(bold("A new version of `create-llama` is available!")) +
-          "\n" +
-          "You can update by running: " +
-          cyan(updateMessage) +
-          "\n",
-      );
-    }
-  } catch {
-    // ignore error
-  }
-}
-
-run()
-  .then(notifyUpdate)
-  .catch(async (reason) => {
-    console.log();
-    console.log("Aborting installation.");
-    if (reason.command) {
-      console.log(`  ${cyan(reason.command)} has failed.`);
-    } else {
-      console.log(
-        red("Unexpected error. Please report it as a bug:") + "\n",
-        reason,
-      );
-    }
-    console.log();
-
-    await notifyUpdate();
-
-    process.exit(1);
-  });

packages/create-llama/package.json

@@ -1,61 +0,0 @@
-{
-  "name": "create-llama",
-  "version": "0.0.27",
-  "keywords": [
-    "rag",
-    "llamaindex",
-    "next.js"
-  ],
-  "description": "Create LlamaIndex-powered apps with one command",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/run-llama/LlamaIndexTS",
-    "directory": "packages/create-llama"
-  },
-  "license": "MIT",
-  "bin": {
-    "create-llama": "./dist/index.js"
-  },
-  "files": [
-    "dist"
-  ],
-  "scripts": {
-    "clean": "rimraf --glob ./dist ./templates/**/__pycache__ ./templates/**/node_modules ./templates/**/poetry.lock",
-    "dev": "ncc build ./index.ts -w -o dist/",
-    "build": "npm run clean && ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register",
-    "lint": "eslint . --ignore-pattern dist",
-    "e2e": "playwright test",
-    "prepublishOnly": "cd ../../ && pnpm run build:release"
-  },
-  "devDependencies": {
-    "@playwright/test": "^1.41.1",
-    "@types/async-retry": "1.4.2",
-    "@types/ci-info": "2.0.0",
-    "@types/cross-spawn": "6.0.0",
-    "@types/node": "^20.11.7",
-    "@types/prompts": "2.0.1",
-    "@types/tar": "6.1.5",
-    "@types/validate-npm-package-name": "3.0.0",
-    "@vercel/ncc": "0.38.1",
-    "async-retry": "1.3.1",
-    "async-sema": "3.0.1",
-    "ci-info": "github:watson/ci-info#f43f6a1cefff47fb361c88cf4b943fdbcaafe540",
-    "commander": "2.20.0",
-    "conf": "10.2.0",
-    "cross-spawn": "7.0.3",
-    "fast-glob": "3.3.1",
-    "got": "10.7.0",
-    "picocolors": "1.0.0",
-    "prompts": "2.1.0",
-    "rimraf": "^5.0.5",
-    "smol-toml": "^1.1.4",
-    "tar": "6.1.15",
-    "terminal-link": "^3.0.0",
-    "update-check": "1.5.4",
-    "validate-npm-package-name": "3.0.0",
-    "wait-port": "^1.1.0"
-  },
-  "engines": {
-    "node": ">=16.14.0"
-  }
-}

packages/create-llama/playwright.config.ts

@@ -1,21 +0,0 @@
-/* eslint-disable turbo/no-undeclared-env-vars */
-import { defineConfig, devices } from "@playwright/test";
-
-export default defineConfig({
-  testDir: "./e2e",
-  fullyParallel: true,
-  forbidOnly: !!process.env.CI,
-  retries: process.env.CI ? 2 : 0,
-  workers: process.env.CI ? 1 : undefined,
-  timeout: 1000 * 60 * 5,
-  reporter: "html",
-  use: {
-    trace: "on-first-retry",
-  },
-  projects: [
-    {
-      name: "chromium",
-      use: { ...devices["Desktop Chrome"] },
-    },
-  ],
-});

packages/create-llama/questions.ts

@@ -1,747 +0,0 @@
-import { execSync } from "child_process";
-import ciInfo from "ci-info";
-import fs from "fs";
-import path from "path";
-import { blue, green, red } from "picocolors";
-import prompts from "prompts";
-import { InstallAppArgs } from "./create-app";
-import {
-  FileSourceConfig,
-  TemplateDataSourceType,
-  TemplateFramework,
-} from "./helpers";
-import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant";
-import { templatesDir } from "./helpers/dir";
-import { getAvailableLlamapackOptions } from "./helpers/llama-pack";
-import { getRepoRootFolders } from "./helpers/repo";
-import { supportedTools, toolsRequireConfig } from "./helpers/tools";
-
-export type QuestionArgs = Omit<
-  InstallAppArgs,
-  "appPath" | "packageManager"
-> & { files?: string; llamaParse?: boolean };
-const supportedContextFileTypes = [
-  ".pdf",
-  ".doc",
-  ".docx",
-  ".xls",
-  ".xlsx",
-  ".csv",
-];
-const MACOS_FILE_SELECTION_SCRIPT = `
-osascript -l JavaScript -e '
-  a = Application.currentApplication();
-  a.includeStandardAdditions = true;
-  a.chooseFile({ withPrompt: "Please select a file to process:" }).toString()
-'`;
-const MACOS_FOLDER_SELECTION_SCRIPT = `
-osascript -l JavaScript -e '
-  a = Application.currentApplication();
-  a.includeStandardAdditions = true;
-  a.chooseFolder({ withPrompt: "Please select a folder to process:" }).toString()
-'`;
-const WINDOWS_FILE_SELECTION_SCRIPT = `
-Add-Type -AssemblyName System.Windows.Forms
-$openFileDialog = New-Object System.Windows.Forms.OpenFileDialog
-$openFileDialog.InitialDirectory = [Environment]::GetFolderPath('Desktop')
-$result = $openFileDialog.ShowDialog()
-if ($result -eq 'OK') {
-  $openFileDialog.FileName
-}
-`;
-const WINDOWS_FOLDER_SELECTION_SCRIPT = `
-Add-Type -AssemblyName System.windows.forms
-$folderBrowser = New-Object System.Windows.Forms.FolderBrowserDialog
-$dialogResult = $folderBrowser.ShowDialog()
-if ($dialogResult -eq [System.Windows.Forms.DialogResult]::OK)
-{
-    $folderBrowser.SelectedPath
-}
-`;
-
-const defaults: QuestionArgs = {
-  template: "streaming",
-  framework: "nextjs",
-  engine: "simple",
-  ui: "html",
-  eslint: true,
-  frontend: false,
-  openAiKey: "",
-  llamaCloudKey: "",
-  model: "gpt-3.5-turbo",
-  embeddingModel: "text-embedding-ada-002",
-  communityProjectPath: "",
-  llamapack: "",
-  postInstallAction: "dependencies",
-  dataSource: {
-    type: "none",
-    config: {},
-  },
-  tools: [],
-};
-
-const handlers = {
-  onCancel: () => {
-    console.error("Exiting.");
-    process.exit(1);
-  },
-};
-
-const getVectorDbChoices = (framework: TemplateFramework) => {
-  const choices = [
-    {
-      title: "No, just store the data in the file system",
-      value: "none",
-    },
-    { title: "MongoDB", value: "mongo" },
-    { title: "PostgreSQL", value: "pg" },
-    { title: "Pinecone", value: "pinecone" },
-  ];
-
-  const vectordbLang = framework === "fastapi" ? "python" : "typescript";
-  const compPath = path.join(templatesDir, "components");
-  const vectordbPath = path.join(compPath, "vectordbs", vectordbLang);
-
-  const availableChoices = fs
-    .readdirSync(vectordbPath)
-    .filter((file) => fs.statSync(path.join(vectordbPath, file)).isDirectory());
-
-  const displayedChoices = choices.filter((choice) =>
-    availableChoices.includes(choice.value),
-  );
-
-  return displayedChoices;
-};
-
-const getDataSourceChoices = (framework: TemplateFramework) => {
-  const choices = [
-    {
-      title: "No data, just a simple chat",
-      value: "simple",
-    },
-    { title: "Use an example PDF", value: "exampleFile" },
-  ];
-  if (process.platform === "win32" || process.platform === "darwin") {
-    choices.push({
-      title: `Use a local file (${supportedContextFileTypes.join(", ")})`,
-      value: "localFile",
-    });
-    choices.push({
-      title: `Use a local folder`,
-      value: "localFolder",
-    });
-  }
-  if (framework === "fastapi") {
-    choices.push({
-      title: "Use website content (requires Chrome)",
-      value: "web",
-    });
-  }
-  return choices;
-};
-
-const selectLocalContextData = async (type: TemplateDataSourceType) => {
-  try {
-    let selectedPath: string = "";
-    let execScript: string;
-    let execOpts: any = {};
-    switch (process.platform) {
-      case "win32": // Windows
-        execScript =
-          type === "file"
-            ? WINDOWS_FILE_SELECTION_SCRIPT
-            : WINDOWS_FOLDER_SELECTION_SCRIPT;
-        execOpts = { shell: "powershell.exe" };
-        break;
-      case "darwin": // MacOS
-        execScript =
-          type === "file"
-            ? MACOS_FILE_SELECTION_SCRIPT
-            : MACOS_FOLDER_SELECTION_SCRIPT;
-        break;
-      default: // Unsupported OS
-        console.log(red("Unsupported OS error!"));
-        process.exit(1);
-    }
-    selectedPath = execSync(execScript, execOpts).toString().trim();
-    if (type === "file") {
-      const fileType = path.extname(selectedPath);
-      if (!supportedContextFileTypes.includes(fileType)) {
-        console.log(
-          red(
-            `Please select a supported file type: ${supportedContextFileTypes}`,
-          ),
-        );
-        process.exit(1);
-      }
-    }
-    return selectedPath;
-  } catch (error) {
-    console.log(
-      red(
-        "Got an error when trying to select local context data! Please try again or select another data source option.",
-      ),
-    );
-    process.exit(1);
-  }
-};
-
-export const onPromptState = (state: any) => {
-  if (state.aborted) {
-    // If we don't re-enable the terminal cursor before exiting
-    // the program, the cursor will remain hidden
-    process.stdout.write("\x1B[?25h");
-    process.stdout.write("\n");
-    process.exit(1);
-  }
-};
-
-export const askQuestions = async (
-  program: QuestionArgs,
-  preferences: QuestionArgs,
-) => {
-  const getPrefOrDefault = <K extends keyof QuestionArgs>(
-    field: K,
-  ): QuestionArgs[K] => preferences[field] ?? defaults[field];
-
-  // Ask for next action after installation
-  async function askPostInstallAction() {
-    if (program.postInstallAction === undefined) {
-      if (ciInfo.isCI) {
-        program.postInstallAction = getPrefOrDefault("postInstallAction");
-      } else {
-        const actionChoices = [
-          {
-            title: "Just generate code (~1 sec)",
-            value: "none",
-          },
-          {
-            title: "Start in VSCode (~1 sec)",
-            value: "VSCode",
-          },
-          {
-            title: "Generate code and install dependencies (~2 min)",
-            value: "dependencies",
-          },
-        ];
-
-        const openAiKeyConfigured =
-          program.openAiKey || process.env["OPENAI_API_KEY"];
-        // If using LlamaParse, require LlamaCloud API key
-        const llamaCloudKeyConfigured = (
-          program.dataSource?.config as FileSourceConfig
-        )?.useLlamaParse
-          ? program.llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
-          : true;
-        const hasVectorDb = program.vectorDb && program.vectorDb !== "none";
-        // Can run the app if all tools do not require configuration
-        if (
-          !hasVectorDb &&
-          openAiKeyConfigured &&
-          llamaCloudKeyConfigured &&
-          !toolsRequireConfig(program.tools) &&
-          !program.llamapack
-        ) {
-          actionChoices.push({
-            title:
-              "Generate code, install dependencies, and run the app (~2 min)",
-            value: "runApp",
-          });
-        }
-
-        const { action } = await prompts(
-          {
-            type: "select",
-            name: "action",
-            message: "How would you like to proceed?",
-            choices: actionChoices,
-            initial: 1,
-          },
-          handlers,
-        );
-
-        program.postInstallAction = action;
-      }
-    }
-  }
-
-  if (!program.template) {
-    if (ciInfo.isCI) {
-      program.template = getPrefOrDefault("template");
-    } else {
-      const styledRepo = blue(
-        `https://github.com/${COMMUNITY_OWNER}/${COMMUNITY_REPO}`,
-      );
-      const { template } = await prompts(
-        {
-          type: "select",
-          name: "template",
-          message: "Which template would you like to use?",
-          choices: [
-            { title: "Chat without streaming", value: "simple" },
-            { title: "Chat with streaming", value: "streaming" },
-            {
-              title: `Community template from ${styledRepo}`,
-              value: "community",
-            },
-            {
-              title: "Example using a LlamaPack",
-              value: "llamapack",
-            },
-          ],
-          initial: 1,
-        },
-        handlers,
-      );
-      program.template = template;
-      preferences.template = template;
-    }
-  }
-
-  if (program.template === "community") {
-    const rootFolderNames = await getRepoRootFolders(
-      COMMUNITY_OWNER,
-      COMMUNITY_REPO,
-    );
-    const { communityProjectPath } = await prompts(
-      {
-        type: "select",
-        name: "communityProjectPath",
-        message: "Select community template",
-        choices: rootFolderNames.map((name) => ({
-          title: name,
-          value: name,
-        })),
-        initial: 0,
-      },
-      handlers,
-    );
-    program.communityProjectPath = communityProjectPath;
-    preferences.communityProjectPath = communityProjectPath;
-    return; // early return - no further questions needed for community projects
-  }
-
-  if (program.template === "llamapack") {
-    const availableLlamaPacks = await getAvailableLlamapackOptions();
-    const { llamapack } = await prompts(
-      {
-        type: "select",
-        name: "llamapack",
-        message: "Select LlamaPack",
-        choices: availableLlamaPacks.map((pack) => ({
-          title: pack.name,
-          value: pack.folderPath,
-        })),
-        initial: 0,
-      },
-      handlers,
-    );
-    program.llamapack = llamapack;
-    preferences.llamapack = llamapack;
-    await askPostInstallAction();
-    return; // early return - no further questions needed for llamapack projects
-  }
-
-  if (!program.framework) {
-    if (ciInfo.isCI) {
-      program.framework = getPrefOrDefault("framework");
-    } else {
-      const choices = [
-        { title: "Express", value: "express" },
-        { title: "FastAPI (Python)", value: "fastapi" },
-      ];
-      if (program.template === "streaming") {
-        // allow NextJS only for streaming template
-        choices.unshift({ title: "NextJS", value: "nextjs" });
-      }
-
-      const { framework } = await prompts(
-        {
-          type: "select",
-          name: "framework",
-          message: "Which framework would you like to use?",
-          choices,
-          initial: 0,
-        },
-        handlers,
-      );
-      program.framework = framework;
-      preferences.framework = framework;
-    }
-  }
-
-  if (
-    program.template === "streaming" &&
-    (program.framework === "express" || program.framework === "fastapi")
-  ) {
-    // if a backend-only framework is selected, ask whether we should create a frontend
-    // (only for streaming backends)
-    if (program.frontend === undefined) {
-      if (ciInfo.isCI) {
-        program.frontend = getPrefOrDefault("frontend");
-      } else {
-        const styledNextJS = blue("NextJS");
-        const styledBackend = green(
-          program.framework === "express"
-            ? "Express "
-            : program.framework === "fastapi"
-              ? "FastAPI (Python) "
-              : "",
-        );
-        const { frontend } = await prompts({
-          onState: onPromptState,
-          type: "toggle",
-          name: "frontend",
-          message: `Would you like to generate a ${styledNextJS} frontend for your ${styledBackend}backend?`,
-          initial: getPrefOrDefault("frontend"),
-          active: "Yes",
-          inactive: "No",
-        });
-        program.frontend = Boolean(frontend);
-        preferences.frontend = Boolean(frontend);
-      }
-    }
-  } else {
-    program.frontend = false;
-  }
-
-  if (program.framework === "nextjs" || program.frontend) {
-    if (!program.ui) {
-      if (ciInfo.isCI) {
-        program.ui = getPrefOrDefault("ui");
-      } else {
-        const { ui } = await prompts(
-          {
-            type: "select",
-            name: "ui",
-            message: "Which UI would you like to use?",
-            choices: [
-              { title: "Just HTML", value: "html" },
-              { title: "Shadcn", value: "shadcn" },
-            ],
-            initial: 0,
-          },
-          handlers,
-        );
-        program.ui = ui;
-        preferences.ui = ui;
-      }
-    }
-  }
-
-  if (!program.model) {
-    if (ciInfo.isCI) {
-      program.model = getPrefOrDefault("model");
-    } else {
-      const { model } = await prompts(
-        {
-          type: "select",
-          name: "model",
-          message: "Which model would you like to use?",
-          choices: [
-            { title: "gpt-3.5-turbo", value: "gpt-3.5-turbo-0125" },
-            { title: "gpt-4-turbo-preview", value: "gpt-4-turbo-preview" },
-            { title: "gpt-4", value: "gpt-4" },
-            {
-              title: "gpt-4-vision-preview",
-              value: "gpt-4-vision-preview",
-            },
-          ],
-          initial: 0,
-        },
-        handlers,
-      );
-      program.model = model;
-      preferences.model = model;
-    }
-  }
-
-  if (!program.embeddingModel && program.framework === "fastapi") {
-    if (ciInfo.isCI) {
-      program.embeddingModel = getPrefOrDefault("embeddingModel");
-    } else {
-      const { embeddingModel } = await prompts(
-        {
-          type: "select",
-          name: "embeddingModel",
-          message: "Which embedding model would you like to use?",
-          choices: [
-            {
-              title: "text-embedding-ada-002",
-              value: "text-embedding-ada-002",
-            },
-            {
-              title: "text-embedding-3-small",
-              value: "text-embedding-3-small",
-            },
-            {
-              title: "text-embedding-3-large",
-              value: "text-embedding-3-large",
-            },
-          ],
-          initial: 0,
-        },
-        handlers,
-      );
-      program.embeddingModel = embeddingModel;
-      preferences.embeddingModel = embeddingModel;
-    }
-  }
-
-  if (program.files) {
-    // If user specified files option, then the program should use context engine
-    program.engine == "context";
-    if (!fs.existsSync(program.files)) {
-      console.log("File or folder not found");
-      process.exit(1);
-    } else {
-      program.dataSource = {
-        type: fs.lstatSync(program.files).isDirectory() ? "folder" : "file",
-        config: {
-          path: program.files,
-        },
-      };
-    }
-  }
-
-  if (!program.engine) {
-    if (ciInfo.isCI) {
-      program.engine = getPrefOrDefault("engine");
-    } else {
-      const { dataSource } = await prompts(
-        {
-          type: "select",
-          name: "dataSource",
-          message: "Which data source would you like to use?",
-          choices: getDataSourceChoices(program.framework),
-          initial: 1,
-        },
-        handlers,
-      );
-      // Initialize with default config
-      program.dataSource = getPrefOrDefault("dataSource");
-      if (program.dataSource) {
-        switch (dataSource) {
-          case "simple":
-            program.engine = "simple";
-            program.dataSource = { type: "none", config: {} };
-            break;
-          case "exampleFile":
-            program.engine = "context";
-            // Treat example as a folder data source with no config
-            program.dataSource = { type: "folder", config: {} };
-            break;
-          case "localFile":
-            program.engine = "context";
-            program.dataSource = {
-              type: "file",
-              config: {
-                path: await selectLocalContextData("file"),
-              },
-            };
-            break;
-          case "localFolder":
-            program.engine = "context";
-            program.dataSource = {
-              type: "folder",
-              config: {
-                path: await selectLocalContextData("folder"),
-              },
-            };
-            break;
-          case "web":
-            program.engine = "context";
-            program.dataSource.type = "web";
-            break;
-        }
-      }
-    }
-  } else if (!program.dataSource) {
-    // Handle a case when engine is specified but dataSource is not
-    if (program.engine === "context") {
-      program.dataSource = {
-        type: "folder",
-        config: {},
-      };
-    } else if (program.engine === "simple") {
-      program.dataSource = {
-        type: "none",
-        config: {},
-      };
-    }
-  }
-
-  if (
-    (program.dataSource?.type === "file" ||
-      program.dataSource?.type === "folder") &&
-    program.framework === "fastapi"
-  ) {
-    if (ciInfo.isCI) {
-      program.llamaCloudKey = getPrefOrDefault("llamaCloudKey");
-    } else {
-      const dataSourceConfig = program.dataSource.config as FileSourceConfig;
-      dataSourceConfig.useLlamaParse = program.llamaParse;
-
-      // Is pdf file selected as data source or is it a folder data source
-      const askingLlamaParse =
-        dataSourceConfig.useLlamaParse === undefined &&
-        (program.dataSource.type === "folder"
-          ? true
-          : dataSourceConfig.path &&
-            path.extname(dataSourceConfig.path) === ".pdf");
-
-      // Ask if user wants to use LlamaParse
-      if (askingLlamaParse) {
-        const { useLlamaParse } = await prompts(
-          {
-            type: "toggle",
-            name: "useLlamaParse",
-            message:
-              "Would you like to use LlamaParse (improved parser for RAG - requires API key)?",
-            initial: true,
-            active: "yes",
-            inactive: "no",
-          },
-          handlers,
-        );
-        dataSourceConfig.useLlamaParse = useLlamaParse;
-        program.dataSource.config = dataSourceConfig;
-      }
-
-      // Ask for LlamaCloud API key
-      if (
-        dataSourceConfig.useLlamaParse &&
-        program.llamaCloudKey === undefined
-      ) {
-        const { llamaCloudKey } = await prompts(
-          {
-            type: "text",
-            name: "llamaCloudKey",
-            message:
-              "Please provide your LlamaIndex Cloud API key (leave blank to skip):",
-          },
-          handlers,
-        );
-        program.llamaCloudKey = llamaCloudKey;
-      }
-    }
-  }
-
-  if (program.dataSource?.type === "web" && program.framework === "fastapi") {
-    let { baseUrl } = await prompts(
-      {
-        type: "text",
-        name: "baseUrl",
-        message: "Please provide base URL of the website:",
-        initial: "https://www.llamaindex.ai",
-      },
-      handlers,
-    );
-    try {
-      if (!baseUrl.includes("://")) {
-        baseUrl = `https://${baseUrl}`;
-      }
-      const checkUrl = new URL(baseUrl);
-      if (checkUrl.protocol !== "https:" && checkUrl.protocol !== "http:") {
-        throw new Error("Invalid protocol");
-      }
-    } catch (error) {
-      console.log(
-        red(
-          "Invalid URL provided! Please provide a valid URL (e.g. https://www.llamaindex.ai)",
-        ),
-      );
-      process.exit(1);
-    }
-    program.dataSource.config = {
-      baseUrl: baseUrl,
-      depth: 1,
-    };
-  }
-
-  if (program.engine !== "simple" && !program.vectorDb) {
-    if (ciInfo.isCI) {
-      program.vectorDb = getPrefOrDefault("vectorDb");
-    } else {
-      const { vectorDb } = await prompts(
-        {
-          type: "select",
-          name: "vectorDb",
-          message: "Would you like to use a vector database?",
-          choices: getVectorDbChoices(program.framework),
-          initial: 0,
-        },
-        handlers,
-      );
-      program.vectorDb = vectorDb;
-      preferences.vectorDb = vectorDb;
-    }
-  }
-
-  if (
-    !program.tools &&
-    program.framework === "fastapi" &&
-    program.engine === "context"
-  ) {
-    if (ciInfo.isCI) {
-      program.tools = getPrefOrDefault("tools");
-    } else {
-      const toolChoices = supportedTools.map((tool) => ({
-        title: tool.display,
-        value: tool.name,
-      }));
-      const { toolsName } = await prompts({
-        type: "multiselect",
-        name: "toolsName",
-        message:
-          "Would you like to build an agent using tools? If so, select the tools here, otherwise just press enter",
-        choices: toolChoices,
-      });
-      const tools = toolsName?.map((tool: string) =>
-        supportedTools.find((t) => t.name === tool),
-      );
-      program.tools = tools;
-      preferences.tools = tools;
-    }
-  }
-
-  if (!program.openAiKey) {
-    const { key } = await prompts(
-      {
-        type: "text",
-        name: "key",
-        message: "Please provide your OpenAI API key (leave blank to skip):",
-      },
-      handlers,
-    );
-    program.openAiKey = key;
-    preferences.openAiKey = key;
-  }
-
-  if (program.framework !== "fastapi" && program.eslint === undefined) {
-    if (ciInfo.isCI) {
-      program.eslint = getPrefOrDefault("eslint");
-    } else {
-      const styledEslint = blue("ESLint");
-      const { eslint } = await prompts({
-        onState: onPromptState,
-        type: "toggle",
-        name: "eslint",
-        message: `Would you like to use ${styledEslint}?`,
-        initial: getPrefOrDefault("eslint"),
-        active: "Yes",
-        inactive: "No",
-      });
-      program.eslint = Boolean(eslint);
-      preferences.eslint = Boolean(eslint);
-    }
-  }
-
-  await askPostInstallAction();
-
-  // TODO: consider using zod to validate the input (doesn't work like this as not every option is required)
-  // templateUISchema.parse(program.ui);
-  // templateEngineSchema.parse(program.engine);
-  // templateFrameworkSchema.parse(program.framework);
-  // templateTypeSchema.parse(program.template);``
-};

packages/create-llama/templates/.gitignore

@@ -1,3 +0,0 @@
-__pycache__
-poetry.lock
-storage

packages/create-llama/templates/README-fullstack.md

@@ -1,18 +0,0 @@
-This is a [LlamaIndex](https://www.llamaindex.ai/) project bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
-
-## Getting Started
-
-First, startup the backend as described in the [backend README](./backend/README.md).
-
-Second, run the development server of the frontend as described in the [frontend README](./frontend/README.md).
-
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
-
-## Learn More
-
-To learn more about LlamaIndex, take a look at the following resources:
-
-- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
-- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
-
-You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

packages/create-llama/templates/components/engines/python/agent/__init__.py

@@ -1,24 +0,0 @@
-from llama_index.core.settings import Settings
-from llama_index.core.agent import AgentRunner
-from llama_index.core.tools.query_engine import QueryEngineTool
-from app.engine.tools import ToolFactory
-from app.engine.index import get_index
-
-
-def get_chat_engine():
-    tools = []
-
-    # Add query tool
-    index = get_index()
-    query_engine = index.as_query_engine(similarity_top_k=3)
-    query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
-    tools.append(query_engine_tool)
-
-    # Add additional tools
-    tools += ToolFactory.from_env()
-
-    return AgentRunner.from_llm(
-        llm=Settings.llm,
-        tools=tools,
-        verbose=True,
-    )

packages/create-llama/templates/components/engines/python/agent/tools.py

@@ -1,33 +0,0 @@
-import json
-import importlib
-
-from llama_index.core.tools.tool_spec.base import BaseToolSpec
-from llama_index.core.tools.function_tool import FunctionTool
-
-
-class ToolFactory:
-
-    @staticmethod
-    def create_tool(tool_name: str, **kwargs) -> list[FunctionTool]:
-        try:
-            tool_package, tool_cls_name = tool_name.split(".")
-            module_name = f"llama_index.tools.{tool_package}"
-            module = importlib.import_module(module_name)
-            tool_class = getattr(module, tool_cls_name)
-            tool_spec: BaseToolSpec = tool_class(**kwargs)
-            return tool_spec.to_tool_list()
-        except (ImportError, AttributeError) as e:
-            raise ValueError(f"Unsupported tool: {tool_name}") from e
-        except TypeError as e:
-            raise ValueError(
-                f"Could not create tool: {tool_name}. With config: {kwargs}"
-            ) from e
-
-    @staticmethod
-    def from_env() -> list[FunctionTool]:
-        tools = []
-        with open("tools_config.json", "r") as f:
-            tool_configs = json.load(f)
-            for name, config in tool_configs.items():
-                tools += ToolFactory.create_tool(name, **config)
-        return tools

packages/create-llama/templates/components/engines/python/chat/__init__.py

@@ -1,7 +0,0 @@
-from app.engine.index import get_index
-
-
-def get_chat_engine():
-    return get_index().as_chat_engine(
-        similarity_top_k=3, chat_mode="condense_plus_context"
-    )

packages/create-llama/templates/components/loaders/python/file/loader.py

@@ -1,7 +0,0 @@
-from llama_index.core.readers import SimpleDirectoryReader
-
-DATA_DIR = "data"  # directory containing the documents
-
-
-def get_documents():
-    return SimpleDirectoryReader(DATA_DIR).load_data()

packages/create-llama/templates/components/loaders/python/llama_parse/loader.py

@@ -1,11 +0,0 @@
-from llama_parse import LlamaParse
-from llama_index.core import SimpleDirectoryReader
-
-DATA_DIR = "data"  # directory containing the documents
-
-
-def get_documents():
-    parser = LlamaParse(result_type="markdown", verbose=True, language="en")
-
-    reader = SimpleDirectoryReader(DATA_DIR, file_extractor={".pdf": parser})
-    return reader.load_data()

packages/create-llama/templates/components/loaders/python/web/loader.py

@@ -1,13 +0,0 @@
-import os
-from llama_index.readers.web import WholeSiteReader
-
-
-def get_documents():
-    # Initialize the scraper with a prefix URL and maximum depth
-    scraper = WholeSiteReader(
-        prefix=os.environ.get("URL_PREFIX"), max_depth=int(os.environ.get("MAX_DEPTH"))
-    )
-    # Start scraping from a base URL
-    documents = scraper.load_data(base_url=os.environ.get("BASE_URL"))
-
-    return documents

packages/create-llama/templates/components/sample-projects/llamapack/README-template.md

@@ -1,16 +0,0 @@
----
-
-## Quickstart
-
-1. Check above instructions for setting up your environment and export required environment variables
-   For example, if you are using bash, you can run the following command to set up OpenAI API key
-
-```bash
-export OPENAI_API_KEY=your_api_key
-```
-
-2. Run the example
-
-```
-poetry run python example.py
-```