chore: remove unused package

Co-authored-by: Alex Yang <himself65@outlook.com>

.changeset/chatty-parents-smell.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat(qdrant): Add Qdrant Vector DB

.changeset/mighty-chicken-trade.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Preview: Add ingestion pipeline (incl. different strategies to handle doc store duplicates)

.changeset/nervous-numbers-swim.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add an option that allows the user to run the generated app

.changeset/popular-grapes-cheat.md

@@ -0,0 +1,16 @@
+---
+"llamaindex": patch
+---
+
+feat: use conditional exports
+
+The benefit of conditional exports is we split the llamaindex into different files. This will improve the tree shake if you are building web apps.
+
+This also requires node16 (see https://nodejs.org/api/packages.html#conditional-exports).
+
+If you are seeing typescript issue `TS2724`('llamaindex' has no exported member named XXX):
+
+1. update `moduleResolution` to `bundler` in `tsconfig.json`, more for the web applications like Next.js, and vite, but still works for ts-node or tsx.
+2. consider the ES module in your project, add `"type": "module"` into `package.json` and update `moduleResolution` to `node16` or `nodenext` in `tsconfig.json`.
+
+We still support both cjs and esm, but you should update `tsconfig.json` to make the typescript happy.

.changeset/three-hounds-count.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat(extractors): add keyword extractor and base extractor

.devcontainer/devcontainer.json

@@ -4,6 +4,6 @@
     "ghcr.io/devcontainers/features/node:1": {},
     "ghcr.io/devcontainers-contrib/features/turborepo-npm:1": {},
     "ghcr.io/devcontainers-contrib/features/typescript:2": {},
-    "ghcr.io/devcontainers-contrib/features/pnpm:2": {}
-  }
+    "ghcr.io/devcontainers-contrib/features/pnpm:2": {},
+  },
 }

.github/workflows/e2e.yml

@@ -21,6 +21,9 @@ jobs:
         node-version: [18, 20]
         python-version: ["3.11"]
         os: [macos-latest, windows-latest]
+    defaults:
+      run:
+        shell: bash
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

.github/workflows/lint_on_push_or_pull.yml

@@ -14,6 +14,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v2
+        with:
+          version: latest
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:

.prettierignore

@@ -2,3 +2,4 @@ apps/docs/i18n
 pnpm-lock.yaml
 lib/
 dist/
+.docusaurus/

README.md

@@ -52,9 +52,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

apps/docs/docs/end_to_end.md

@@ -26,7 +26,7 @@ Create and load a vector index. Persistance to disk in LlamaIndex.TS happens aut
 
 ## [Customized Vector Index](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/vectorIndexCustomize.ts)
 
-Create a vector index and query it, while also configuring the the `LLM`, the `ServiceContext`, and the `similarity_top_k`.
+Create a vector index and query it, while also configuring the `LLM`, the `ServiceContext`, and the `similarity_top_k`.
 
 ## [OpenAI LLM](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/openai.ts)
 

apps/docs/tsconfig.json

@@ -6,6 +6,6 @@
     "composite": true,
     "incremental": true,
     "outDir": "./lib",
-    "tsBuildInfoFile": "./lib/.tsbuildinfo"
-  }
+    "tsBuildInfoFile": "./lib/.tsbuildinfo",
+  },
 }

examples/chatEngine.ts

@@ -10,7 +10,7 @@ import {
   VectorStoreIndex,
 } from "llamaindex";
 
-import essay from "./essay";
+import essay from "./essay.js";
 
 async function main() {
   const document = new Document({ text: essay });

examples/chatHistory.ts

@@ -1,6 +1,4 @@
 import { stdin as input, stdout as output } from "node:process";
-// readline/promises is still experimental so not in @types/node yet
-// @ts-ignore
 import readline from "node:readline/promises";
 
 import { OpenAI, SimpleChatEngine, SummaryChatHistory } from "llamaindex";

examples/chromadb/preFilters.ts

@@ -0,0 +1,57 @@
+import {
+  ChromaVectorStore,
+  Document,
+  VectorStoreIndex,
+  storageContextFromDefaults,
+} from "llamaindex";
+
+const collectionName = "dog_colors";
+
+async function main() {
+  try {
+    const docs = [
+      new Document({
+        text: "The dog is brown",
+        metadata: {
+          dogId: "1",
+        },
+      }),
+      new Document({
+        text: "The dog is red",
+        metadata: {
+          dogId: "2",
+        },
+      }),
+    ];
+
+    console.log("Creating ChromaDB vector store");
+    const chromaVS = new ChromaVectorStore({ collectionName });
+    const ctx = await storageContextFromDefaults({ vectorStore: chromaVS });
+
+    console.log("Embedding documents and adding to index");
+    const index = await VectorStoreIndex.fromDocuments(docs, {
+      storageContext: ctx,
+    });
+
+    console.log("Querying index");
+    const queryEngine = index.asQueryEngine({
+      preFilters: {
+        filters: [
+          {
+            key: "dogId",
+            value: "2",
+            filterType: "ExactMatch",
+          },
+        ],
+      },
+    });
+    const response = await queryEngine.query({
+      query: "What is the color of the dog?",
+    });
+    console.log(response.toString());
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+main();

examples/extractors/keywordExtractor.ts

@@ -0,0 +1,24 @@
+import {
+  Document,
+  KeywordExtractor,
+  OpenAI,
+  SimpleNodeParser,
+} from "llamaindex";
+
+(async () => {
+  const openaiLLM = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0 });
+
+  const nodeParser = new SimpleNodeParser();
+
+  const nodes = nodeParser.getNodesFromDocuments([
+    new Document({ text: "banana apple orange pear peach watermelon" }),
+  ]);
+
+  console.log(nodes);
+
+  const keywordExtractor = new KeywordExtractor(openaiLLM, 5);
+
+  const nodesWithKeywordMetadata = await keywordExtractor.processNodes(nodes);
+
+  process.stdout.write(JSON.stringify(nodesWithKeywordMetadata, null, 2));
+})();

examples/extractors/questionsAnsweredExtractor.ts

@@ -0,0 +1,31 @@
+import {
+  Document,
+  OpenAI,
+  QuestionsAnsweredExtractor,
+  SimpleNodeParser,
+} from "llamaindex";
+
+(async () => {
+  const openaiLLM = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0 });
+
+  const nodeParser = new SimpleNodeParser();
+
+  const nodes = nodeParser.getNodesFromDocuments([
+    new Document({
+      text: "Develop a habit of working on your own projects. Don't let work mean something other people tell you to do. If you do manage to do great work one day, it will probably be on a project of your own. It may be within some bigger project, but you'll be driving your part of it.",
+    }),
+    new Document({
+      text: "The best way to get a good idea is to get a lot of ideas. The best way to get a lot of ideas is to get a lot of bad ideas. The best way to get a lot of bad ideas is to get a lot of ideas.",
+    }),
+  ]);
+
+  const questionsAnsweredExtractor = new QuestionsAnsweredExtractor(
+    openaiLLM,
+    5,
+  );
+
+  const nodesWithQuestionsMetadata =
+    await questionsAnsweredExtractor.processNodes(nodes);
+
+  process.stdout.write(JSON.stringify(nodesWithQuestionsMetadata, null, 2));
+})();

examples/extractors/summaryExtractor.ts

@@ -0,0 +1,24 @@
+import {
+  Document,
+  OpenAI,
+  SimpleNodeParser,
+  SummaryExtractor,
+} from "llamaindex";
+
+(async () => {
+  const openaiLLM = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0 });
+
+  const nodeParser = new SimpleNodeParser();
+
+  const nodes = nodeParser.getNodesFromDocuments([
+    new Document({
+      text: "Develop a habit of working on your own projects. Don't let work mean something other people tell you to do. If you do manage to do great work one day, it will probably be on a project of your own. It may be within some bigger project, but you'll be driving your part of it.",
+    }),
+  ]);
+
+  const summaryExtractor = new SummaryExtractor(openaiLLM);
+
+  const nodesWithSummaryMetadata = await summaryExtractor.processNodes(nodes);
+
+  process.stdout.write(JSON.stringify(nodesWithSummaryMetadata, null, 2));
+})();

examples/extractors/titleExtractor.ts

@@ -0,0 +1,19 @@
+import { Document, OpenAI, SimpleNodeParser, TitleExtractor } from "llamaindex";
+
+(async () => {
+  const openaiLLM = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0 });
+
+  const nodeParser = new SimpleNodeParser();
+
+  const nodes = nodeParser.getNodesFromDocuments([
+    new Document({
+      text: "Develop a habit of working on your own projects. Don't let work mean something other people tell you to do. If you do manage to do great work one day, it will probably be on a project of your own. It may be within some bigger project, but you'll be driving your part of it.",
+    }),
+  ]);
+
+  const titleExtractor = new TitleExtractor(openaiLLM, 1);
+
+  const nodesWithTitledMetadata = await titleExtractor.processNodes(nodes);
+
+  process.stdout.write(JSON.stringify(nodesWithTitledMetadata, null, 2));
+})();

examples/keywordIndex.ts

@@ -3,7 +3,7 @@ import {
   KeywordTableIndex,
   KeywordTableRetrieverMode,
 } from "llamaindex";
-import essay from "./essay";
+import essay from "./essay.js";
 
 async function main() {
   const document = new Document({ text: essay, id_: "essay" });

examples/pipeline/ingestion.ts

@@ -0,0 +1,36 @@
+import fs from "node:fs/promises";
+
+import {
+  Document,
+  IngestionPipeline,
+  MetadataMode,
+  OpenAIEmbedding,
+  SimpleNodeParser,
+} from "llamaindex";
+
+async function main() {
+  // Load essay from abramov.txt in Node
+  const path = "node_modules/llamaindex/examples/abramov.txt";
+
+  const essay = await fs.readFile(path, "utf-8");
+
+  // Create Document object with essay
+  const document = new Document({ text: essay, id_: path });
+  const pipeline = new IngestionPipeline({
+    transformations: [
+      new SimpleNodeParser({ chunkSize: 1024, chunkOverlap: 20 }),
+      // new TitleExtractor(llm),
+      new OpenAIEmbedding(),
+    ],
+  });
+
+  // run the pipeline
+  const nodes = await pipeline.run({ documents: [document] });
+
+  // print out the result of the pipeline run
+  for (const node of nodes) {
+    console.log(node.getContent(MetadataMode.NONE));
+  }
+}
+
+main().catch(console.error);

examples/sentenceWindow.ts

@@ -6,7 +6,7 @@ import {
   VectorStoreIndex,
   serviceContextFromDefaults,
 } from "llamaindex";
-import essay from "./essay";
+import essay from "./essay.js";
 
 async function main() {
   const document = new Document({ text: essay, id_: "essay" });

examples/storageContext.ts

@@ -3,7 +3,7 @@ import {
   storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
-import essay from "./essay";
+import essay from "./essay.js";
 
 async function main() {
   // Create Document object with essay

examples/subquestion.ts

@@ -1,6 +1,6 @@
 import { Document, SubQuestionQueryEngine, VectorStoreIndex } from "llamaindex";
 
-import essay from "./essay";
+import essay from "./essay.js";
 
 (async () => {
   const document = new Document({ text: essay, id_: essay });

examples/summaryIndex.ts

@@ -5,7 +5,7 @@ import {
   SummaryRetrieverMode,
   serviceContextFromDefaults,
 } from "llamaindex";
-import essay from "./essay";
+import essay from "./essay.js";
 
 async function main() {
   const serviceContext = serviceContextFromDefaults({

examples/tsconfig.json

@@ -1,12 +1,10 @@
 {
   "extends": "../tsconfig.json",
-  "compilerOptions": {
-    "target": "es2016",
-    "module": "commonjs",
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "strict": true,
-    "skipLibCheck": true
+  "ts-node": {
+    "files": true,
+    "compilerOptions": {
+      "module": "commonjs",
+    },
   },
-  "include": ["./**/*.ts"]
+  "include": ["./**/*.ts"],
 }

examples/vectorIndexCustomize.ts

@@ -6,7 +6,7 @@ import {
   SimilarityPostprocessor,
   VectorStoreIndex,
 } from "llamaindex";
-import essay from "./essay";
+import essay from "./essay.js";
 
 // Customize retrieval and query args
 async function main() {

package.json

@@ -1,7 +1,9 @@
 {
+  "name": "@llamaindex/monorepo",
   "private": true,
   "scripts": {
     "build": "turbo run build",
+    "build:release": "turbo run build lint test --filter=\"!docs\"",
     "dev": "turbo run dev",
     "format": "prettier --ignore-unknown --cache --check .",
     "format:write": "prettier --ignore-unknown --write .",
@@ -9,8 +11,9 @@
     "prepare": "husky install",
     "test": "turbo run test",
     "type-check": "tsc -b --diagnostics",
-    "new-version": "turbo run build lint test --filter=\"!docs\" && changeset version",
-    "new-snapshot": "turbo run build lint test --filter=\"!docs\" && changeset version --snapshot"
+    "release": "pnpm run build:release && changeset publish",
+    "new-version": "pnpm run build:release && changeset version",
+    "new-snapshot": "pnpm run build:release && changeset version --snapshot"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",
@@ -21,7 +24,7 @@
     "husky": "^8.0.3",
     "jest": "^29.7.0",
     "lint-staged": "^15.2.0",
-    "prettier": "^3.1.1",
+    "prettier": "^3.2.4",
     "prettier-plugin-organize-imports": "^3.2.4",
     "ts-jest": "^29.1.1",
     "turbo": "^1.11.2",

packages/core/package.json

@@ -8,6 +8,7 @@
     "@mistralai/mistralai": "^0.0.7",
     "@notionhq/client": "^2.2.14",
     "@pinecone-database/pinecone": "^1.1.2",
+    "@qdrant/js-client-rest": "^1.7.0",
     "@xenova/transformers": "^2.10.0",
     "assemblyai": "^4.0.0",
     "chromadb": "^1.7.3",
@@ -30,13 +31,14 @@
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
+    "@aws-crypto/sha256-js": "^5.2.0",
     "@types/jest": "^29.5.11",
     "@types/lodash": "^4.14.202",
     "@types/node": "^18.19.6",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.10.9",
-    "bunchee": "^4.3.3",
-    "node-stdlib-browser": "^1.2.0",
+    "bunchee": "^4.4.1",
+    "madge": "^6.1.0",
     "typescript": "^5.3.3"
   },
   "engines": {
@@ -47,10 +49,147 @@
   "exports": {
     ".": {
       "types": "./dist/index.d.mts",
+      "edge-light": "./dist/index.edge-light.mjs",
       "import": "./dist/index.mjs",
       "require": "./dist/index.js"
     },
-    "./examples/*": "./examples/*"
+    "./env": {
+      "types": "./dist/env.d.mts",
+      "edge-light": "./dist/env.edge-light.mjs",
+      "import": "./dist/env.mjs",
+      "require": "./dist/env.js"
+    },
+    "./storage/FileSystem": {
+      "types": "./dist/storage/FileSystem.d.mts",
+      "edge-light": "./dist/storage/FileSystem.edge-light.mjs",
+      "import": "./dist/storage/FileSystem.mjs",
+      "require": "./dist/storage/FileSystem.js"
+    },
+    "./ChatHistory": {
+      "types": "./dist/ChatHistory.d.mts",
+      "import": "./dist/ChatHistory.mjs",
+      "require": "./dist/ChatHistory.js"
+    },
+    "./constants": {
+      "types": "./dist/constants.d.mts",
+      "import": "./dist/constants.mjs",
+      "require": "./dist/constants.js"
+    },
+    "./GlobalsHelper": {
+      "types": "./dist/GlobalsHelper.d.mts",
+      "import": "./dist/GlobalsHelper.mjs",
+      "require": "./dist/GlobalsHelper.js"
+    },
+    "./Node": {
+      "types": "./dist/Node.d.mts",
+      "import": "./dist/Node.mjs",
+      "require": "./dist/Node.js"
+    },
+    "./OutputParser": {
+      "types": "./dist/OutputParser.d.mts",
+      "import": "./dist/OutputParser.mjs",
+      "require": "./dist/OutputParser.js"
+    },
+    "./Prompt": {
+      "types": "./dist/Prompt.d.mts",
+      "import": "./dist/Prompt.mjs",
+      "require": "./dist/Prompt.js"
+    },
+    "./PromptHelper": {
+      "types": "./dist/PromptHelper.d.mts",
+      "import": "./dist/PromptHelper.mjs",
+      "require": "./dist/PromptHelper.js"
+    },
+    "./QueryEngine": {
+      "types": "./dist/QueryEngine.d.mts",
+      "import": "./dist/QueryEngine.mjs",
+      "require": "./dist/QueryEngine.js"
+    },
+    "./QuestionGenerator": {
+      "types": "./dist/QuestionGenerator.d.mts",
+      "import": "./dist/QuestionGenerator.mjs",
+      "require": "./dist/QuestionGenerator.js"
+    },
+    "./Response": {
+      "types": "./dist/Response.d.mts",
+      "import": "./dist/Response.mjs",
+      "require": "./dist/Response.js"
+    },
+    "./Retriever": {
+      "types": "./dist/Retriever.d.mts",
+      "import": "./dist/Retriever.mjs",
+      "require": "./dist/Retriever.js"
+    },
+    "./ServiceContext": {
+      "types": "./dist/ServiceContext.d.mts",
+      "import": "./dist/ServiceContext.mjs",
+      "require": "./dist/ServiceContext.js"
+    },
+    "./TextSplitter": {
+      "types": "./dist/TextSplitter.d.mts",
+      "import": "./dist/TextSplitter.mjs",
+      "require": "./dist/TextSplitter.js"
+    },
+    "./Tool": {
+      "types": "./dist/Tool.d.mts",
+      "import": "./dist/Tool.mjs",
+      "require": "./dist/Tool.js"
+    },
+    "./readers/AssemblyAI": {
+      "types": "./dist/readers/AssemblyAI.d.mts",
+      "import": "./dist/readers/AssemblyAI.mjs",
+      "require": "./dist/readers/AssemblyAI.js"
+    },
+    "./readers/base": {
+      "types": "./dist/readers/base.d.mts",
+      "import": "./dist/readers/base.mjs",
+      "require": "./dist/readers/base.js"
+    },
+    "./readers/CSVReader": {
+      "types": "./dist/readers/CSVReader.d.mts",
+      "import": "./dist/readers/CSVReader.mjs",
+      "require": "./dist/readers/CSVReader.js"
+    },
+    "./readers/DocxReader": {
+      "types": "./dist/readers/DocxReader.d.mts",
+      "import": "./dist/readers/DocxReader.mjs",
+      "require": "./dist/readers/DocxReader.js"
+    },
+    "./readers/HTMLReader": {
+      "types": "./dist/readers/HTMLReader.d.mts",
+      "import": "./dist/readers/HTMLReader.mjs",
+      "require": "./dist/readers/HTMLReader.js"
+    },
+    "./readers/ImageReader": {
+      "types": "./dist/readers/ImageReader.d.mts",
+      "import": "./dist/readers/ImageReader.mjs",
+      "require": "./dist/readers/ImageReader.js"
+    },
+    "./readers/MarkdownReader": {
+      "types": "./dist/readers/MarkdownReader.d.mts",
+      "import": "./dist/readers/MarkdownReader.mjs",
+      "require": "./dist/readers/MarkdownReader.js"
+    },
+    "./readers/NotionReader": {
+      "types": "./dist/readers/NotionReader.d.mts",
+      "import": "./dist/readers/NotionReader.mjs",
+      "require": "./dist/readers/NotionReader.js"
+    },
+    "./readers/PDFReader": {
+      "types": "./dist/readers/PDFReader.d.mts",
+      "import": "./dist/readers/PDFReader.mjs",
+      "require": "./dist/readers/PDFReader.js"
+    },
+    "./readers/SimpleDirectoryReader": {
+      "types": "./dist/readers/SimpleDirectoryReader.d.mts",
+      "import": "./dist/readers/SimpleDirectoryReader.mjs",
+      "require": "./dist/readers/SimpleDirectoryReader.js"
+    },
+    "./readers/SimpleMongoReader": {
+      "types": "./dist/readers/SimpleMongoReader.d.mts",
+      "import": "./dist/readers/SimpleMongoReader.mjs",
+      "require": "./dist/readers/SimpleMongoReader.js"
+    }
   },
   "files": [
     "dist",
@@ -68,6 +207,7 @@
     "lint": "eslint .",
     "test": "jest",
     "build": "bunchee",
-    "dev": "bunchee -w"
+    "dev": "bunchee -w",
+    "circular-check": "madge --circular ./src/*.ts"
   }
 }

packages/core/src/GlobalsHelper.ts

@@ -1,7 +1,7 @@
 import { encodingForModel } from "js-tiktoken";
 
-import { randomUUID } from "node:crypto";
 import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
+import { randomUUID } from "./env";
 
 export enum Tokenizers {
   CL100K_BASE = "cl100k_base",

packages/core/src/Node.ts

@@ -1,6 +1,6 @@
 import _ from "lodash";
-import { createHash, randomUUID } from "node:crypto";
 import path from "node:path";
+import { createSHA256, randomUUID } from "./env";
 
 export enum NodeRelationship {
   SOURCE = "SOURCE",
@@ -168,6 +168,8 @@ export abstract class BaseNode<T extends Metadata = Metadata> {
  */
 export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
   text: string = "";
+  textTemplate: string = "";
+
   startCharIdx?: number;
   endCharIdx?: number;
   // textTemplate: NOTE write your own formatter if needed
@@ -181,7 +183,7 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
     if (new.target === TextNode) {
       // Don't generate the hash repeatedly so only do it if this is
       // constructing the derived class
-      this.hash = this.generateHash();
+      this.hash = init?.hash ?? this.generateHash();
     }
   }
 
@@ -191,13 +193,13 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
    * @returns
    */
   generateHash() {
-    const hashFunction = createHash("sha256");
+    const hashFunction = createSHA256();
     hashFunction.update(`type=${this.getType()}`);
     hashFunction.update(
       `startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`,
     );
     hashFunction.update(this.getContent(MetadataMode.ALL));
-    return hashFunction.digest("base64");
+    return hashFunction.digest();
   }
 
   getType(): ObjectType {
@@ -232,7 +234,6 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
 
   setContent(value: string) {
     this.text = value;
-
     this.hash = this.generateHash();
   }
 
@@ -253,7 +254,7 @@ export class IndexNode<T extends Metadata = Metadata> extends TextNode<T> {
     Object.assign(this, init);
 
     if (new.target === IndexNode) {
-      this.hash = this.generateHash();
+      this.hash = init?.hash ?? this.generateHash();
     }
   }
 
@@ -271,7 +272,7 @@ export class Document<T extends Metadata = Metadata> extends TextNode<T> {
     Object.assign(this, init);
 
     if (new.target === Document) {
-      this.hash = this.generateHash();
+      this.hash = init?.hash ?? this.generateHash();
     }
   }
 
@@ -332,7 +333,7 @@ export class ImageDocument<T extends Metadata = Metadata> extends ImageNode<T> {
     super(init);
 
     if (new.target === ImageDocument) {
-      this.hash = this.generateHash();
+      this.hash = init?.hash ?? this.generateHash();
     }
   }
 

packages/core/src/QueryEngine.ts

@@ -1,4 +1,3 @@
-import { randomUUID } from "node:crypto";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   BaseQuestionGenerator,
@@ -10,6 +9,7 @@ import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { QueryEngineTool, ToolMetadata } from "./Tool";
 import { Event } from "./callbacks/CallbackManager";
+import { randomUUID } from "./env";
 import { BaseNodePostprocessor } from "./postprocessors";
 import {
   BaseSynthesizer,

packages/core/src/TextSplitter.ts

@@ -1,4 +1,4 @@
-import { EOL } from "node:os";
+import { EOL } from "./env";
 // GitHub translated
 import { globalsHelper } from "./GlobalsHelper";
 import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants";

packages/core/src/embeddings/types.ts

@@ -1,3 +1,5 @@
+import { BaseNode, MetadataMode } from "../Node";
+import { TransformComponent } from "../ingestion";
 import { similarity } from "./utils";
 
 /**
@@ -10,7 +12,7 @@ export enum SimilarityType {
   EUCLIDEAN = "euclidean",
 }
 
-export abstract class BaseEmbedding {
+export abstract class BaseEmbedding implements TransformComponent {
   similarity(
     embedding1: number[],
     embedding2: number[],
@@ -21,4 +23,13 @@ export abstract class BaseEmbedding {
 
   abstract getTextEmbedding(text: string): Promise<number[]>;
   abstract getQueryEmbedding(query: string): Promise<number[]>;
+
+  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
+    for (const node of nodes) {
+      node.embedding = await this.getTextEmbedding(
+        node.getContent(MetadataMode.EMBED),
+      );
+    }
+    return nodes;
+  }
 }

packages/core/src/embeddings/utils.ts

@@ -1,7 +1,8 @@
 import _ from "lodash";
 import { ImageType } from "../Node";
 import { DEFAULT_SIMILARITY_TOP_K } from "../constants";
-import { DEFAULT_FS, VectorStoreQueryMode } from "../storage";
+import { defaultFS } from "../env";
+import { VectorStoreQueryMode } from "../storage";
 import { SimilarityType } from "./types";
 
 /**
@@ -244,8 +245,7 @@ export async function imageToDataUrl(input: ImageType): Promise<string> {
     _.isString(input)
   ) {
     // string or file URL
-    const fs = DEFAULT_FS;
-    const dataBuffer = await fs.readFile(
+    const dataBuffer = await defaultFS.readFile(
       input instanceof URL ? input.pathname : input,
     );
     input = new Blob([dataBuffer]);

packages/core/src/engines/chat/ContextChatEngine.ts

@@ -1,9 +1,9 @@
-import { randomUUID } from "node:crypto";
 import { ChatHistory, getHistory } from "../../ChatHistory";
 import { ContextSystemPrompt } from "../../Prompt";
 import { Response } from "../../Response";
 import { BaseRetriever } from "../../Retriever";
 import { Event } from "../../callbacks/CallbackManager";
+import { randomUUID } from "../../env";
 import { ChatMessage, ChatResponseChunk, LLM, OpenAI } from "../../llm";
 import { MessageContent } from "../../llm/types";
 import { extractText, streamConverter, streamReducer } from "../../llm/utils";

packages/core/src/engines/chat/DefaultContextGenerator.ts

@@ -1,8 +1,8 @@
-import { randomUUID } from "node:crypto";
 import { NodeWithScore, TextNode } from "../../Node";
 import { ContextSystemPrompt, defaultContextSystemPrompt } from "../../Prompt";
 import { BaseRetriever } from "../../Retriever";
 import { Event } from "../../callbacks/CallbackManager";
+import { randomUUID } from "../../env";
 import { BaseNodePostprocessor } from "../../postprocessors";
 import { Context, ContextGenerator } from "./types";
 

packages/core/src/env/index.edge-light.ts

@@ -0,0 +1,37 @@
+import { Sha256 } from "@aws-crypto/sha256-js";
+import { CompleteFileSystem, InMemoryFileSystem } from "../storage";
+
+export interface SHA256 {
+  update(data: string | Uint8Array): void;
+  // to base64
+  digest(): string;
+}
+
+export const EOL = "\n";
+
+export const defaultFS: CompleteFileSystem = new InMemoryFileSystem();
+
+export function ok(value: unknown, message?: string): asserts value {
+  if (!value) {
+    const error = Error(message);
+    error.name = "AssertionError";
+    error.message = message ?? "The expression evaluated to a falsy value.";
+    throw error;
+  }
+}
+
+export function createSHA256(): SHA256 {
+  const sha256 = new Sha256();
+  return {
+    update(data: string | Uint8Array): void {
+      sha256.update(data);
+    },
+    digest() {
+      return globalThis.btoa(sha256.digestSync().toString());
+    },
+  };
+}
+
+export function randomUUID(): string {
+  return crypto.randomUUID();
+}

packages/core/src/env/index.ts

@@ -0,0 +1,22 @@
+import { ok } from "node:assert";
+import { createHash, randomUUID } from "node:crypto";
+import fs from "node:fs/promises";
+import { EOL } from "node:os";
+import type { CompleteFileSystem } from "../storage";
+import type { SHA256 } from "./index.edge-light";
+
+export function createSHA256(): SHA256 {
+  const hash = createHash("sha256");
+  return {
+    update(data: string | Uint8Array): void {
+      hash.update(data);
+    },
+    digest() {
+      return hash.digest("base64");
+    },
+  };
+}
+
+export const defaultFS: CompleteFileSystem = fs;
+
+export { EOL, ok, randomUUID };

packages/core/src/extractors/MetadataExtractors.ts

@@ -0,0 +1,410 @@
+import { BaseNode, MetadataMode, TextNode } from "../Node";
+import { LLM } from "../llm";
+import {
+  defaultKeywordExtractorPromptTemplate,
+  defaultQuestionAnswerPromptTemplate,
+  defaultSummaryExtractorPromptTemplate,
+  defaultTitleCombinePromptTemplate,
+  defaultTitleExtractorPromptTemplate,
+} from "./prompts";
+import { BaseExtractor } from "./types";
+
+const STRIP_REGEX = /(\r\n|\n|\r)/gm;
+
+type ExtractKeyword = {
+  excerptKeywords: string;
+};
+
+/**
+ * Extract keywords from a list of nodes.
+ */
+export class KeywordExtractor extends BaseExtractor {
+  /**
+   * LLM instance.
+   * @type {LLM}
+   */
+  llm: LLM;
+
+  /**
+   * Number of keywords to extract.
+   * @type {number}
+   * @default 5
+   */
+  keywords: number = 5;
+
+  /**
+   * Constructor for the KeywordExtractor class.
+   * @param {LLM} llm LLM instance.
+   * @param {number} keywords Number of keywords to extract.
+   * @throws {Error} If keywords is less than 1.
+   */
+  constructor(llm: LLM, keywords: number = 5) {
+    if (keywords < 1) throw new Error("Keywords must be greater than 0");
+
+    super();
+    this.llm = llm;
+    this.keywords = keywords;
+  }
+
+  /**
+   *
+   * @param node Node to extract keywords from.
+   * @returns Keywords extracted from the node.
+   */
+  async extractKeywordsFromNodes(node: BaseNode): Promise<ExtractKeyword | {}> {
+    if (this.isTextNodeOnly && !(node instanceof TextNode)) {
+      return {};
+    }
+
+    const completion = await this.llm.complete({
+      prompt: defaultKeywordExtractorPromptTemplate({
+        contextStr: node.getContent(MetadataMode.ALL),
+        keywords: this.keywords,
+      }),
+    });
+
+    return {
+      excerptKeywords: completion.text,
+    };
+  }
+
+  /**
+   *
+   * @param nodes Nodes to extract keywords from.
+   * @returns Keywords extracted from the nodes.
+   */
+  async extract(nodes: BaseNode[]): Promise<Array<ExtractKeyword> | Array<{}>> {
+    const results = await Promise.all(
+      nodes.map((node) => this.extractKeywordsFromNodes(node)),
+    );
+    return results;
+  }
+}
+
+type ExtractTitle = {
+  documentTitle: string;
+};
+
+/**
+ * Extract title from a list of nodes.
+ */
+export class TitleExtractor extends BaseExtractor {
+  /**
+   * LLM instance.
+   * @type {LLM}
+   */
+  llm: LLM;
+
+  /**
+   * Can work for mixture of text and non-text nodes
+   * @type {boolean}
+   * @default false
+   */
+  isTextNodeOnly: boolean = false;
+
+  /**
+   * Number of nodes to extrct titles from.
+   * @type {number}
+   * @default 5
+   */
+  nodes: number = 5;
+
+  /**
+   * The prompt template to use for the title extractor.
+   * @type {string}
+   */
+  nodeTemplate: string;
+
+  /**
+   * The prompt template to merge title with..
+   * @type {string}
+   */
+  combineTemplate: string;
+
+  /**
+   * Constructor for the TitleExtractor class.
+   * @param {LLM} llm LLM instance.
+   * @param {number} nodes Number of nodes to extract titles from.
+   * @param {string} node_template The prompt template to use for the title extractor.
+   * @param {string} combine_template The prompt template to merge title with..
+   */
+  constructor(
+    llm: LLM,
+    nodes: number = 5,
+    node_template?: string,
+    combine_template?: string,
+  ) {
+    super();
+
+    this.llm = llm;
+    this.nodes = nodes;
+
+    this.nodeTemplate = node_template ?? defaultTitleExtractorPromptTemplate();
+    this.combineTemplate =
+      combine_template ?? defaultTitleCombinePromptTemplate();
+  }
+
+  /**
+   * Extract titles from a list of nodes.
+   * @param {BaseNode[]} nodes Nodes to extract titles from.
+   * @returns {Promise<BaseNode<ExtractTitle>[]>} Titles extracted from the nodes.
+   */
+  async extract(nodes: BaseNode[]): Promise<Array<ExtractTitle>> {
+    const nodesToExtractTitle: BaseNode[] = [];
+
+    for (let i = 0; i < this.nodes; i++) {
+      if (nodesToExtractTitle.length >= nodes.length) break;
+
+      if (this.isTextNodeOnly && !(nodes[i] instanceof TextNode)) continue;
+
+      nodesToExtractTitle.push(nodes[i]);
+    }
+
+    if (nodesToExtractTitle.length === 0) return [];
+
+    let titlesCandidates: string[] = [];
+    let title: string = "";
+
+    for (let i = 0; i < nodesToExtractTitle.length; i++) {
+      const completion = await this.llm.complete({
+        prompt: defaultTitleExtractorPromptTemplate({
+          contextStr: nodesToExtractTitle[i].getContent(MetadataMode.ALL),
+        }),
+      });
+
+      titlesCandidates.push(completion.text);
+    }
+
+    if (nodesToExtractTitle.length > 1) {
+      const combinedTitles = titlesCandidates.join(",");
+
+      const completion = await this.llm.complete({
+        prompt: defaultTitleCombinePromptTemplate({
+          contextStr: combinedTitles,
+        }),
+      });
+
+      title = completion.text;
+    }
+
+    if (nodesToExtractTitle.length === 1) {
+      title = titlesCandidates[0];
+    }
+
+    return nodes.map((_) => ({
+      documentTitle: title.trim().replace(STRIP_REGEX, ""),
+    }));
+  }
+}
+
+type ExtractQuestion = {
+  questionsThisExcerptCanAnswer: string;
+};
+
+/**
+ * Extract questions from a list of nodes.
+ */
+export class QuestionsAnsweredExtractor extends BaseExtractor {
+  /**
+   * LLM instance.
+   * @type {LLM}
+   */
+  llm: LLM;
+
+  /**
+   * Number of questions to generate.
+   * @type {number}
+   * @default 5
+   */
+  questions: number = 5;
+
+  /**
+   * The prompt template to use for the question extractor.
+   * @type {string}
+   */
+  promptTemplate: string;
+
+  /**
+   * Wheter to use metadata for embeddings only
+   * @type {boolean}
+   * @default false
+   */
+  embeddingOnly: boolean = false;
+
+  /**
+   * Constructor for the QuestionsAnsweredExtractor class.
+   * @param {LLM} llm LLM instance.
+   * @param {number} questions Number of questions to generate.
+   * @param {string} promptTemplate The prompt template to use for the question extractor.
+   * @param {boolean} embeddingOnly Wheter to use metadata for embeddings only.
+   */
+  constructor(
+    llm: LLM,
+    questions: number = 5,
+    promptTemplate?: string,
+    embeddingOnly: boolean = false,
+  ) {
+    if (questions < 1) throw new Error("Questions must be greater than 0");
+
+    super();
+
+    this.llm = llm;
+    this.questions = questions;
+    this.promptTemplate =
+      promptTemplate ??
+      defaultQuestionAnswerPromptTemplate({
+        numQuestions: questions,
+        contextStr: "",
+      });
+    this.embeddingOnly = embeddingOnly;
+  }
+
+  /**
+   * Extract answered questions from a node.
+   * @param {BaseNode} node Node to extract questions from.
+   * @returns {Promise<Array<ExtractQuestion> | Array<{}>>} Questions extracted from the node.
+   */
+  async extractQuestionsFromNode(
+    node: BaseNode,
+  ): Promise<ExtractQuestion | {}> {
+    if (this.isTextNodeOnly && !(node instanceof TextNode)) {
+      return {};
+    }
+
+    const contextStr = node.getContent(this.metadataMode);
+
+    const prompt = defaultQuestionAnswerPromptTemplate({
+      contextStr,
+      numQuestions: this.questions,
+    });
+
+    const questions = await this.llm.complete({
+      prompt,
+    });
+
+    return {
+      questionsThisExcerptCanAnswer: questions.text.replace(STRIP_REGEX, ""),
+    };
+  }
+
+  /**
+   * Extract answered questions from a list of nodes.
+   * @param {BaseNode[]} nodes Nodes to extract questions from.
+   * @returns {Promise<Array<ExtractQuestion> | Array<{}>>} Questions extracted from the nodes.
+   */
+  async extract(
+    nodes: BaseNode[],
+  ): Promise<Array<ExtractQuestion> | Array<{}>> {
+    const results = await Promise.all(
+      nodes.map((node) => this.extractQuestionsFromNode(node)),
+    );
+
+    return results;
+  }
+}
+
+type ExtractSummary = {
+  sectionSummary: string;
+  prevSectionSummary: string;
+  nextSectionSummary: string;
+};
+
+/**
+ * Extract summary from a list of nodes.
+ */
+export class SummaryExtractor extends BaseExtractor {
+  /**
+   * LLM instance.
+   * @type {LLM}
+   */
+  llm: LLM;
+
+  /**
+   * List of summaries to extract: 'self', 'prev', 'next'
+   * @type {string[]}
+   */
+  summaries: string[];
+
+  /**
+   * The prompt template to use for the summary extractor.
+   * @type {string}
+   */
+  promptTemplate: string;
+
+  private _selfSummary: boolean;
+  private _prevSummary: boolean;
+  private _nextSummary: boolean;
+
+  constructor(
+    llm: LLM,
+    summaries: string[] = ["self"],
+    promptTemplate?: string,
+  ) {
+    if (!summaries.some((s) => ["self", "prev", "next"].includes(s)))
+      throw new Error("Summaries must be one of 'self', 'prev', 'next'");
+
+    super();
+
+    this.llm = llm;
+    this.summaries = summaries;
+    this.promptTemplate =
+      promptTemplate ?? defaultSummaryExtractorPromptTemplate();
+
+    this._selfSummary = summaries.includes("self");
+    this._prevSummary = summaries.includes("prev");
+    this._nextSummary = summaries.includes("next");
+  }
+
+  /**
+   * Extract summary from a node.
+   * @param {BaseNode} node Node to extract summary from.
+   * @returns {Promise<string>} Summary extracted from the node.
+   */
+  async generateNodeSummary(node: BaseNode): Promise<string> {
+    if (this.isTextNodeOnly && !(node instanceof TextNode)) {
+      return "";
+    }
+
+    const contextStr = node.getContent(this.metadataMode);
+
+    const prompt = defaultSummaryExtractorPromptTemplate({
+      contextStr,
+    });
+
+    const summary = await this.llm.complete({
+      prompt,
+    });
+
+    return summary.text.replace(STRIP_REGEX, "");
+  }
+
+  /**
+   * Extract summaries from a list of nodes.
+   * @param {BaseNode[]} nodes Nodes to extract summaries from.
+   * @returns {Promise<Array<ExtractSummary> | Arry<{}>>} Summaries extracted from the nodes.
+   */
+  async extract(nodes: BaseNode[]): Promise<Array<ExtractSummary> | Array<{}>> {
+    if (!nodes.every((n) => n instanceof TextNode))
+      throw new Error("Only `TextNode` is allowed for `Summary` extractor");
+
+    const nodeSummaries = await Promise.all(
+      nodes.map((node) => this.generateNodeSummary(node)),
+    );
+
+    let metadataList: any[] = nodes.map(() => ({}));
+
+    for (let i = 0; i < nodes.length; i++) {
+      if (i > 0 && this._prevSummary && nodeSummaries[i - 1]) {
+        metadataList[i]["prevSectionSummary"] = nodeSummaries[i - 1];
+      }
+      if (i < nodes.length - 1 && this._nextSummary && nodeSummaries[i + 1]) {
+        metadataList[i]["nextSectionSummary"] = nodeSummaries[i + 1];
+      }
+      if (this._selfSummary && nodeSummaries[i]) {
+        metadataList[i]["sectionSummary"] = nodeSummaries[i];
+      }
+    }
+
+    return metadataList;
+  }
+}

packages/core/src/extractors/index.ts

@@ -0,0 +1,6 @@
+export {
+  KeywordExtractor,
+  QuestionsAnsweredExtractor,
+  SummaryExtractor,
+  TitleExtractor,
+} from "./MetadataExtractors";

packages/core/src/extractors/prompts.ts

@@ -0,0 +1,81 @@
+export interface DefaultPromptTemplate {
+  contextStr: string;
+}
+
+export interface DefaultKeywordExtractorPromptTemplate
+  extends DefaultPromptTemplate {
+  keywords: number;
+}
+
+export interface DefaultQuestionAnswerPromptTemplate
+  extends DefaultPromptTemplate {
+  numQuestions: number;
+}
+
+export interface DefaultNodeTextTemplate {
+  metadataStr: string;
+  content: string;
+}
+
+export const defaultKeywordExtractorPromptTemplate = ({
+  contextStr = "",
+  keywords = 5,
+}: DefaultKeywordExtractorPromptTemplate) => `
+  ${contextStr}. Give ${keywords} unique keywords for thiss
+  document. Format as comma separated. Keywords:
+`;
+
+export const defaultTitleExtractorPromptTemplate = (
+  { contextStr = "" }: DefaultPromptTemplate = {
+    contextStr: "",
+  },
+) => `
+  ${contextStr}. Give a title that summarizes all of the unique entities, titles or themes found in the context. Title:
+`;
+
+export const defaultTitleCombinePromptTemplate = (
+  { contextStr = "" }: DefaultPromptTemplate = {
+    contextStr: "",
+  },
+) => `
+  ${contextStr}. Based on the above candidate titles and content,s
+  what is the comprehensive title for this document? Title:
+`;
+
+export const defaultQuestionAnswerPromptTemplate = (
+  { contextStr = "", numQuestions = 5 }: DefaultQuestionAnswerPromptTemplate = {
+    contextStr: "",
+    numQuestions: 5,
+  },
+) => `
+  ${contextStr}. Given the contextual information,s
+  generate ${numQuestions} questions this context can provides
+  specific answers to which are unlikely to be found elsewhere.
+
+  Higher-level summaries of surrounding context may be provideds
+  as well. Try using these summaries to generate better questionss
+  that this context can answer.
+`;
+
+export const defaultSummaryExtractorPromptTemplate = (
+  { contextStr = "" }: DefaultPromptTemplate = {
+    contextStr: "",
+  },
+) => `
+  ${contextStr}. Summarize the key topics and entities of the section.s
+  Summary:
+`;
+
+export const defaultNodeTextTemplate = ({
+  metadataStr = "",
+  content = "",
+}: {
+  metadataStr?: string;
+  content?: string;
+} = {}) => `[Excerpt from document]
+${metadataStr}
+Excerpt:
+-----
+${content}
+-----
+`;

packages/core/src/extractors/types.ts

@@ -0,0 +1,73 @@
+import { BaseNode, MetadataMode, TextNode } from "../Node";
+import { TransformComponent } from "../ingestion";
+import { defaultNodeTextTemplate } from "./prompts";
+
+/*
+ * Abstract class for all extractors.
+ */
+export abstract class BaseExtractor implements TransformComponent {
+  isTextNodeOnly: boolean = true;
+  showProgress: boolean = true;
+  metadataMode: MetadataMode = MetadataMode.ALL;
+  disableTemplateRewrite: boolean = false;
+  inPlace: boolean = true;
+  numWorkers: number = 4;
+
+  abstract extract(nodes: BaseNode[]): Promise<Record<string, any>[]>;
+
+  async transform(nodes: BaseNode[], options?: any): Promise<BaseNode[]> {
+    return this.processNodes(
+      nodes,
+      options?.excludedEmbedMetadataKeys,
+      options?.excludedLlmMetadataKeys,
+    );
+  }
+
+  /**
+   *
+   * @param nodes Nodes to extract metadata from.
+   * @param excludedEmbedMetadataKeys Metadata keys to exclude from the embedding.
+   * @param excludedLlmMetadataKeys Metadata keys to exclude from the LLM.
+   * @returns Metadata extracted from the nodes.
+   */
+  async processNodes(
+    nodes: BaseNode[],
+    excludedEmbedMetadataKeys: string[] | undefined = undefined,
+    excludedLlmMetadataKeys: string[] | undefined = undefined,
+  ): Promise<BaseNode[]> {
+    let newNodes: BaseNode[];
+
+    if (this.inPlace) {
+      newNodes = nodes;
+    } else {
+      newNodes = nodes.slice();
+    }
+
+    let curMetadataList = await this.extract(newNodes);
+
+    for (let idx in newNodes) {
+      newNodes[idx].metadata = curMetadataList[idx];
+    }
+
+    for (let idx in newNodes) {
+      if (excludedEmbedMetadataKeys) {
+        newNodes[idx].excludedEmbedMetadataKeys.concat(
+          excludedEmbedMetadataKeys,
+        );
+      }
+      if (excludedLlmMetadataKeys) {
+        newNodes[idx].excludedLlmMetadataKeys.concat(excludedLlmMetadataKeys);
+      }
+      if (!this.disableTemplateRewrite) {
+        if (newNodes[idx] instanceof TextNode) {
+          newNodes[idx] = new TextNode({
+            ...newNodes[idx],
+            textTemplate: defaultNodeTextTemplate(),
+          });
+        }
+      }
+    }
+
+    return newNodes;
+  }
+}

packages/core/src/index.ts

@@ -15,7 +15,9 @@ export * from "./callbacks/CallbackManager";
 export * from "./constants";
 export * from "./embeddings";
 export * from "./engines/chat";
+export * from "./extractors";
 export * from "./indices";
+export * from "./ingestion";
 export * from "./llm";
 export * from "./nodeParsers";
 export * from "./postprocessors";

packages/core/src/indices/BaseIndex.ts

@@ -1,8 +1,8 @@
-import { randomUUID } from "node:crypto";
 import { BaseNode, Document, jsonToNode } from "../Node";
 import { BaseQueryEngine } from "../QueryEngine";
 import { BaseRetriever } from "../Retriever";
 import { ServiceContext } from "../ServiceContext";
+import { randomUUID } from "../env";
 import { StorageContext } from "../storage/StorageContext";
 import { BaseDocumentStore } from "../storage/docStore/types";
 import { BaseIndexStore } from "../storage/indexStore/types";

packages/core/src/indices/vectorStore/VectorIndexRetriever.ts

@@ -6,6 +6,7 @@ import { Event } from "../../callbacks/CallbackManager";
 import { DEFAULT_SIMILARITY_TOP_K } from "../../constants";
 import { BaseEmbedding } from "../../embeddings";
 import {
+  MetadataFilters,
   VectorStoreQuery,
   VectorStoreQueryMode,
   VectorStoreQueryResult,
@@ -40,7 +41,7 @@ export class VectorIndexRetriever implements BaseRetriever {
   async retrieve(
     query: string,
     parentEvent?: Event,
-    preFilters?: unknown,
+    preFilters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
     let nodesWithScores = await this.textRetrieve(query, preFilters);
     nodesWithScores = nodesWithScores.concat(
@@ -52,18 +53,23 @@ export class VectorIndexRetriever implements BaseRetriever {
 
   protected async textRetrieve(
     query: string,
-    preFilters?: unknown,
+    preFilters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
+    const options = {};
     const q = await this.buildVectorStoreQuery(
       this.index.embedModel,
       query,
       this.similarityTopK,
+      preFilters,
     );
-    const result = await this.index.vectorStore.query(q, preFilters);
+    const result = await this.index.vectorStore.query(q, options);
     return this.buildNodeListFromQueryResult(result);
   }
 
-  private async textToImageRetrieve(query: string, preFilters?: unknown) {
+  private async textToImageRetrieve(
+    query: string,
+    preFilters?: MetadataFilters,
+  ) {
     if (!this.index.imageEmbedModel || !this.index.imageVectorStore) {
       // no-op if image embedding and vector store are not set
       return [];
@@ -72,6 +78,7 @@ export class VectorIndexRetriever implements BaseRetriever {
       this.index.imageEmbedModel,
       query,
       this.imageSimilarityTopK,
+      preFilters,
     );
     const result = await this.index.imageVectorStore.query(q, preFilters);
     return this.buildNodeListFromQueryResult(result);
@@ -98,6 +105,7 @@ export class VectorIndexRetriever implements BaseRetriever {
     embedModel: BaseEmbedding,
     query: string,
     similarityTopK: number,
+    preFilters?: MetadataFilters,
   ): Promise<VectorStoreQuery> {
     const queryEmbedding = await embedModel.getQueryEmbedding(query);
 
@@ -105,6 +113,7 @@ export class VectorIndexRetriever implements BaseRetriever {
       queryEmbedding: queryEmbedding,
       mode: VectorStoreQueryMode.DEFAULT,
       similarityTopK: similarityTopK,
+      filters: preFilters ?? undefined,
     };
   }
 

packages/core/src/indices/vectorStore/VectorStoreIndex.ts

@@ -20,6 +20,7 @@ import {
 import { BaseNodePostprocessor } from "../../postprocessors";
 import {
   BaseIndexStore,
+  MetadataFilters,
   StorageContext,
   VectorStore,
   storageContextFromDefaults,
@@ -263,7 +264,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
   asQueryEngine(options?: {
     retriever?: BaseRetriever;
     responseSynthesizer?: BaseSynthesizer;
-    preFilters?: unknown;
+    preFilters?: MetadataFilters;
     nodePostprocessors?: BaseNodePostprocessor[];
   }): BaseQueryEngine {
     const { retriever, responseSynthesizer } = options ?? {};

packages/core/src/ingestion/IngestionPipeline.ts

@@ -0,0 +1,94 @@
+import { BaseNode, Document } from "../Node";
+import { BaseReader } from "../readers/base";
+import { BaseDocumentStore, VectorStore } from "../storage";
+import { DocStoreStrategy, createDocStoreStrategy } from "./strategies";
+import { TransformComponent } from "./types";
+
+interface IngestionRunArgs {
+  documents?: Document[];
+  nodes?: BaseNode[];
+  inPlace?: boolean;
+}
+
+export async function runTransformations(
+  nodesToRun: BaseNode[],
+  transformations: TransformComponent[],
+  transformOptions: any = {},
+  { inPlace = true }: IngestionRunArgs,
+): Promise<BaseNode[]> {
+  let nodes = nodesToRun;
+  if (!inPlace) {
+    nodes = [...nodesToRun];
+  }
+  for (const transform of transformations) {
+    nodes = await transform.transform(nodes, transformOptions);
+  }
+  return nodes;
+}
+
+// TODO: add caching, add concurrency
+export class IngestionPipeline {
+  transformations: TransformComponent[] = [];
+  documents?: Document[];
+  reader?: BaseReader;
+  vectorStore?: VectorStore;
+  docStore?: BaseDocumentStore;
+  docStoreStrategy: DocStoreStrategy = DocStoreStrategy.UPSERTS;
+  disableCache: boolean = true;
+
+  private _docStoreStrategy?: TransformComponent;
+
+  constructor(init?: Partial<IngestionPipeline>) {
+    Object.assign(this, init);
+    this._docStoreStrategy = createDocStoreStrategy(
+      this.docStoreStrategy,
+      this.docStore,
+      this.vectorStore,
+    );
+  }
+
+  async prepareInput(
+    documents?: Document[],
+    nodes?: BaseNode[],
+  ): Promise<BaseNode[]> {
+    const inputNodes: BaseNode[] = [];
+    if (documents) {
+      inputNodes.push(...documents);
+    }
+    if (nodes) {
+      inputNodes.push(...nodes);
+    }
+    if (this.documents) {
+      inputNodes.push(...this.documents);
+    }
+    if (this.reader) {
+      inputNodes.push(...(await this.reader.loadData()));
+    }
+    return inputNodes;
+  }
+
+  async run(
+    args: IngestionRunArgs = {},
+    transformOptions?: any,
+  ): Promise<BaseNode[]> {
+    const inputNodes = await this.prepareInput(args.documents, args.nodes);
+    let nodesToRun;
+    if (this._docStoreStrategy) {
+      nodesToRun = await this._docStoreStrategy.transform(inputNodes);
+    } else {
+      nodesToRun = inputNodes;
+    }
+
+    const nodes = await runTransformations(
+      nodesToRun,
+      this.transformations,
+      transformOptions,
+      args,
+    );
+    if (this.vectorStore) {
+      const nodesToAdd = nodes.filter((node) => node.embedding);
+      await this.vectorStore.add(nodesToAdd);
+    }
+    return nodes;
+  }
+}

packages/core/src/ingestion/index.ts

@@ -0,0 +1,2 @@
+export * from "./IngestionPipeline";
+export * from "./types";

packages/core/src/ingestion/strategies/DuplicatesStrategy.ts

@@ -0,0 +1,32 @@
+import { BaseNode } from "../../Node";
+import { BaseDocumentStore } from "../../storage";
+import { TransformComponent } from "../types";
+
+/**
+ * Handle doc store duplicates by checking all hashes.
+ */
+export class DuplicatesStrategy implements TransformComponent {
+  private docStore: BaseDocumentStore;
+
+  constructor(docStore: BaseDocumentStore) {
+    this.docStore = docStore;
+  }
+
+  async transform(nodes: BaseNode[]): Promise<BaseNode[]> {
+    const hashes = await this.docStore.getAllDocumentHashes();
+    const currentHashes = new Set<string>();
+    const nodesToRun: BaseNode[] = [];
+
+    for (const node of nodes) {
+      if (!(node.hash in hashes) && !currentHashes.has(node.hash)) {
+        this.docStore.setDocumentHash(node.id_, node.hash);
+        nodesToRun.push(node);
+        currentHashes.add(node.hash);
+      }
+    }
+
+    this.docStore.addDocuments(nodesToRun, true);
+
+    return nodesToRun;
+  }
+}

packages/core/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts

@@ -0,0 +1,44 @@
+import { BaseNode } from "../../Node";
+import { BaseDocumentStore, VectorStore } from "../../storage";
+import { classify } from "./classify";
+
+/**
+ * Handle docstore upserts by checking hashes and ids.
+ * Identify missing docs and delete them from docstore and vector store
+ */
+export class UpsertsAndDeleteStrategy {
+  protected docStore: BaseDocumentStore;
+  protected vectorStore?: VectorStore;
+
+  constructor(docStore: BaseDocumentStore, vectorStore?: VectorStore) {
+    this.docStore = docStore;
+    this.vectorStore = vectorStore;
+  }
+
+  async transform(nodes: BaseNode[]): Promise<BaseNode[]> {
+    const { dedupedNodes, missingDocs, unusedDocs } = await classify(
+      this.docStore,
+      nodes,
+    );
+
+    // remove unused docs
+    for (const refDocId of unusedDocs) {
+      await this.docStore.deleteRefDoc(refDocId, false);
+      if (this.vectorStore) {
+        await this.vectorStore.delete(refDocId);
+      }
+    }
+
+    // remove missing docs
+    for (const docId of missingDocs) {
+      await this.docStore.deleteDocument(docId, true);
+      if (this.vectorStore) {
+        await this.vectorStore.delete(docId);
+      }
+    }
+
+    await this.docStore.addDocuments(dedupedNodes, true);
+
+    return dedupedNodes;
+  }
+}

packages/core/src/ingestion/strategies/UpsertsStrategy.ts

@@ -0,0 +1,31 @@
+import { BaseNode } from "../../Node";
+import { BaseDocumentStore, VectorStore } from "../../storage";
+import { TransformComponent } from "../types";
+import { classify } from "./classify";
+
+/**
+ * Handles doc store upserts by checking hashes and ids.
+ */
+export class UpsertsStrategy implements TransformComponent {
+  protected docStore: BaseDocumentStore;
+  protected vectorStore?: VectorStore;
+
+  constructor(docStore: BaseDocumentStore, vectorStore?: VectorStore) {
+    this.docStore = docStore;
+    this.vectorStore = vectorStore;
+  }
+
+  async transform(nodes: BaseNode[]): Promise<BaseNode[]> {
+    const { dedupedNodes, unusedDocs } = await classify(this.docStore, nodes);
+    // remove unused docs
+    for (const refDocId of unusedDocs) {
+      await this.docStore.deleteRefDoc(refDocId, false);
+      if (this.vectorStore) {
+        await this.vectorStore.delete(refDocId);
+      }
+    }
+    // add non-duplicate docs
+    this.docStore.addDocuments(dedupedNodes, true);
+    return dedupedNodes;
+  }
+}

packages/core/src/ingestion/strategies/classify.ts

@@ -0,0 +1,27 @@
+import { BaseNode } from "../../Node";
+import { BaseDocumentStore } from "../../storage";
+
+export async function classify(docStore: BaseDocumentStore, nodes: BaseNode[]) {
+  const existingDocIds = Object.values(await docStore.getAllDocumentHashes());
+  const docIdsFromNodes = new Set<string>();
+  const dedupedNodes: BaseNode[] = [];
+  const unusedDocs: string[] = [];
+
+  for (const node of nodes) {
+    const refDocId = node.sourceNode?.nodeId || node.id_;
+    docIdsFromNodes.add(refDocId);
+    const existingHash = await docStore.getDocumentHash(refDocId);
+
+    if (!existingHash) {
+      // document doesn't exist, so add it
+      dedupedNodes.push(node);
+    } else if (existingHash && existingHash !== node.hash) {
+      // document exists but hash is different, so mark doc as unused and add node as deduped
+      unusedDocs.push(refDocId);
+      dedupedNodes.push(node);
+    }
+    // otherwise, document exists and hash is the same, so do nothing
+  }
+  const missingDocs = existingDocIds.filter((id) => !docIdsFromNodes.has(id));
+  return { dedupedNodes, missingDocs, unusedDocs };
+}

packages/core/src/ingestion/strategies/index.ts

@@ -0,0 +1,40 @@
+import { BaseDocumentStore, VectorStore } from "../../storage";
+import { TransformComponent } from "../types";
+import { DuplicatesStrategy } from "./DuplicatesStrategy";
+import { UpsertsStrategy } from "./UpsertsStrategy";
+
+export enum DocStoreStrategy {
+  UPSERTS = "upserts",
+  DUPLICATES_ONLY = "duplicates_only",
+  UPSERTS_AND_DELETE = "upserts_and_delete",
+}
+
+export function createDocStoreStrategy(
+  docStoreStrategy: DocStoreStrategy,
+  docStore?: BaseDocumentStore,
+  vectorStore?: VectorStore,
+): TransformComponent | undefined {
+  if (docStore && vectorStore) {
+    if (
+      docStoreStrategy === DocStoreStrategy.UPSERTS ||
+      docStoreStrategy === DocStoreStrategy.UPSERTS_AND_DELETE
+    ) {
+      return new UpsertsStrategy(docStore, vectorStore);
+    } else if (docStoreStrategy === DocStoreStrategy.DUPLICATES_ONLY) {
+      return new DuplicatesStrategy(docStore);
+    } else {
+      throw new Error(`Invalid docstore strategy: ${docStoreStrategy}`);
+    }
+  } else if (docStore && !vectorStore) {
+    if (docStoreStrategy === DocStoreStrategy.UPSERTS) {
+      console.warn(
+        "Docstore strategy set to upserts, but no vector store. Switching to duplicates_only strategy.",
+      );
+    } else if (docStoreStrategy === DocStoreStrategy.UPSERTS_AND_DELETE) {
+      console.warn(
+        "Docstore strategy set to upserts and delete, but no vector store. Switching to duplicates_only strategy.",
+      );
+    }
+    return new DuplicatesStrategy(docStore);
+  }
+}

packages/core/src/ingestion/types.ts

@@ -0,0 +1,5 @@
+import { BaseNode } from "../Node";
+
+export interface TransformComponent {
+  transform(nodes: BaseNode[], options?: any): Promise<BaseNode[]>;
+}

packages/core/src/llm/ollama.ts

@@ -1,6 +1,6 @@
-import { ok } from "node:assert";
 import { CallbackManager, Event } from "../callbacks/CallbackManager";
 import { BaseEmbedding } from "../embeddings";
+import { ok } from "../env";
 import {
   ChatMessage,
   ChatResponse,

packages/core/src/nodeParsers/SentenceWindowNodeParser.ts

@@ -44,6 +44,10 @@ export class SentenceWindowNodeParser implements NodeParser {
     return new SentenceWindowNodeParser(init);
   }
 
+  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
+    return this.getNodesFromDocuments(nodes);
+  }
+
   getNodesFromDocuments(documents: BaseNode[]) {
     return documents
       .map((document) => this.buildWindowNodesFromDocument(document))

packages/core/src/nodeParsers/SimpleNodeParser.ts

@@ -38,6 +38,10 @@ export class SimpleNodeParser implements NodeParser {
     this.includePrevNextRel = init?.includePrevNextRel ?? true;
   }
 
+  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
+    return this.getNodesFromDocuments(nodes);
+  }
+
   static fromDefaults(init?: {
     chunkSize?: number;
     chunkOverlap?: number;

packages/core/src/nodeParsers/types.ts

@@ -1,9 +1,10 @@
 import { BaseNode } from "../Node";
+import { TransformComponent } from "../ingestion";
 
 /**
  * A NodeParser generates Nodes from Documents
  */
-export interface NodeParser {
+export interface NodeParser extends TransformComponent {
   /**
    * Generates an array of nodes from an array of documents.
    * @param documents - The documents to generate nodes from.

packages/core/src/readers/CSVReader.ts

@@ -1,6 +1,7 @@
 import Papa, { ParseConfig } from "papaparse";
 import { Document } from "../Node";
-import { DEFAULT_FS, GenericFileSystem } from "../storage/FileSystem";
+import { defaultFS } from "../env";
+import { GenericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
 /**
@@ -40,7 +41,7 @@ export class PapaCSVReader implements BaseReader {
    */
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const fileContent: string = await fs.readFile(file, "utf-8");
     const result = Papa.parse(fileContent, this.papaConfig);

packages/core/src/readers/DocxReader.ts

@@ -1,14 +1,14 @@
 import mammoth from "mammoth";
 import { Document } from "../Node";
+import { defaultFS } from "../env";
 import { GenericFileSystem } from "../storage/FileSystem";
-import { DEFAULT_FS } from "../storage/constants";
 import { BaseReader } from "./base";
 
 export class DocxReader implements BaseReader {
   /** DocxParser */
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const dataBuffer = (await fs.readFile(file)) as any;
     const { value } = await mammoth.extractRawText({ buffer: dataBuffer });

packages/core/src/readers/HTMLReader.ts

@@ -1,5 +1,5 @@
 import { Document } from "../Node";
-import { DEFAULT_FS } from "../storage/constants";
+import { defaultFS } from "../env";
 import { GenericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
@@ -20,7 +20,7 @@ export class HTMLReader implements BaseReader {
    */
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readFile(file, "utf-8");
     const htmlOptions = this.getOptions();

packages/core/src/readers/ImageReader.ts

@@ -1,5 +1,5 @@
 import { Document, ImageDocument } from "../Node";
-import { DEFAULT_FS } from "../storage/constants";
+import { defaultFS } from "../env";
 import { GenericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
@@ -16,7 +16,7 @@ export class ImageReader implements BaseReader {
    */
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readFile(file);
     const blob = new Blob([dataBuffer]);

packages/core/src/readers/MarkdownReader.ts

@@ -1,5 +1,6 @@
 import { Document } from "../Node";
-import { DEFAULT_FS, GenericFileSystem } from "../storage";
+import { defaultFS } from "../env";
+import { GenericFileSystem } from "../storage";
 import { BaseReader } from "./base";
 
 type MarkdownTuple = [string | null, string];
@@ -89,7 +90,7 @@ export class MarkdownReader implements BaseReader {
 
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const content = await fs.readFile(file, { encoding: "utf-8" });
     const tups = this.parseTups(content);

packages/core/src/readers/PDFReader.ts

@@ -1,6 +1,6 @@
 import { Document } from "../Node";
+import { defaultFS } from "../env";
 import { GenericFileSystem } from "../storage/FileSystem";
-import { DEFAULT_FS } from "../storage/constants";
 import { BaseReader } from "./base";
 
 /**
@@ -9,7 +9,7 @@ import { BaseReader } from "./base";
 export class PDFReader implements BaseReader {
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const content = (await fs.readFile(file)) as any;
     if (!(content instanceof Buffer)) {

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -1,7 +1,7 @@
 import _ from "lodash";
 import { Document } from "../Node";
+import { defaultFS } from "../env";
 import { CompleteFileSystem, walk } from "../storage/FileSystem";
-import { DEFAULT_FS } from "../storage/constants";
 import { PapaCSVReader } from "./CSVReader";
 import { DocxReader } from "./DocxReader";
 import { HTMLReader } from "./HTMLReader";
@@ -28,7 +28,7 @@ enum ReaderStatus {
 export class TextFileReader implements BaseReader {
   async loadData(
     file: string,
-    fs: CompleteFileSystem = DEFAULT_FS as CompleteFileSystem,
+    fs: CompleteFileSystem = defaultFS,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readFile(file, "utf-8");
     return [new Document({ text: dataBuffer, id_: file })];
@@ -66,7 +66,7 @@ export class SimpleDirectoryReader implements BaseReader {
 
   async loadData({
     directoryPath,
-    fs = DEFAULT_FS as CompleteFileSystem,
+    fs = defaultFS,
     defaultReader = new TextFileReader(),
     fileExtToReader = FILE_EXT_TO_READER,
   }: SimpleDirectoryReaderLoadDataProps): Promise<Document[]> {

packages/core/src/storage/FileSystem.ts

@@ -1,34 +1,51 @@
 import _ from "lodash";
+import type nodeFS from "node:fs/promises";
+
 /**
  * A filesystem interface that is meant to be compatible with
  * the 'fs' module from Node.js.
  * Allows for the use of similar inteface implementation on
  * browsers.
  */
-
-export interface GenericFileSystem {
-  writeFile(path: string, content: string, options?: any): Promise<void>;
-  readFile(path: string, options?: any): Promise<string>;
+export type GenericFileSystem = {
+  writeFile(
+    path: string,
+    content: string,
+    options?: Parameters<typeof nodeFS.writeFile>[2],
+  ): Promise<void>;
+  readFile(
+    path: string,
+    options?: Parameters<typeof nodeFS.readFile>[1],
+  ): Promise<string>;
   access(path: string): Promise<void>;
-  mkdir(path: string, options?: any): Promise<void>;
-}
+  mkdir(
+    path: string,
+    options?: Parameters<typeof nodeFS.mkdir>[1],
+  ): Promise<void>;
+};
 
-export interface WalkableFileSystem {
+export type WalkableFileSystem = {
   readdir(path: string): Promise<string[]>;
   stat(path: string): Promise<any>;
-}
+};
+
+export type CompleteFileSystem = GenericFileSystem & WalkableFileSystem;
 
 /**
  * A filesystem implementation that stores files in memory.
  */
-export class InMemoryFileSystem implements GenericFileSystem {
+export class InMemoryFileSystem implements CompleteFileSystem {
   private files: Record<string, any> = {};
 
-  async writeFile(path: string, content: string, options?: any): Promise<void> {
+  async writeFile(
+    path: string,
+    content: string,
+    options?: unknown,
+  ): Promise<void> {
     this.files[path] = _.cloneDeep(content);
   }
 
-  async readFile(path: string, options?: any): Promise<string> {
+  async readFile(path: string, options?: unknown): Promise<string> {
     if (!(path in this.files)) {
       throw new Error(`File ${path} does not exist`);
     }
@@ -41,27 +58,19 @@ export class InMemoryFileSystem implements GenericFileSystem {
     }
   }
 
-  async mkdir(path: string, options?: any): Promise<void> {
+  async mkdir(path: string, options?: unknown): Promise<void> {
     this.files[path] = _.get(this.files, path, null);
   }
-}
 
-export type CompleteFileSystem = GenericFileSystem & WalkableFileSystem;
+  async readdir(path: string): Promise<string[]> {
+    throw new Error("Not implemented");
+  }
 
-export function getNodeFS(): CompleteFileSystem {
-  const fs = require("fs/promises");
-  return fs;
+  async stat(path: string): Promise<any> {
+    throw new Error("Not implemented");
+  }
 }
 
-let fs = null;
-try {
-  fs = getNodeFS();
-} catch (e) {
-  fs = new InMemoryFileSystem();
-}
-export const DEFAULT_FS: GenericFileSystem | CompleteFileSystem =
-  fs as GenericFileSystem;
-
 // FS utility functions
 
 /**
@@ -92,12 +101,6 @@ export async function* walk(
   fs: WalkableFileSystem,
   dirPath: string,
 ): AsyncIterable<string> {
-  if (fs instanceof InMemoryFileSystem) {
-    throw new Error(
-      "The InMemoryFileSystem does not support directory traversal.",
-    );
-  }
-
   const entries = await fs.readdir(dirPath);
   for (const entry of entries) {
     const fullPath = `${dirPath}/${entry}`;

packages/core/src/storage/StorageContext.ts

@@ -1,10 +1,7 @@
 import path from "path";
+import { defaultFS } from "../env";
 import { GenericFileSystem } from "./FileSystem";
-import {
-  DEFAULT_FS,
-  DEFAULT_IMAGE_VECTOR_NAMESPACE,
-  DEFAULT_NAMESPACE,
-} from "./constants";
+import { DEFAULT_IMAGE_VECTOR_NAMESPACE, DEFAULT_NAMESPACE } from "./constants";
 import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore";
 import { BaseDocumentStore } from "./docStore/types";
 import { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
@@ -19,14 +16,14 @@ export interface StorageContext {
   imageVectorStore?: VectorStore;
 }
 
-type BuilderParams = {
-  docStore?: BaseDocumentStore;
-  indexStore?: BaseIndexStore;
-  vectorStore?: VectorStore;
-  imageVectorStore?: VectorStore;
-  storeImages?: boolean;
-  persistDir?: string;
-  fs?: GenericFileSystem;
+export type BuilderParams = {
+  docStore: BaseDocumentStore;
+  indexStore: BaseIndexStore;
+  vectorStore: VectorStore;
+  imageVectorStore: VectorStore;
+  storeImages: boolean;
+  persistDir: string;
+  fs: GenericFileSystem;
 };
 
 export async function storageContextFromDefaults({
@@ -37,14 +34,14 @@ export async function storageContextFromDefaults({
   storeImages,
   persistDir,
   fs,
-}: BuilderParams): Promise<StorageContext> {
+}: Partial<BuilderParams>): Promise<StorageContext> {
   if (!persistDir) {
     docStore = docStore || new SimpleDocumentStore();
     indexStore = indexStore || new SimpleIndexStore();
     vectorStore = vectorStore || new SimpleVectorStore();
     imageVectorStore = storeImages ? new SimpleVectorStore() : imageVectorStore;
   } else {
-    fs = fs || DEFAULT_FS;
+    fs = fs || defaultFS;
     docStore =
       docStore ||
       (await SimpleDocumentStore.fromPersistDir(

packages/core/src/storage/constants.ts

@@ -6,4 +6,3 @@ export const DEFAULT_VECTOR_STORE_PERSIST_FILENAME = "vector_store.json";
 export const DEFAULT_GRAPH_STORE_PERSIST_FILENAME = "graph_store.json";
 export const DEFAULT_NAMESPACE = "docstore";
 export const DEFAULT_IMAGE_VECTOR_NAMESPACE = "images";
-export { DEFAULT_FS } from "./FileSystem";

packages/core/src/storage/docStore/KVDocumentStore.ts

@@ -175,4 +175,16 @@ export class KVDocumentStore extends BaseDocumentStore {
     let metadata = await this.kvstore.get(docId, this.metadataCollection);
     return _.get(metadata, "docHash");
   }
+
+  async getAllDocumentHashes(): Promise<Record<string, string>> {
+    let hashes: Record<string, string> = {};
+    const metadataDocs = await this.kvstore.getAll(this.metadataCollection);
+    for (const docId in metadataDocs) {
+      const hash = await this.getDocumentHash(docId);
+      if (hash) {
+        hashes[hash] = docId;
+      }
+    }
+    return hashes;
+  }
 }

packages/core/src/storage/docStore/SimpleDocumentStore.ts

@@ -1,9 +1,9 @@
 import _ from "lodash";
 import path from "path";
+import { defaultFS } from "../../env";
 import { GenericFileSystem } from "../FileSystem";
 import {
   DEFAULT_DOC_STORE_PERSIST_FILENAME,
-  DEFAULT_FS,
   DEFAULT_NAMESPACE,
   DEFAULT_PERSIST_DIR,
 } from "../constants";
@@ -44,7 +44,7 @@ export class SimpleDocumentStore extends KVDocumentStore {
     namespace?: string,
     fs?: GenericFileSystem,
   ): Promise<SimpleDocumentStore> {
-    fs = fs || DEFAULT_FS;
+    fs = fs || defaultFS;
     const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
     return new SimpleDocumentStore(simpleKVStore, namespace);
   }
@@ -56,7 +56,7 @@ export class SimpleDocumentStore extends KVDocumentStore {
     ),
     fs?: GenericFileSystem,
   ): Promise<void> {
-    fs = fs || DEFAULT_FS;
+    fs = fs || defaultFS;
     if (
       _.isObject(this.kvStore) &&
       this.kvStore instanceof BaseInMemoryKVStore

packages/core/src/storage/docStore/types.ts

@@ -40,6 +40,8 @@ export abstract class BaseDocumentStore {
 
   abstract getDocumentHash(docId: string): Promise<string | undefined>;
 
+  abstract getAllDocumentHashes(): Promise<Record<string, string>>;
+
   // Ref Docs
   abstract getAllRefDocInfo(): Promise<Record<string, RefDocInfo> | undefined>;
 

packages/core/src/storage/index.ts

@@ -12,5 +12,6 @@ export { ChromaVectorStore } from "./vectorStore/ChromaVectorStore";
 export { MongoDBAtlasVectorSearch } from "./vectorStore/MongoDBAtlasVectorStore";
 export { PGVectorStore } from "./vectorStore/PGVectorStore";
 export { PineconeVectorStore } from "./vectorStore/PineconeVectorStore";
+export { QdrantVectorStore } from "./vectorStore/QdrantVectorStore";
 export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
 export * from "./vectorStore/types";

packages/core/src/storage/indexStore/SimpleIndexStore.ts

@@ -1,7 +1,7 @@
 import path from "path";
+import { defaultFS } from "../../env";
 import { GenericFileSystem } from "../FileSystem";
 import {
-  DEFAULT_FS,
   DEFAULT_INDEX_STORE_PERSIST_FILENAME,
   DEFAULT_PERSIST_DIR,
 } from "../constants";
@@ -20,7 +20,7 @@ export class SimpleIndexStore extends KVIndexStore {
 
   static async fromPersistDir(
     persistDir: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<SimpleIndexStore> {
     const persistPath = path.join(
       persistDir,
@@ -31,7 +31,7 @@ export class SimpleIndexStore extends KVIndexStore {
 
   static async fromPersistPath(
     persistPath: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<SimpleIndexStore> {
     let simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
     return new SimpleIndexStore(simpleKVStore);
@@ -39,7 +39,7 @@ export class SimpleIndexStore extends KVIndexStore {
 
   async persist(
     persistPath: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<void> {
     await this.kvStore.persist(persistPath, fs);
   }

packages/core/src/storage/kvStore/SimpleKVStore.ts

@@ -1,19 +1,18 @@
 import _ from "lodash";
 import path from "path";
+import { defaultFS } from "../../env";
 import { GenericFileSystem, exists } from "../FileSystem";
-import { DEFAULT_COLLECTION, DEFAULT_FS } from "../constants";
+import { DEFAULT_COLLECTION } from "../constants";
 import { BaseKVStore } from "./types";
 
 export type DataType = Record<string, Record<string, any>>;
 
 export class SimpleKVStore extends BaseKVStore {
-  private data: DataType;
   private persistPath: string | undefined;
   private fs: GenericFileSystem | undefined;
 
-  constructor(data?: DataType) {
+  constructor(private data: DataType = {}) {
     super();
-    this.data = data || {};
   }
 
   async put(
@@ -61,7 +60,7 @@ export class SimpleKVStore extends BaseKVStore {
   }
 
   async persist(persistPath: string, fs?: GenericFileSystem): Promise<void> {
-    fs = fs || DEFAULT_FS;
+    fs = fs || defaultFS;
     // TODO: decide on a way to polyfill path
     let dirPath = path.dirname(persistPath);
     if (!(await exists(fs, dirPath))) {
@@ -74,7 +73,7 @@ export class SimpleKVStore extends BaseKVStore {
     persistPath: string,
     fs?: GenericFileSystem,
   ): Promise<SimpleKVStore> {
-    fs = fs || DEFAULT_FS;
+    fs = fs || defaultFS;
     let dirPath = path.dirname(persistPath);
     if (!(await exists(fs, dirPath))) {
       await fs.mkdir(dirPath);

packages/core/src/storage/vectorStore/ChromaVectorStore.ts

@@ -52,7 +52,7 @@ export class ChromaVectorStore implements VectorStore {
 
   async getCollection(): Promise<Collection> {
     if (!this.collection) {
-      const coll = await this.chromaClient.createCollection({
+      const coll = await this.chromaClient.getOrCreateCollection({
         name: this.collectionName,
       });
       this.collection = coll;
@@ -107,7 +107,7 @@ export class ChromaVectorStore implements VectorStore {
     }
 
     const chromaWhere: { [x: string]: string | number | boolean } = {};
-    if (query.filters) {
+    if (query.filters?.filters) {
       query.filters.filters.map((filter) => {
         const filterKey = filter.key;
         const filterValue = filter.value;

packages/core/src/storage/vectorStore/MongoDBAtlasVectorStore.ts

@@ -13,7 +13,7 @@ function toMongoDBFilter(
   standardFilters: MetadataFilters,
 ): Record<string, any> {
   const filters: Record<string, any> = {};
-  for (const filter of standardFilters.filters) {
+  for (const filter of standardFilters?.filters ?? []) {
     filters[filter.key] = filter.value;
   }
   return filters;

packages/core/src/storage/vectorStore/QdrantVectorStore.ts

@@ -0,0 +1,339 @@
+import { BaseNode } from "../../Node";
+import { VectorStore, VectorStoreQuery, VectorStoreQueryResult } from "./types";
+
+import { QdrantClient } from "@qdrant/js-client-rest";
+import { metadataDictToNode, nodeToMetadata } from "./utils";
+
+type PointStruct = {
+  id: string;
+  payload: Record<string, string>;
+  vector: number[];
+};
+
+type QdrantParams = {
+  collectionName?: string;
+  client?: QdrantClient;
+  url?: string;
+  apiKey?: string;
+  batchSize?: number;
+};
+
+type QuerySearchResult = {
+  id: string;
+  score: number;
+  payload: Record<string, unknown>;
+  vector: number[] | null;
+  version: number;
+};
+
+/**
+ * Qdrant vector store.
+ */
+export class QdrantVectorStore implements VectorStore {
+  storesText: boolean = true;
+
+  db: QdrantClient;
+
+  collectionName: string;
+  batchSize: number;
+
+  private _collectionInitialized: boolean = false;
+
+  /**
+   * Creates a new QdrantVectorStore.
+   * @param collectionName Qdrant collection name
+   * @param client Qdrant client
+   * @param url Qdrant URL
+   * @param apiKey Qdrant API key
+   * @param batchSize Number of vectors to upload in a single batch
+   */
+  constructor({
+    collectionName,
+    client,
+    url,
+    apiKey,
+    batchSize,
+  }: QdrantParams) {
+    if (!client && (!url || !apiKey)) {
+      if (!url || !apiKey || !collectionName) {
+        throw new Error(
+          "QdrantVectorStore requires url, apiKey and collectionName",
+        );
+      }
+    }
+
+    if (client) {
+      this.db = client;
+    } else {
+      this.db = new QdrantClient({
+        url: url,
+        apiKey: apiKey,
+      });
+    }
+
+    this.collectionName = collectionName ?? "default";
+    this.batchSize = batchSize ?? 100;
+  }
+
+  /**
+   * Returns the Qdrant client.
+   * @returns Qdrant client
+   */
+  client() {
+    return this.db;
+  }
+
+  /**
+   * Creates a collection in Qdrant.
+   * @param collectionName Qdrant collection name
+   * @param vectorSize Dimensionality of the vectors
+   */
+  async createCollection(collectionName: string, vectorSize: number) {
+    await this.db.createCollection(collectionName, {
+      vectors: {
+        size: vectorSize,
+        distance: "Cosine",
+      },
+    });
+  }
+
+  /**
+   * Checks if the collection exists in Qdrant and creates it if not.
+   * @param collectionName Qdrant collection name
+   * @returns
+   */
+  async collectionExists(collectionName: string): Promise<boolean> {
+    try {
+      await this.db.getCollection(collectionName);
+      return true;
+    } catch (e) {
+      return false;
+    }
+  }
+
+  /**
+   * Initializes the collection in Qdrant.
+   * @param vectorSize Dimensionality of the vectors
+   */
+  async initializeCollection(vectorSize: number) {
+    const exists = await this.collectionExists(this.collectionName);
+    if (!exists) {
+      await this.createCollection(this.collectionName, vectorSize);
+    }
+    this._collectionInitialized = true;
+  }
+
+  /**
+   * Builds a list of points from the given nodes.
+   * @param nodes
+   * @returns
+   */
+  async buildPoints(nodes: BaseNode[]): Promise<{
+    points: PointStruct[];
+    ids: string[];
+  }> {
+    const points: PointStruct[] = [];
+    const ids = [];
+
+    for (let i = 0; i < nodes.length; i++) {
+      const nodeIds = [];
+      const vectors = [];
+      const payloads = [];
+
+      for (let j = 0; j < this.batchSize && i < nodes.length; j++, i++) {
+        const node = nodes[i];
+
+        nodeIds.push(node);
+
+        vectors.push(node.getEmbedding());
+
+        const metadata = nodeToMetadata(node);
+
+        payloads.push(metadata);
+      }
+
+      for (let k = 0; k < nodeIds.length; k++) {
+        const point: PointStruct = {
+          id: nodeIds[k].id_,
+          payload: payloads[k],
+          vector: vectors[k],
+        };
+
+        points.push(point);
+      }
+
+      ids.push(...nodeIds.map((node) => node.id_));
+    }
+
+    return {
+      points: points,
+      ids: ids,
+    };
+  }
+
+  /**
+   * Adds the given nodes to the vector store.
+   * @param embeddingResults List of nodes
+   * @returns List of node IDs
+   */
+  async add(embeddingResults: BaseNode[]): Promise<string[]> {
+    if (embeddingResults.length > 0 && !this._collectionInitialized) {
+      await this.initializeCollection(
+        embeddingResults[0].getEmbedding().length,
+      );
+    }
+
+    const { points, ids } = await this.buildPoints(embeddingResults);
+
+    const batchUpsert = async (points: PointStruct[]) => {
+      await this.db.upsert(this.collectionName, {
+        points: points,
+      });
+    };
+
+    for (let i = 0; i < points.length; i += this.batchSize) {
+      const chunk = points.slice(i, i + this.batchSize);
+      await batchUpsert(chunk);
+    }
+
+    return ids;
+  }
+
+  /**
+   * Deletes the given nodes from the vector store.
+   * @param refDocId Node ID
+   */
+  async delete(refDocId: string): Promise<void> {
+    const mustFilter = [
+      {
+        key: "doc_id",
+        match: {
+          value: refDocId,
+        },
+      },
+    ];
+
+    await this.db.delete(this.collectionName, {
+      filter: {
+        must: mustFilter,
+      },
+    });
+  }
+
+  /**
+   * Converts the result of a query to a VectorStoreQueryResult.
+   * @param response Query response
+   * @returns VectorStoreQueryResult
+   */
+  private parseToQueryResult(
+    response: Array<QuerySearchResult>,
+  ): VectorStoreQueryResult {
+    const nodes = [];
+    const similarities = [];
+    const ids = [];
+
+    for (let i = 0; i < response.length; i++) {
+      const item = response[i];
+      const payload = item.payload;
+
+      const node = metadataDictToNode(payload);
+
+      ids.push(item.id);
+      nodes.push(node);
+      similarities.push(item.score);
+    }
+
+    return {
+      nodes: nodes,
+      similarities: similarities,
+      ids: ids,
+    };
+  }
+
+  /**
+   * Queries the vector store for the closest matching data to the query embeddings.
+   * @param query The VectorStoreQuery to be used
+   * @param options Required by VectorStore interface.  Currently ignored.
+   * @returns Zero or more Document instances with data from the vector store.
+   */
+  async query(
+    query: VectorStoreQuery,
+    options?: any,
+  ): Promise<VectorStoreQueryResult> {
+    const qdrantFilters = options?.qdrant_filters;
+
+    let queryFilters;
+
+    if (!query.queryEmbedding) {
+      throw new Error("No query embedding provided");
+    }
+
+    if (qdrantFilters) {
+      queryFilters = qdrantFilters;
+    } else {
+      queryFilters = await this.buildQueryFilter(query);
+    }
+
+    const result = (await this.db.search(this.collectionName, {
+      vector: query.queryEmbedding,
+      limit: query.similarityTopK,
+      ...(queryFilters && { filter: queryFilters }),
+    })) as Array<QuerySearchResult>;
+
+    return this.parseToQueryResult(result);
+  }
+
+  /**
+   * Qdrant filter builder
+   * @param query The VectorStoreQuery to be used
+   */
+  private async buildQueryFilter(query: VectorStoreQuery) {
+    if (!query.docIds && !query.queryStr) {
+      return null;
+    }
+
+    const mustConditions = [];
+
+    if (query.docIds) {
+      mustConditions.push({
+        key: "doc_id",
+        match: {
+          any: query.docIds,
+        },
+      });
+    }
+
+    if (!query.filters) {
+      return {
+        must: mustConditions,
+      };
+    }
+
+    const metadataFilters = query.filters.filters;
+
+    for (let i = 0; i < metadataFilters.length; i++) {
+      const filter = metadataFilters[i];
+
+      if (typeof filter.key === "number") {
+        mustConditions.push({
+          key: filter.key,
+          match: {
+            gt: filter.value,
+            lt: filter.value,
+          },
+        });
+      } else {
+        mustConditions.push({
+          key: filter.key,
+          match: {
+            value: filter.value,
+          },
+        });
+      }
+    }
+
+    return {
+      must: mustConditions,
+    };
+  }
+}

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -6,8 +6,9 @@ import {
   getTopKEmbeddingsLearner,
   getTopKMMREmbeddings,
 } from "../../embeddings";
+import { defaultFS } from "../../env";
 import { GenericFileSystem, exists } from "../FileSystem";
-import { DEFAULT_FS, DEFAULT_PERSIST_DIR } from "../constants";
+import { DEFAULT_PERSIST_DIR } from "../constants";
 import {
   VectorStore,
   VectorStoreQuery,
@@ -31,17 +32,17 @@ class SimpleVectorStoreData {
 export class SimpleVectorStore implements VectorStore {
   storesText: boolean = false;
   private data: SimpleVectorStoreData = new SimpleVectorStoreData();
-  private fs: GenericFileSystem = DEFAULT_FS;
+  private fs: GenericFileSystem = defaultFS;
   private persistPath: string | undefined;
 
   constructor(data?: SimpleVectorStoreData, fs?: GenericFileSystem) {
     this.data = data || new SimpleVectorStoreData();
-    this.fs = fs || DEFAULT_FS;
+    this.fs = fs || defaultFS;
   }
 
   static async fromPersistDir(
     persistDir: string = DEFAULT_PERSIST_DIR,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = defaultFS,
   ): Promise<SimpleVectorStore> {
     let persistPath = `${persistDir}/vector_store.json`;
     return await SimpleVectorStore.fromPersistPath(persistPath, fs);
@@ -160,7 +161,7 @@ export class SimpleVectorStore implements VectorStore {
     persistPath: string,
     fs?: GenericFileSystem,
   ): Promise<SimpleVectorStore> {
-    fs = fs || DEFAULT_FS;
+    fs = fs || defaultFS;
 
     let dirPath = path.dirname(persistPath);
     if (!(await exists(fs, dirPath))) {

packages/core/src/tests/GenericFileSystem.test.ts

@@ -1,10 +1,10 @@
+import nodeFS from "node:fs/promises";
 import os from "os";
 import path from "path";
 import {
   GenericFileSystem,
   InMemoryFileSystem,
   exists,
-  getNodeFS,
   walk,
 } from "../storage/FileSystem";
 
@@ -16,8 +16,6 @@ type FileSystemUnderTest = {
   tempDir: string;
 };
 
-const nodeFS = getNodeFS() as GenericFileSystem & any;
-
 describe.each<FileSystemUnderTest>([
   {
     name: "InMemoryFileSystem",
@@ -102,14 +100,13 @@ describe.each<FileSystemUnderTest>([
 });
 
 describe("Test walk for Node.js fs", () => {
-  const fs = getNodeFS();
   let tempDir: string;
 
   beforeAll(async () => {
     tempDir = await nodeFS.mkdtemp(path.join(os.tmpdir(), "jest-"));
-    await fs.writeFile(`${tempDir}/test.txt`, "Hello, world!");
-    await fs.mkdir(`${tempDir}/subDir`);
-    await fs.writeFile(`${tempDir}/subDir/test2.txt`, "Hello, again!");
+    await nodeFS.writeFile(`${tempDir}/test.txt`, "Hello, world!");
+    await nodeFS.mkdir(`${tempDir}/subDir`);
+    await nodeFS.writeFile(`${tempDir}/subDir/test2.txt`, "Hello, again!");
   });
 
   it("walks directory", async () => {
@@ -119,7 +116,7 @@ describe("Test walk for Node.js fs", () => {
     ]);
 
     const actualFiles = new Set<string>();
-    for await (let file of walk(fs, tempDir)) {
+    for await (let file of walk(nodeFS, tempDir)) {
       expect(file).toBeTruthy();
       actualFiles.add(file);
     }

packages/core/src/tests/MetadataExtractors.test.ts

@@ -0,0 +1,138 @@
+import { Document } from "../Node";
+import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext";
+import {
+  CallbackManager,
+  RetrievalCallbackResponse,
+  StreamCallbackResponse,
+} from "../callbacks/CallbackManager";
+import { OpenAIEmbedding } from "../embeddings";
+import {
+  KeywordExtractor,
+  QuestionsAnsweredExtractor,
+  SummaryExtractor,
+  TitleExtractor,
+} from "../extractors";
+import { OpenAI } from "../llm/LLM";
+import { SimpleNodeParser } from "../nodeParsers";
+import {
+  DEFAULT_LLM_TEXT_OUTPUT,
+  mockEmbeddingModel,
+  mockLlmGeneration,
+} from "./utility/mockOpenAI";
+
+// Mock the OpenAI getOpenAISession function during testing
+jest.mock("../llm/openai", () => {
+  return {
+    getOpenAISession: jest.fn().mockImplementation(() => null),
+  };
+});
+
+describe("[MetadataExtractor]: Extractors should populate the metadata", () => {
+  let serviceContext: ServiceContext;
+  let streamCallbackData: StreamCallbackResponse[] = [];
+  let retrieveCallbackData: RetrievalCallbackResponse[] = [];
+
+  beforeAll(async () => {
+    const callbackManager = new CallbackManager({
+      onLLMStream: (data) => {
+        streamCallbackData.push(data);
+      },
+      onRetrieve: (data) => {
+        retrieveCallbackData.push(data);
+      },
+    });
+
+    const languageModel = new OpenAI({
+      model: "gpt-3.5-turbo",
+      callbackManager,
+    });
+
+    mockLlmGeneration({ languageModel, callbackManager });
+
+    const embedModel = new OpenAIEmbedding();
+    mockEmbeddingModel(embedModel);
+
+    serviceContext = serviceContextFromDefaults({
+      callbackManager,
+      llm: languageModel,
+      embedModel,
+    });
+  });
+
+  beforeEach(() => {
+    streamCallbackData = [];
+    retrieveCallbackData = [];
+  });
+
+  afterAll(() => {
+    jest.clearAllMocks();
+  });
+
+  test("[MetadataExtractor] KeywordExtractor returns excerptKeywords metadata", async () => {
+    const nodeParser = new SimpleNodeParser();
+
+    const nodes = nodeParser.getNodesFromDocuments([
+      new Document({ text: DEFAULT_LLM_TEXT_OUTPUT }),
+    ]);
+
+    const keywordExtractor = new KeywordExtractor(serviceContext.llm, 5);
+
+    const nodesWithKeywordMetadata = await keywordExtractor.processNodes(nodes);
+
+    expect(nodesWithKeywordMetadata[0].metadata).toMatchObject({
+      excerptKeywords: DEFAULT_LLM_TEXT_OUTPUT,
+    });
+  });
+
+  test("[MetadataExtractor] TitleExtractor returns documentTitle metadata", async () => {
+    const nodeParser = new SimpleNodeParser();
+
+    const nodes = nodeParser.getNodesFromDocuments([
+      new Document({ text: DEFAULT_LLM_TEXT_OUTPUT }),
+    ]);
+
+    const titleExtractor = new TitleExtractor(serviceContext.llm, 5);
+
+    const nodesWithKeywordMetadata = await titleExtractor.processNodes(nodes);
+
+    expect(nodesWithKeywordMetadata[0].metadata).toMatchObject({
+      documentTitle: DEFAULT_LLM_TEXT_OUTPUT,
+    });
+  });
+
+  test("[MetadataExtractor] QuestionsAnsweredExtractor returns questionsThisExcerptCanAnswer metadata", async () => {
+    const nodeParser = new SimpleNodeParser();
+
+    const nodes = nodeParser.getNodesFromDocuments([
+      new Document({ text: DEFAULT_LLM_TEXT_OUTPUT }),
+    ]);
+
+    const questionsAnsweredExtractor = new QuestionsAnsweredExtractor(
+      serviceContext.llm,
+      5,
+    );
+
+    const nodesWithKeywordMetadata =
+      await questionsAnsweredExtractor.processNodes(nodes);
+
+    expect(nodesWithKeywordMetadata[0].metadata).toMatchObject({
+      questionsThisExcerptCanAnswer: DEFAULT_LLM_TEXT_OUTPUT,
+    });
+  });
+
+  test("[MetadataExtractor] SumamryExtractor returns sectionSummary metadata", async () => {
+    const nodeParser = new SimpleNodeParser();
+
+    const nodes = nodeParser.getNodesFromDocuments([
+      new Document({ text: DEFAULT_LLM_TEXT_OUTPUT }),
+    ]);
+
+    const summaryExtractor = new SummaryExtractor(serviceContext.llm);
+
+    const nodesWithKeywordMetadata = await summaryExtractor.processNodes(nodes);
+
+    expect(nodesWithKeywordMetadata[0].metadata).toMatchObject({
+      sectionSummary: DEFAULT_LLM_TEXT_OUTPUT,
+    });
+  });
+});

packages/core/src/tests/Node.test.ts

@@ -7,9 +7,13 @@ describe("TextNode", () => {
     node = new TextNode({ text: "Hello World" });
   });
 
-  describe("generateHash", () => {
-    it("should generate a hash", () => {
-      expect(node.hash).toBe("nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do=");
-    });
+  test("should generate a hash", () => {
+    expect(node.hash).toBe("nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do=");
+  });
+
+  test("clone should have the same hash", () => {
+    const hash = node.hash;
+    const clone = node.clone();
+    expect(clone.hash).toBe(hash);
   });
 });

packages/core/src/tests/mocks/TestableQdrantVectorStore.ts

@@ -0,0 +1,20 @@
+import { BaseNode } from "../../Node";
+import { QdrantVectorStore } from "../../storage";
+
+export class TestableQdrantVectorStore extends QdrantVectorStore {
+  public nodes: BaseNode[] = [];
+
+  public add(nodes: BaseNode[]): Promise<string[]> {
+    this.nodes.push(...nodes);
+    return super.add(nodes);
+  }
+
+  public delete(refDocId: string): Promise<void> {
+    this.nodes = this.nodes.filter((node) => node.id_ !== refDocId);
+    return super.delete(refDocId);
+  }
+
+  public getNodes(): BaseNode[] {
+    return this.nodes;
+  }
+}

packages/core/src/tests/utility/mockOpenAI.ts

@@ -4,6 +4,8 @@ import { globalsHelper } from "../../GlobalsHelper";
 import { OpenAI } from "../../llm/LLM";
 import { LLMChatParamsBase } from "../../llm/types";
 
+export const DEFAULT_LLM_TEXT_OUTPUT = "MOCK_TOKEN_1-MOCK_TOKEN_2";
+
 export function mockLlmGeneration({
   languageModel,
   callbackManager,
@@ -15,7 +17,7 @@ export function mockLlmGeneration({
     .spyOn(languageModel, "chat")
     .mockImplementation(
       async ({ messages, parentEvent }: LLMChatParamsBase) => {
-        const text = "MOCK_TOKEN_1-MOCK_TOKEN_2";
+        const text = DEFAULT_LLM_TEXT_OUTPUT;
         const event = globalsHelper.createEvent({
           parentEvent,
           type: "llmPredict",

packages/core/src/tests/vectorStores/QdrantVectorStore.test.ts

@@ -0,0 +1,135 @@
+import { BaseNode, TextNode } from "../../Node";
+
+import { QdrantClient } from "@qdrant/js-client-rest";
+import { VectorStoreQueryMode } from "../../storage";
+import { TestableQdrantVectorStore } from "../mocks/TestableQdrantVectorStore";
+
+jest.mock("@qdrant/js-client-rest");
+
+describe("QdrantVectorStore", () => {
+  let store: TestableQdrantVectorStore;
+  let mockQdrantClient: jest.Mocked<QdrantClient>;
+
+  beforeEach(() => {
+    mockQdrantClient = new QdrantClient() as jest.Mocked<QdrantClient>;
+    store = new TestableQdrantVectorStore({
+      client: mockQdrantClient,
+      collectionName: "testCollection",
+      url: "http://example.com",
+      apiKey: "testApiKey",
+      batchSize: 100,
+    });
+  });
+
+  describe("[QdrantVectorStore] createCollection", () => {
+    it("should create a new collection", async () => {
+      mockQdrantClient.createCollection.mockResolvedValue(true);
+
+      await store.createCollection("testCollection", 128);
+
+      expect(mockQdrantClient.createCollection).toHaveBeenCalledWith(
+        "testCollection",
+        {
+          vectors: {
+            size: 128,
+            distance: "Cosine",
+          },
+        },
+      );
+    });
+
+    describe("[QdrantVectorStore] add", () => {
+      it("should add nodes to the vector store", async () => {
+        // Mocking the dependent methods and Qdrant client responses
+        const mockInitializeCollection = jest
+          .spyOn(store, "initializeCollection")
+          .mockResolvedValue();
+
+        const mockBuildPoints = jest
+          .spyOn(store, "buildPoints")
+          .mockResolvedValue({
+            points: [{ id: "1", payload: {}, vector: [0.1, 0.2] }],
+            ids: ["1"],
+          });
+
+        mockQdrantClient.upsert.mockResolvedValue({
+          operation_id: 1,
+          status: "completed",
+        });
+
+        const nodes: BaseNode[] = [
+          new TextNode({
+            embedding: [0.1, 0.2],
+            metadata: { meta1: "Some metadata" },
+          }),
+        ];
+
+        const ids = await store.add(nodes);
+
+        expect(mockInitializeCollection).toHaveBeenCalledWith(
+          nodes[0].getEmbedding().length,
+        );
+        expect(mockBuildPoints).toHaveBeenCalledWith(nodes);
+        expect(mockQdrantClient.upsert).toHaveBeenCalled();
+
+        expect(ids).toEqual(["1"]);
+      });
+    });
+
+    describe("[QdrantVectorStore] delete", () => {
+      it("should delete from the vector store", async () => {
+        jest.spyOn(store, "initializeCollection").mockResolvedValue();
+
+        jest.spyOn(store, "buildPoints").mockResolvedValue({
+          points: [{ id: "1", payload: {}, vector: [0.1, 0.2] }],
+          ids: ["1"],
+        });
+
+        mockQdrantClient.upsert.mockResolvedValue({
+          operation_id: 1,
+          status: "completed",
+        });
+
+        const nodes: BaseNode[] = [
+          new TextNode({
+            id_: "1",
+            embedding: [0.1, 0.2],
+            metadata: { meta1: "Some metadata" },
+          }),
+        ];
+
+        await store.add(nodes);
+
+        expect(store.getNodes()).toContain(nodes[0]);
+
+        await store.delete("1");
+
+        expect(store.getNodes()).not.toContain(nodes[0]);
+        expect(mockQdrantClient.upsert).toHaveBeenCalled();
+      });
+    });
+
+    describe("[QdrantVectorStore] search", () => {
+      it("should search in the vector store", async () => {
+        mockQdrantClient.search.mockResolvedValue([
+          {
+            id: "1",
+            score: 0.1,
+            version: 1,
+            payload: { _node_content: JSON.stringify({ text: "hello world" }) },
+          },
+        ]);
+
+        const searchResult = await store.query({
+          queryEmbedding: [0.1, 0.2],
+          similarityTopK: 1,
+          mode: VectorStoreQueryMode.DEFAULT,
+        });
+
+        expect(mockQdrantClient.search).toHaveBeenCalled();
+        expect(searchResult.ids).toEqual(["1"]);
+        expect(searchResult.similarities).toEqual([0.1]);
+      });
+    });
+  });
+});

packages/core/tsconfig.json

@@ -12,8 +12,8 @@
     "strict": true,
     "lib": ["es2015", "dom"],
     "target": "ES2015",
-    "resolveJsonModule": true
+    "resolveJsonModule": true,
   },
   "include": ["./src"],
-  "exclude": ["node_modules"]
+  "exclude": ["node_modules"],
 }

packages/create-llama/create-app.ts

@@ -34,7 +34,7 @@ export async function createApp({
   communityProjectPath,
   vectorDb,
   externalPort,
-  installDependencies,
+  postInstallAction,
 }: InstallAppArgs): Promise<void> {
   const root = path.resolve(appPath);
 
@@ -76,7 +76,7 @@ export async function createApp({
     communityProjectPath,
     vectorDb,
     externalPort,
-    installDependencies,
+    postInstallAction,
   };
 
   if (frontend) {

packages/create-llama/e2e/basic.spec.ts

@@ -9,7 +9,7 @@ import type {
   TemplateType,
   TemplateUI,
 } from "../helpers";
-import { createTestDir, runApp, runCreateLlama, type AppType } from "./utils";
+import { createTestDir, runCreateLlama, type AppType } from "./utils";
 
 const templateTypes: TemplateType[] = ["streaming", "simple"];
 const templateFrameworks: TemplateFramework[] = [
@@ -47,27 +47,26 @@ for (const templateType of templateTypes) {
           let externalPort: number;
           let cwd: string;
           let name: string;
-          let cps: ChildProcess[] = [];
+          let appProcess: ChildProcess;
+          const postInstallAction = "runApp";
 
           test.beforeAll(async () => {
             port = Math.floor(Math.random() * 10000) + 10000;
             externalPort = port + 1;
-
             cwd = await createTestDir();
-            name = runCreateLlama(
+            const result = await runCreateLlama(
               cwd,
               templateType,
               templateFramework,
               templateEngine,
               templateUI,
               appType,
+              port,
               externalPort,
+              postInstallAction,
             );
-
-            if (templateFramework !== "fastapi") {
-              // don't run the app for fastapi for now (adds python dependency)
-              cps = await runApp(cwd, name, appType, port, externalPort);
-            }
+            name = result.projectName;
+            appProcess = result.appProcess;
           });
 
           test("App folder should exist", async () => {
@@ -75,9 +74,7 @@ for (const templateType of templateTypes) {
             expect(dirExists).toBeTruthy();
           });
           test("Frontend should have a title", async ({ page }) => {
-            test.skip(
-              appType === "--no-frontend" || templateFramework === "fastapi",
-            );
+            test.skip(appType === "--no-frontend");
             await page.goto(`http://localhost:${port}`);
             await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
           });
@@ -85,9 +82,7 @@ for (const templateType of templateTypes) {
           test("Frontend should be able to submit a message and receive a response", async ({
             page,
           }) => {
-            test.skip(
-              appType === "--no-frontend" || templateFramework === "fastapi",
-            );
+            test.skip(appType === "--no-frontend");
             await page.goto(`http://localhost:${port}`);
             await page.fill("form input", "hello");
             await page.click("form button[type=submit]");
@@ -107,11 +102,10 @@ for (const templateType of templateTypes) {
           test("Backend should response when calling API", async ({
             request,
           }) => {
-            test.skip(
-              appType !== "--no-frontend" || templateFramework === "fastapi",
-            );
+            test.skip(appType !== "--no-frontend");
+            const backendPort = appType === "" ? port : externalPort;
             const response = await request.post(
-              `http://localhost:${port}/api/chat`,
+              `http://localhost:${backendPort}/api/chat`,
               {
                 data: {
                   messages: [
@@ -130,7 +124,7 @@ for (const templateType of templateTypes) {
 
           // clean processes
           test.afterAll(async () => {
-            cps.map((cp) => cp.kill());
+            appProcess?.kill();
           });
         });
       }

packages/create-llama/e2e/tsconfig.json

@@ -1,12 +1,12 @@
 {
   "extends": "../../../tsconfig.json",
   "compilerOptions": {
-    "tsBuildInfoFile": "./lib/.e2e.tsbuildinfo"
+    "tsBuildInfoFile": "./lib/.e2e.tsbuildinfo",
   },
   "include": ["./**/*.ts"],
   "references": [
     {
-      "path": ".."
-    }
-  ]
+      "path": "..",
+    },
+  ],
 }

packages/create-llama/e2e/utils.ts

@@ -1,81 +1,77 @@
-import { ChildProcess, exec, execSync } from "child_process";
+import { ChildProcess, exec } from "child_process";
 import crypto from "node:crypto";
 import { mkdir } from "node:fs/promises";
 import * as path from "path";
 import waitPort from "wait-port";
+import {
+  TemplateEngine,
+  TemplateFramework,
+  TemplatePostInstallAction,
+  TemplateType,
+  TemplateUI,
+} from "../helpers";
 
 export type AppType = "--frontend" | "--no-frontend" | "";
 const MODEL = "gpt-3.5-turbo";
+export type CreateLlamaResult = {
+  projectName: string;
+  appProcess: ChildProcess;
+};
 
 // eslint-disable-next-line max-params
-export async function runApp(
+export async function checkAppHasStarted(
+  frontend: boolean,
+  framework: TemplateFramework,
+  port: number,
+  externalPort: number,
+  timeout: number,
+) {
+  if (frontend) {
+    await Promise.all([
+      waitPort({
+        host: "localhost",
+        port: port,
+        timeout,
+      }),
+      waitPort({
+        host: "localhost",
+        port: externalPort,
+        timeout,
+      }),
+    ]).catch((err) => {
+      console.error(err);
+      throw err;
+    });
+  } else {
+    let wPort: number;
+    if (framework === "nextjs") {
+      wPort = port;
+    } else {
+      wPort = externalPort;
+    }
+    await waitPort({
+      host: "localhost",
+      port: wPort,
+      timeout,
+    }).catch((err) => {
+      console.error(err);
+      throw err;
+    });
+  }
+}
+
+// eslint-disable-next-line max-params
+export async function runCreateLlama(
   cwd: string,
-  name: string,
+  templateType: TemplateType,
+  templateFramework: TemplateFramework,
+  templateEngine: TemplateEngine,
+  templateUI: TemplateUI,
   appType: AppType,
   port: number,
   externalPort: number,
-): Promise<ChildProcess[]> {
-  const cps: ChildProcess[] = [];
-
-  try {
-    switch (appType) {
-      case "--frontend":
-        cps.push(
-          await createProcess(
-            "npm run dev",
-            path.join(cwd, name, "backend"),
-            externalPort,
-          ),
-        );
-        cps.push(
-          await createProcess(
-            "npm run dev",
-            path.join(cwd, name, "frontend"),
-            port,
-          ),
-        );
-        break;
-      default:
-        cps.push(
-          await createProcess("npm run dev", path.join(cwd, name), port),
-        );
-        break;
-    }
-  } catch (e) {
-    cps.forEach((cp) => cp.kill());
-    throw e;
-  }
-  return cps;
-}
-
-async function createProcess(command: string, cwd: string, port: number) {
-  const cp = exec(command, {
-    cwd,
-    env: {
-      ...process.env,
-      PORT: `${port}`,
-    },
-  });
-  if (!cp) throw new Error(`Can't start process ${command} in ${cwd}`);
-
-  await waitPort({
-    host: "localhost",
-    port,
-    timeout: 1000 * 60,
-  });
-  return cp;
-}
-
-// eslint-disable-next-line max-params
-export function runCreateLlama(
-  cwd: string,
-  templateType: string,
-  templateFramework: string,
-  templateEngine: string,
-  templateUI: string,
-  appType: AppType,
-  externalPort: number,
-) {
+  postInstallAction: TemplatePostInstallAction,
+): Promise<CreateLlamaResult> {
   const createLlama = path.join(__dirname, "..", "dist", "index.js");
 
   const name = [
@@ -104,17 +100,46 @@ export function runCreateLlama(
     appType,
     "--eslint",
     "--use-npm",
+    "--port",
+    port,
     "--external-port",
     externalPort,
-    "--install-dependencies",
+    "--post-install-action",
+    postInstallAction,
   ].join(" ");
   console.log(`running command '${command}' in ${cwd}`);
-  execSync(command, {
-    stdio: "inherit",
+  let appProcess = exec(command, {
     cwd,
+    env: {
+      ...process.env,
+    },
   });
-  return name;
+  appProcess.stderr?.on("data", (data) => {
+    console.log(data.toString());
+  });
+  appProcess.on("exit", (code) => {
+    if (code !== 0 && code !== null) {
+      throw new Error(`create-llama command was failed!`);
+    }
+  });
+
+  // Wait for app to start
+  if (postInstallAction === "runApp") {
+    await checkAppHasStarted(
+      appType === "--frontend",
+      templateFramework,
+      port,
+      externalPort,
+      1000 * 60 * 5,
+    );
+  }
+
+  return {
+    projectName: name,
+    appProcess,
+  };
 }
+
 export async function createTestDir() {
   const cwd = path.join(__dirname, ".cache", crypto.randomUUID());
   await mkdir(cwd, { recursive: true });

packages/create-llama/helpers/python.ts

@@ -1,6 +1,6 @@
 import fs from "fs/promises";
 import path from "path";
-import { cyan, yellow } from "picocolors";
+import { cyan, red, yellow } from "picocolors";
 import { parse, stringify } from "smol-toml";
 import terminalLink from "terminal-link";
 import { copy } from "./copy";
@@ -92,13 +92,39 @@ export const addDependencies = async (
   }
 };
 
+export const installPythonDependencies = (root: string) => {
+  if (isPoetryAvailable()) {
+    console.log(
+      `Installing python dependencies using poetry. This may take a while...`,
+    );
+    const installSuccessful = tryPoetryInstall();
+    if (!installSuccessful) {
+      console.error(
+        red("Install failed. Please install dependencies manually."),
+      );
+      process.exit(1);
+    }
+  } else {
+    console.warn(
+      yellow(
+        `Poetry is not available in the current environment. The Python dependencies will not be installed automatically.
+Please check ${terminalLink(
+          "Poetry Installation",
+          `https://python-poetry.org/docs/#installation`,
+        )} to install poetry first, then install the dependencies manually.`,
+      ),
+    );
+    process.exit(1);
+  }
+};
+
 export const installPythonTemplate = async ({
   root,
   template,
   framework,
   engine,
   vectorDb,
-  installDependencies,
+  postInstallAction,
 }: Pick<
   InstallTemplateArgs,
   | "root"
@@ -106,7 +132,7 @@ export const installPythonTemplate = async ({
   | "template"
   | "engine"
   | "vectorDb"
-  | "installDependencies"
+  | "postInstallAction"
 >) => {
   console.log("\nInitializing Python project with template:", template, "\n");
   const templatePath = path.join(
@@ -154,28 +180,7 @@ export const installPythonTemplate = async ({
   const addOnDependencies = getAdditionalDependencies(vectorDb);
   await addDependencies(root, addOnDependencies);
 
-  // install python dependencies
-  if (installDependencies) {
-    if (isPoetryAvailable()) {
-      console.log(
-        `Installing python dependencies using poetry. This may take a while...`,
-      );
-      const installSuccessful = tryPoetryInstall();
-      if (!installSuccessful) {
-        console.warn(
-          yellow("Install failed. Please install dependencies manually."),
-        );
-      }
-    } else {
-      console.warn(
-        yellow(
-          `Poetry is not available in the current environment. The Python dependencies will not be installed automatically.
-Please check ${terminalLink(
-            "Poetry Installation",
-            `https://python-poetry.org/docs/#installation`,
-          )} to install poetry first, then install the dependencies manually.`,
-        ),
-      );
-    }
+  if (postInstallAction !== "none") {
+    installPythonDependencies(root);
   }
 };

packages/create-llama/helpers/run-app.ts

@@ -0,0 +1,88 @@
+import { ChildProcess, SpawnOptions, spawn } from "child_process";
+import path from "path";
+import { TemplateFramework } from "./types";
+
+const createProcess = (
+  command: string,
+  args: string[],
+  options: SpawnOptions,
+) => {
+  return spawn(command, args, {
+    ...options,
+    shell: true,
+  })
+    .on("exit", function (code) {
+      if (code !== 0) {
+        console.log(`Child process exited with code=${code}`);
+        process.exit(1);
+      }
+    })
+    .on("error", function (err) {
+      console.log("Error when running chill process: ", err);
+      process.exit(1);
+    });
+};
+
+// eslint-disable-next-line max-params
+export async function runApp(
+  appPath: string,
+  frontend: boolean,
+  framework: TemplateFramework,
+  port?: number,
+  externalPort?: number,
+): Promise<any> {
+  let backendAppProcess: ChildProcess;
+  let frontendAppProcess: ChildProcess | undefined;
+  let frontendPort = port || 3000;
+  let backendPort = externalPort || 8000;
+
+  // Callback to kill app processes
+  process.on("exit", () => {
+    console.log("Killing app processes...");
+    backendAppProcess.kill();
+    frontendAppProcess?.kill();
+  });
+
+  let backendCommand = "";
+  let backendArgs: string[];
+  if (framework === "fastapi") {
+    backendCommand = "poetry";
+    backendArgs = [
+      "run",
+      "uvicorn",
+      "main:app",
+      "--host=0.0.0.0",
+      "--port=" + backendPort,
+    ];
+  } else if (framework === "nextjs") {
+    backendCommand = "npm";
+    backendArgs = ["run", "dev"];
+    backendPort = frontendPort;
+  } else {
+    backendCommand = "npm";
+    backendArgs = ["run", "dev"];
+  }
+
+  if (frontend) {
+    return new Promise((resolve, reject) => {
+      backendAppProcess = createProcess(backendCommand, backendArgs, {
+        stdio: "inherit",
+        cwd: path.join(appPath, "backend"),
+        env: { ...process.env, PORT: `${backendPort}` },
+      });
+      frontendAppProcess = createProcess("npm", ["run", "dev"], {
+        stdio: "inherit",
+        cwd: path.join(appPath, "frontend"),
+        env: { ...process.env, PORT: `${frontendPort}` },
+      });
+    });
+  } else {
+    return new Promise((resolve, reject) => {
+      backendAppProcess = createProcess(backendCommand, backendArgs, {
+        stdio: "inherit",
+        cwd: path.join(appPath),
+        env: { ...process.env, PORT: `${backendPort}` },
+      });
+    });
+  }
+}

packages/create-llama/helpers/types.ts

@@ -5,6 +5,7 @@ export type TemplateFramework = "nextjs" | "express" | "fastapi";
 export type TemplateEngine = "simple" | "context";
 export type TemplateUI = "html" | "shadcn";
 export type TemplateVectorDB = "none" | "mongo" | "pg";
+export type TemplatePostInstallAction = "none" | "dependencies" | "runApp";
 
 export interface InstallTemplateArgs {
   appName: string;
@@ -23,5 +24,5 @@ export interface InstallTemplateArgs {
   communityProjectPath?: string;
   vectorDb?: TemplateVectorDB;
   externalPort?: number;
-  installDependencies?: boolean;
+  postInstallAction?: TemplatePostInstallAction;
 }

packages/create-llama/helpers/typescript.ts

@@ -5,6 +5,7 @@ import { bold, cyan } from "picocolors";
 import { version } from "../../core/package.json";
 import { copy } from "../helpers/copy";
 import { callPackageManager } from "../helpers/install";
+import { PackageManager } from "./get-pkg-manager";
 import { InstallTemplateArgs } from "./types";
 
 const rename = (name: string) => {
@@ -23,6 +24,28 @@ const rename = (name: string) => {
     }
   }
 };
+
+export const installTSDependencies = async (
+  packageJson: any,
+  packageManager: PackageManager,
+  isOnline: boolean,
+): Promise<void> => {
+  console.log("\nInstalling dependencies:");
+  for (const dependency in packageJson.dependencies)
+    console.log(`- ${cyan(dependency)}`);
+
+  console.log("\nInstalling devDependencies:");
+  for (const dependency in packageJson.devDependencies)
+    console.log(`- ${cyan(dependency)}`);
+
+  console.log();
+
+  await callPackageManager(packageManager, isOnline).catch((error) => {
+    console.error("Failed to install TS dependencies. Exiting...");
+    process.exit(1);
+  });
+};
+
 /**
  * Install a LlamaIndex internal template to a given `root` directory.
  */
@@ -39,7 +62,7 @@ export const installTSTemplate = async ({
   customApiPath,
   forBackend,
   vectorDb,
-  installDependencies,
+  postInstallAction,
 }: InstallTemplateArgs) => {
   console.log(bold(`Using ${packageManager}.`));
 
@@ -211,17 +234,7 @@ export const installTSTemplate = async ({
     JSON.stringify(packageJson, null, 2) + os.EOL,
   );
 
-  if (installDependencies) {
-    console.log("\nInstalling dependencies:");
-    for (const dependency in packageJson.dependencies)
-      console.log(`- ${cyan(dependency)}`);
-
-    console.log("\nInstalling devDependencies:");
-    for (const dependency in packageJson.devDependencies)
-      console.log(`- ${cyan(dependency)}`);
-
-    console.log();
-
-    await callPackageManager(packageManager, isOnline);
+  if (postInstallAction !== "none") {
+    await installTSDependencies(packageJson, packageManager, isOnline);
   }
 };

packages/create-llama/index.ts

@@ -10,6 +10,7 @@ import checkForUpdate from "update-check";
 import { createApp } from "./create-app";
 import { getPkgManager } from "./helpers/get-pkg-manager";
 import { isFolderEmpty } from "./helpers/is-folder-empty";
+import { runApp } from "./helpers/run-app";
 import { validateNpmName } from "./helpers/validate-pkg";
 import packageJson from "./package.json";
 import { QuestionArgs, askQuestions, onPromptState } from "./questions";
@@ -113,17 +114,24 @@ const program = new Commander.Command(packageJson.name)
 `,
   )
   .option(
-    "--external-port <external>",
+    "--port <port>",
     `
 
-Select external port.
+  Select UI port.
 `,
   )
   .option(
-    "--install-dependencies",
+    "--external-port <external>",
     `
 
-Whether install dependencies (backend/frontend) automatically or not.
+  Select external port.
+`,
+  )
+  .option(
+    "--post-install-action <action>",
+    `
+
+  Choose an action after installation. For example, 'runApp' or 'dependencies'. The default option is just to generate the app.
 `,
   )
   .allowUnknownOption()
@@ -231,9 +239,20 @@ async function run(): Promise<void> {
     communityProjectPath: program.communityProjectPath,
     vectorDb: program.vectorDb,
     externalPort: program.externalPort,
-    installDependencies: program.installDependencies,
+    postInstallAction: program.postInstallAction,
   });
   conf.set("preferences", preferences);
+
+  if (program.postInstallAction === "runApp") {
+    console.log("Running app...");
+    await runApp(
+      root,
+      program.frontend,
+      program.framework,
+      program.port,
+      program.externalPort,
+    );
+  }
 }
 
 const update = checkForUpdate(packageJson).catch(() => null);

packages/create-llama/package.json

@@ -26,7 +26,7 @@
     "build": "npm run clean && ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register",
     "lint": "eslint . --ignore-pattern dist",
     "e2e": "playwright test",
-    "prepublishOnly": "cd ../../ && turbo run build"
+    "prepublishOnly": "cd ../../ && pnpm run build:release"
   },
   "devDependencies": {
     "@playwright/test": "^1.40.0",

packages/create-llama/questions.ts

@@ -20,6 +20,7 @@ const defaults: QuestionArgs = {
   openAiKey: "",
   model: "gpt-3.5-turbo",
   communityProjectPath: "",
+  postInstallAction: "dependencies",
 };
 
 const handlers = {
@@ -39,9 +40,9 @@ const getVectorDbChoices = (framework: TemplateFramework) => {
     { title: "PostgreSQL", value: "pg" },
   ];
 
-  const vectodbLang = framework === "fastapi" ? "python" : "typescript";
+  const vectordbLang = framework === "fastapi" ? "python" : "typescript";
   const compPath = path.join(__dirname, "..", "templates", "components");
-  const vectordbPath = path.join(compPath, "vectordbs", vectodbLang);
+  const vectordbPath = path.join(compPath, "vectordbs", vectordbLang);
 
   const availableChoices = fs
     .readdirSync(vectordbPath)
@@ -211,19 +212,6 @@ export const askQuestions = async (
     }
   }
 
-  if (program.installDependencies === undefined) {
-    const { installDependencies } = await prompts({
-      onState: onPromptState,
-      type: "toggle",
-      name: "installDependencies",
-      message: `Would you like to install dependencies automatically? This may take a while`,
-      initial: getPrefOrDefault("installDependencies"),
-      active: "Yes",
-      inactive: "No",
-    });
-    program.installDependencies = Boolean(installDependencies);
-  }
-
   if (!program.model) {
     if (ciInfo.isCI) {
       program.model = getPrefOrDefault("model");
@@ -326,6 +314,46 @@ export const askQuestions = async (
     }
   }
 
+  // Ask for next action after installation
+  if (program.postInstallAction === undefined) {
+    if (ciInfo.isCI) {
+      program.postInstallAction = getPrefOrDefault("postInstallAction");
+    } else {
+      let actionChoices = [
+        {
+          title: "Just generate code (~1 sec)",
+          value: "none",
+        },
+        {
+          title: "Generate code and install dependencies (~2 min)",
+          value: "dependencies",
+        },
+      ];
+
+      const hasOpenAiKey = program.openAiKey || process.env["OPENAI_API_KEY"];
+      if (program.vectorDb === "none" && hasOpenAiKey) {
+        actionChoices.push({
+          title:
+            "Generate code, install dependencies, and run the app (~2 min)",
+          value: "runApp",
+        });
+      }
+
+      const { action } = await prompts(
+        {
+          type: "select",
+          name: "action",
+          message: "How would you like to proceed?",
+          choices: actionChoices,
+          initial: 1,
+        },
+        handlers,
+      );
+
+      program.postInstallAction = action;
+    }
+  }
+
   // TODO: consider using zod to validate the input (doesn't work like this as not every option is required)
   // templateUISchema.parse(program.ui);
   // templateEngineSchema.parse(program.engine);

packages/create-llama/templates/types/simple/express/package.json

@@ -14,6 +14,9 @@
     "express": "^4.18.2",
     "llamaindex": "0.0.37"
   },
+  "overrides": {
+    "chromadb": "1.7.3"
+  },
   "devDependencies": {
     "@types/cors": "^2.8.17",
     "@types/express": "^4.17.21",

packages/create-llama/templates/types/simple/express/tsconfig.json

@@ -5,6 +5,6 @@
     "forceConsistentCasingInFileNames": true,
     "strict": true,
     "skipLibCheck": true,
-    "moduleResolution": "node"
-  }
+    "moduleResolution": "node",
+  },
 }

packages/create-llama/templates/types/streaming/express/package.json

@@ -15,6 +15,9 @@
     "express": "^4.18.2",
     "llamaindex": "0.0.37"
   },
+  "overrides": {
+    "chromadb": "1.7.3"
+  },
   "devDependencies": {
     "@types/cors": "^2.8.16",
     "@types/express": "^4.17.21",