Merge pull request #170 from Swimburger/assemblyai

Add AssemblyAI integration

.changeset/fluffy-spiders-turn.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Give HistoryChatEngine pluggable options (thanks @marcusschiesser)

.changeset/ninety-fans-admire.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Add SimilarityPostProcessor (thanks @TomPenguin)

.changeset/pretty-points-kneel.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Added LLMMetadata (thanks @marcusschiesser)

.github/ISSUE_TEMPLATE/bugfix.yml

@@ -1,7 +1,6 @@
 name: Bugfix
-title: "Sweep: "
+title: ""
 description: Write something like "We notice ... behavior when ... happens instead of ...""
-labels: sweep
 body:
   - type: textarea
     id: description

.github/ISSUE_TEMPLATE/feature.yml

@@ -1,11 +1,10 @@
 name: Feature Request
-title: "Sweep: "
-description: Write something like "Write an api endpoint that does "..." in the "..." file"
-labels: sweep
+title: ""
+description: Write something like "Write an api endpoint that does "..." in the "..." file". If you would like to use sweep.dev prefix with "Sweep:"
 body:
   - type: textarea
     id: description
     attributes:
       label: Details
-      description: More details for Sweep
+      description: More details
       placeholder: The new endpoint should use the ... class from ... file because it contains ... logic

.github/ISSUE_TEMPLATE/refactor.yml

@@ -1,11 +1,10 @@
 name: Refactor
-title: "Sweep: "
-description: Write something like "Modify the ... api endpoint to use ... version and ... framework"
-labels: sweep
+title: ""
+description: Write something like "Modify the ... api endpoint to use ... version and ... framework" If you would like to use sweep.dev prefix with "Sweep:"
 body:
   - type: textarea
     id: description
     attributes:
       label: Details
-      description: More details for Sweep
+      description: More details
       placeholder: We are migrating this function to ... version because ...

.gitignore

@@ -3,6 +3,7 @@
 # dependencies
 node_modules
 .pnp
+.pnpm-store
 .pnp.js
 
 # testing
@@ -36,3 +37,6 @@ yarn-error.log*
 .vercel
 
 dist/
+
+# vs code
+.vscode/launch.json

.husky/pre-commit

@@ -2,3 +2,4 @@
 . "$(dirname -- "$0")/_/husky.sh"
 
 pnpm lint
+npx lint-staged

.vscode/settings.json

@@ -4,5 +4,6 @@
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "[xml]": {
     "editor.defaultFormatter": "redhat.vscode-xml"
-  }
-}
+  },
+  "jest.rootPath": "./packages/core"
+}

README.md

@@ -89,7 +89,7 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
 If you're using NextJS App Router, you'll need to use the NodeJS runtime (default) and add the follow config to your next.config.js to have it use imports/exports in the same way Node does.
 
 ```js
-export const runtime = "nodejs" // default
+export const runtime = "nodejs"; // default
 ```
 
 ```js

apps/docs/docs/end_to_end.md

@@ -6,6 +6,8 @@ sidebar_position: 4
 
 We include several end-to-end examples using LlamaIndex.TS in the repository
 
+Check out the examples below or try them out and complete them in minutes with interactive Github Codespace tutorials provided by Dev-Docs [here](https://codespaces.new/team-dev-docs/lits-dev-docs-playground?devcontainer_path=.devcontainer%2Fjavascript_ltsquickstart%2Fdevcontainer.json):
+
 ## [Chat Engine](https://github.com/run-llama/LlamaIndexTS/blob/main/apps/simple/chatEngine.ts)
 
 Read a file and chat about it with the LLM.

apps/docs/docs/environments.md

@@ -11,7 +11,7 @@ LlamaIndex currently officially supports NodeJS 18 and NodeJS 20.
 If you're using NextJS App Router route handlers/serverless functions, you'll need to use the NodeJS mode:
 
 ```js
-export const runtime = "nodejs" // default
+export const runtime = "nodejs"; // default
 ```
 
 and you'll need to add an exception for pdf-parse in your next.config.js

apps/mongodb/1_import.ts

@@ -0,0 +1,34 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import * as dotenv from "dotenv";
+import * as fs from "fs";
+import { MongoClient } from "mongodb";
+
+// Load environment variables from local .env file
+dotenv.config();
+
+const jsonFile = "tinytweets.json";
+const mongoUri = process.env.MONGODB_URI!;
+const databaseName = process.env.MONGODB_DATABASE!;
+const collectionName = process.env.MONGODB_COLLECTION!;
+
+async function importJsonToMongo() {
+  // Load the tweets from a local file
+  const tweets = JSON.parse(fs.readFileSync(jsonFile, "utf-8"));
+
+  // Create a new client and connect to the server
+  const client = new MongoClient(mongoUri);
+
+  const db = client.db(databaseName);
+  const collection = db.collection(collectionName);
+
+  // Insert the tweets into mongo
+  await collection.insertMany(tweets);
+
+  console.log(
+    `Data imported successfully to the MongoDB collection ${collectionName}.`,
+  );
+  await client.close();
+}
+
+// Run the import function
+importJsonToMongo();

apps/mongodb/2_load_and_index.ts

@@ -0,0 +1,50 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import * as dotenv from "dotenv";
+import {
+  MongoDBAtlasVectorSearch,
+  SimpleMongoReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { MongoClient } from "mongodb";
+
+// Load environment variables from local .env file
+dotenv.config();
+
+const mongoUri = process.env.MONGODB_URI!;
+const databaseName = process.env.MONGODB_DATABASE!;
+const collectionName = process.env.MONGODB_COLLECTION!;
+const vectorCollectionName = process.env.MONGODB_VECTORS!;
+const indexName = process.env.MONGODB_VECTOR_INDEX!;
+
+async function loadAndIndex() {
+  // Create a new client and connect to the server
+  const client = new MongoClient(mongoUri);
+  // load objects from mongo and convert them into LlamaIndex Document objects
+  // llamaindex has a special class that does this for you
+  // it pulls every object in a given collection
+  const reader = new SimpleMongoReader(client);
+  const documents = await reader.loadData(databaseName, collectionName, [
+    "full_text",
+  ]);
+
+  // create Atlas as a vector store
+  const vectorStore = new MongoDBAtlasVectorSearch({
+    mongodbClient: client,
+    dbName: databaseName,
+    collectionName: vectorCollectionName, // this is where your embeddings will be stored
+    indexName: indexName, // this is the name of the index you will need to create
+  });
+
+  // now create an index from all the Documents and store them in Atlas
+  const storageContext = await storageContextFromDefaults({ vectorStore });
+  await VectorStoreIndex.fromDocuments(documents, { storageContext });
+  console.log(
+    `Successfully created embeddings in the MongoDB collection ${vectorCollectionName}.`,
+  );
+  await client.close();
+}
+
+loadAndIndex();
+
+// you can't query your index yet because you need to create a vector search index in mongodb's UI now

apps/mongodb/3_query.ts

@@ -0,0 +1,34 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import * as dotenv from "dotenv";
+import {
+  MongoDBAtlasVectorSearch,
+  serviceContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { MongoClient } from "mongodb";
+
+// Load environment variables from local .env file
+dotenv.config();
+
+async function query() {
+  const client = new MongoClient(process.env.MONGODB_URI!);
+  const serviceContext = serviceContextFromDefaults();
+  const store = new MongoDBAtlasVectorSearch({
+    mongodbClient: client,
+    dbName: process.env.MONGODB_DATABASE!,
+    collectionName: process.env.MONGODB_VECTORS!,
+    indexName: process.env.MONGODB_VECTOR_INDEX!,
+  });
+
+  const index = await VectorStoreIndex.fromVectorStore(store, serviceContext);
+
+  const retriever = index.asRetriever({ similarityTopK: 20 });
+  const queryEngine = index.asQueryEngine({ retriever });
+  const result = await queryEngine.query(
+    "What does the author think of web frameworks?",
+  );
+  console.log(result.response);
+  await client.close();
+}
+
+query();

apps/mongodb/README.md

@@ -0,0 +1,127 @@
+# LlamaIndexTS retrieval augmented generation with MongoDB
+
+### Prepare Environment
+
+Make sure to run `pnpm install` and set your OpenAI environment variable before running these examples.
+
+```
+pnpm install
+export OPENAI_API_KEY="sk-..."
+```
+
+### Sign up for MongoDB Atlas
+
+We'll be using MongoDB's hosted database service, [MongoDB Atlas](https://www.mongodb.com/cloud/atlas/register). You can sign up for free and get a small hosted cluster for free:
+
+![MongoDB Atlas signup](./docs/1_signup.png)
+
+The signup process will walk you through the process of creating your cluster and ensuring it's configured for you to access. Once the cluster is created, choose "Connect" and then "Connect to your application". Choose Python, and you'll be presented with a connection string that looks like this:
+
+![MongoDB Atlas connection string](./docs/2_connection_string.png)
+
+### Set up environment variables
+
+Copy the connection string (make sure you include your password) and put it into a file called `.env` in the root of this repo. It should look like this:
+
+```
+MONGODB_URI=mongodb+srv://seldo:xxxxxxxxxxx@llamaindexdemocluster.xfrdhpz.mongodb.net/?retryWrites=true&w=majority
+```
+
+You will also need to choose a name for your database, and the collection where we will store the tweets, and also include them in .env. They can be any string, but this is what we used:
+
+```
+MONGODB_DATABASE=tiny_tweets_db
+MONGODB_COLLECTION=tiny_tweets_collection
+```
+
+### Import tweets into MongoDB
+
+You are now ready to import our ready-made data set into Mongo. This is the file `tinytweets.json`, a selection of approximately 1000 tweets from @seldo on Twitter in mid-2019. With your environment set up you can do this by running
+
+```
+pnpm ts-node 1_import.ts
+```
+
+If you don't want to use tweets, you can replace `json_file` with any other array of JSON objects, but you will need to modify some code later to make sure the correct field gets indexed. There is no LlamaIndex-specific code here; you can load your data into Mongo any way you want to.
+
+### Load and index your data
+
+Now we're ready to index our data. To do this, LlamaIndex will pull your text out of Mongo, split it into chunks, and then send those chunks to OpenAI to be turned into [vector embeddings](https://docs.llamaindex.ai/en/stable/understanding/indexing/indexing.html#what-is-an-embedding). The embeddings will then be stored in a new collection in Mongo. This will take a while depending how much text you have, but the good news is that once it's done you will be able to query quickly without needing to re-index.
+
+We'll be using OpenAI to do the embedding, so now is when you need to [generate an OpenAI API key](https://platform.openai.com/account/api-keys) if you haven't already and add it to your `.env` file like this:
+
+```
+OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+```
+
+You'll also need to pick a name for the new collection where the embeddings will be stored, and add it to `.env`, along with the name of a vector search index (we'll be creating this in the next step, after you've indexed your data):
+
+```
+MONGODB_VECTORS=tiny_tweets_vectors
+MONGODB_VECTOR_INDEX=tiny_tweets_vector_index
+```
+
+If the data you're indexing is the tweets we gave you, you're ready to go:
+
+```bash
+pnpm ts-node 2_load_and_index.ts
+```
+
+> Note: this script is running a couple of minutes and currently doesn't show any progress.
+
+What you're doing here is creating a Reader which loads the data out of Mongo in the collection and database specified. It looks for text in a set of specific keys in each object. In this case we've given it just one key, "full_text".
+
+Now you're creating a vector search client for Mongo. In addition to a MongoDB client object, you again tell it what database everything is in. This time you give it the name of the collection where you'll store the vector embeddings, and the name of the vector search index you'll create in the next step.
+
+### Create a vector search index
+
+Now if all has gone well you should be able to log in to the Mongo Atlas UI and see two collections in your database: the original data in `tiny_tweets_collection`, and the vector embeddings in `tiny_tweets_vectors`.
+
+![MongoDB Atlas collections](./docs/3_vectors_in_db.png)
+
+Now it's time to create the vector search index so that you can query the data.
+It's not yet possible to programmatically create a vector search index using the [`createIndex`](https://www.mongodb.com/docs/manual/reference/method/db.collection.createIndex/) function, therefore we have to create one manually in the UI.
+To do so, first, click the Search tab, and then click "Create Search Index":
+
+![MongoDB Atlas create search index](./docs/4_search_tab.png)
+
+We have to use the JSON editor, as the Visual Editor does not yet support to create a vector search index:
+
+![MongoDB Atlas JSON editor](./docs/5_json_editor.png)
+
+Now under "database and collection" select `tiny_tweets_db` and within that select `tiny_tweets_vectors`. Then under "Index name" enter `tiny_tweets_vector_index` (or whatever value you put for MONGODB_VECTOR_INDEX in `.env`). Under that, you'll want to enter this JSON object:
+
+```json
+{
+  "mappings": {
+    "dynamic": true,
+    "fields": {
+      "embedding": {
+        "dimensions": 1536,
+        "similarity": "cosine",
+        "type": "knnVector"
+      }
+    }
+  }
+}
+```
+
+This tells Mongo that the `embedding` field in each document (in the `tiny_tweets_vectors` collection) is a vector of 1536 dimensions (this is the size of embeddings used by OpenAI), and that we want to use cosine similarity to compare vectors. You don't need to worry too much about these values unless you want to use a different LLM to OpenAI entirely.
+
+The UI will ask you to review and confirm your choices, then you need to wait a minute or two while it generates the index. If all goes well, you should see something like this screen:
+
+![MongoDB Atlas index created](./docs/7_index_created.png)
+
+Now you're ready to query your data!
+
+### Run a test query
+
+You can do this by running
+
+```bash
+pnpm ts-node 3_query.ts
+```
+
+This sets up a connection to Atlas just like `2_load_and_index.ts` did, then it creates a [query engine](https://docs.llamaindex.ai/en/stable/understanding/querying/querying.html#getting-started) and runs a query against it.
+
+If all is well, you should get a nuanced opinion about web frameworks.

apps/mongodb/package.json

@@ -0,0 +1,17 @@
+{
+  "version": "0.0.1",
+  "private": true,
+  "name": "mongodb-llamaindexts",
+  "dependencies": {
+    "llamaindex": "workspace:*",
+    "dotenv": "^16.3.1",
+    "mongodb": "^6.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18.18.6",
+    "ts-node": "^10.9.1"
+  },
+  "scripts": {
+    "lint": "eslint ."
+  }
+}

apps/simple/CHANGELOG.md

@@ -1,5 +1,43 @@
 # simple
 
+## 0.0.33
+
+### Patch Changes
+
+- Updated dependencies [63f2108]
+  - llamaindex@0.0.35
+
+## 0.0.32
+
+### Patch Changes
+
+- Updated dependencies [2a27e21]
+  - llamaindex@0.0.34
+
+## 0.0.31
+
+### Patch Changes
+
+- Updated dependencies [5e2e92c]
+  - llamaindex@0.0.33
+
+## 0.0.30
+
+### Patch Changes
+
+- Updated dependencies [90c0b83]
+- Updated dependencies [dfd22aa]
+  - llamaindex@0.0.32
+
+## 0.0.29
+
+### Patch Changes
+
+- Updated dependencies [6c55b2d]
+- Updated dependencies [8aa8c65]
+- Updated dependencies [6c55b2d]
+  - llamaindex@0.0.31
+
 ## 0.0.28
 
 ### Patch Changes

apps/simple/anthropic.ts

@@ -4,8 +4,6 @@ import { Anthropic } from "llamaindex";
   const anthropic = new Anthropic();
   const result = await anthropic.chat([
     { content: "You want to talk in rhymes.", role: "system" },
-    { content: "Hello, world!", role: "user" },
-    { content: "Hello!", role: "assistant" },
     {
       content:
         "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",

apps/simple/assemblyai.ts

@@ -0,0 +1,60 @@
+import { program } from "commander";
+import { AudioTranscriptReader, CreateTranscriptParameters } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+import { VectorStoreIndex } from "../../packages/core/src/indices";
+
+program
+  .option(
+    "-a, --audio-url [string]",
+    "URL or path of the audio file to transcribe",
+  )
+  .option("-i, --transcript-id [string]", "ID of the AssemblyAI transcript")
+  .action(async (options) => {
+    if (!process.env.ASSEMBLYAI_API_KEY) {
+      console.log("No ASSEMBLYAI_API_KEY found in environment variables.");
+      return;
+    }
+
+    const reader = new AudioTranscriptReader();
+    let params: CreateTranscriptParameters | string;
+    console.log(options);
+    if (options.audioUrl) {
+      params = {
+        audio_url: options.audioUrl,
+      };
+    } else if (options.transcriptId) {
+      params = options.transcriptId;
+    } else {
+      console.log(
+        "You must provide either an --audio-url or a --transcript-id",
+      );
+      return;
+    }
+
+    const documents = await reader.loadData(params);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Ask a question: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

apps/simple/directory.ts

@@ -0,0 +1,24 @@
+import { SimpleDirectoryReader } from "llamaindex";
+
+function callback(
+  category: string,
+  name: string,
+  status: any,
+  message?: string,
+): boolean {
+  console.log(category, name, status, message);
+  if (name.endsWith(".pdf")) {
+    console.log("I DON'T WANT PDF FILES!");
+    return false;
+  }
+  return true;
+}
+
+async function main() {
+  // Load page
+  const reader = new SimpleDirectoryReader(callback);
+  const params = { directoryPath: "./data" };
+  await reader.loadData(params);
+}
+
+main().catch(console.error);

apps/simple/html.ts

@@ -0,0 +1,21 @@
+import { HTMLReader, VectorStoreIndex } from "llamaindex";
+
+async function main() {
+  // Load page
+  const reader = new HTMLReader();
+  const documents = await reader.loadData("data/18-1_Changelog.html");
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  const index = await VectorStoreIndex.fromDocuments(documents);
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query(
+    "What were the notable changes in 18.1?",
+  );
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);

apps/simple/llmStream.ts

@@ -1,47 +0,0 @@
-import { ChatMessage, OpenAI, SimpleChatEngine } from "llamaindex";
-import {Anthropic} from "../../packages/core/src/llm/LLM";
-import { stdin as input, stdout as output } from "node:process";
-import readline from "node:readline/promises";
-
-async function main() {
-  const query: string = `
-Where is Istanbul?
-  `;
-
-  // const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 });
-  const llm = new Anthropic();
-  const message: ChatMessage = { content: query, role: "user" };
-
-  //TODO: Add callbacks later
-
-  //Stream Complete
-  //Note: Setting streaming flag to true or false will auto-set your return type to
-  //either an AsyncGenerator or a Response.
-  // Omitting the streaming flag automatically sets streaming to false
-
-  const chatEngine: SimpleChatEngine = new SimpleChatEngine({
-    chatHistory: undefined,
-    llm: llm,
-  });
-
-  const rl = readline.createInterface({ input, output });
-  while (true) {
-    const query = await rl.question("Query: ");
-
-    if (!query) {
-      break;
-    }
-
-    //Case 1: .chat(query, undefined, true) => Stream
-    //Case 2: .chat(query, undefined, false) => Response object
-    //Case 3: .chat(query, undefined) => Response object
-    const chatStream = await chatEngine.chat(query, undefined, true);
-    var accumulated_result = "";
-    for await (const part of chatStream) {
-      accumulated_result += part;
-      process.stdout.write(part);
-    }
-  }
-}
-
-main();

apps/simple/openai.ts

@@ -1,7 +1,7 @@
 import { OpenAI } from "llamaindex";
 
 (async () => {
-  const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 });
+  const llm = new OpenAI({ model: "gpt-4-1106-preview", temperature: 0.1 });
 
   // complete api
   const response1 = await llm.complete("How are you?");
@@ -9,7 +9,7 @@ import { OpenAI } from "llamaindex";
 
   // chat api
   const response2 = await llm.chat([
-    { content: "Tell me a joke!", role: "user" },
+    { content: "Tell me a joke.", role: "user" },
   ]);
   console.log(response2.message.content);
 })();

apps/simple/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.28",
+  "version": "0.0.33",
   "private": true,
   "name": "simple",
   "dependencies": {
@@ -9,7 +9,8 @@
     "llamaindex": "workspace:*"
   },
   "devDependencies": {
-    "@types/node": "^18.18.6"
+    "@types/node": "^18.18.6",
+    "ts-node": "^10.9.1"
   },
   "scripts": {
     "lint": "eslint ."

apps/simple/pg-vector-store/README.md

@@ -0,0 +1,33 @@
+# Postgres Vector Store
+
+There are two scripts available here: load-docs.ts and query.ts
+
+## Prerequisites
+
+You'll need a postgres database instance against which to run these scripts. A simple docker command would look like this:
+
+> `docker run -d --rm --name vector-db -p 5432:5432 -e "POSTGRES_HOST_AUTH_METHOD=trust" ankane/pgvector`
+
+Set the PGHOST and PGUSER (and PGPASSWORD) environment variables to match your database setup.
+
+You'll also need a value for OPENAI_API_KEY in your environment.
+
+**NOTE:** Using `--rm` in the example docker command above means that the vector store will be deleted every time the container is stopped. For production purposes, use a volume to ensure persistence across restarts.
+
+## Setup and Loading Docs
+
+Read and follow the instructions in the README.md file located one directory up to make sure your JS/TS dependencies are set up. The commands listed below are also run from that parent directory.
+
+To import documents and save the embedding vectors to your database:
+
+> `npx ts-node pg-vector-store/load-docs.ts data`
+
+where data is the directory containing your input files. Using the _data_ directory in the example above will read all of the files in that directory using the llamaindexTS default readers for each file type.
+
+## RAG Querying
+
+To query using the resulting vector store:
+
+> `npx ts-node pg-vector-store/query.ts`
+
+The script will prompt for a question, then process and present the answer using the PGVectorStore data and your OpenAI API key. It will continue to prompt until you enter `q`, `quit` or `exit` as the next query.

apps/simple/pg-vector-store/load-docs.ts

@@ -0,0 +1,68 @@
+// load-docs.ts
+import fs from "fs/promises";
+import {
+  SimpleDirectoryReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { PGVectorStore } from "../../../packages/core/src/storage/vectorStore/PGVectorStore";
+
+async function getSourceFilenames(sourceDir: string) {
+  return await fs
+    .readdir(sourceDir)
+    .then((fileNames) => fileNames.map((file) => sourceDir + "/" + file));
+}
+
+function callback(
+  category: string,
+  name: string,
+  status: any,
+  message: string = "",
+): boolean {
+  console.log(category, name, status, message);
+  return true;
+}
+
+async function main(args: any) {
+  const sourceDir: string = args.length > 2 ? args[2] : "../data";
+
+  console.log(`Finding documents in ${sourceDir}`);
+  const fileList = await getSourceFilenames(sourceDir);
+  const count = fileList.length;
+  console.log(`Found ${count} files`);
+
+  console.log(`Importing contents from ${count} files in ${sourceDir}`);
+  var fileName = "";
+  try {
+    // Passing callback fn to the ctor here
+    // will enable looging to console.
+    // See callback fn, defined above.
+    const rdr = new SimpleDirectoryReader(callback);
+    const docs = await rdr.loadData({ directoryPath: sourceDir });
+
+    const pgvs = new PGVectorStore();
+    pgvs.setCollection(sourceDir);
+    pgvs.clearCollection();
+
+    const ctx = await storageContextFromDefaults({ vectorStore: pgvs });
+
+    console.debug("  - creating vector store");
+    const index = await VectorStoreIndex.fromDocuments(docs, {
+      storageContext: ctx,
+    });
+    console.debug("  - done.");
+  } catch (err) {
+    console.error(fileName, err);
+    console.log(
+      "If your PGVectorStore init failed, make sure to set env vars for PGUSER or USER, PGHOST, PGPORT and PGPASSWORD as needed.",
+    );
+    process.exit(1);
+  }
+
+  console.log(
+    "Done. Try running query.ts to ask questions against the imported embeddings.",
+  );
+  process.exit(0);
+}
+
+main(process.argv).catch((err) => console.error(err));

apps/simple/pg-vector-store/query.ts

@@ -0,0 +1,67 @@
+import { VectorStoreIndex } from "../../../packages/core/src/indices/vectorStore/VectorStoreIndex";
+import { serviceContextFromDefaults } from "../../../packages/core/src/ServiceContext";
+import { PGVectorStore } from "../../../packages/core/src/storage/vectorStore/PGVectorStore";
+
+async function main() {
+  const readline = require("readline").createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  try {
+    const pgvs = new PGVectorStore();
+    // Optional - set your collection name, default is no filter on this field.
+    // pgvs.setCollection();
+
+    const ctx = serviceContextFromDefaults();
+    const index = await VectorStoreIndex.fromVectorStore(pgvs, ctx);
+
+    // Query the index
+    const queryEngine = await index.asQueryEngine();
+
+    let question = "";
+    while (!isQuit(question)) {
+      question = await getUserInput(readline);
+
+      if (isQuit(question)) {
+        readline.close();
+        process.exit(0);
+      }
+
+      try {
+        const answer = await queryEngine.query(question);
+        console.log(answer.response);
+      } catch (error) {
+        console.error("Error:", error);
+      }
+    }
+  } catch (err) {
+    console.error(err);
+    console.log(
+      "If your PGVectorStore init failed, make sure to set env vars for PGUSER or USER, PGHOST, PGPORT and PGPASSWORD as needed.",
+    );
+    process.exit(1);
+  }
+}
+
+function isQuit(question: string) {
+  return ["q", "quit", "exit"].includes(question.trim().toLowerCase());
+}
+
+// Function to get user input as a promise
+function getUserInput(readline: any): Promise<string> {
+  return new Promise((resolve) => {
+    readline.question(
+      "What would you like to know?\n>",
+      (userInput: string) => {
+        resolve(userInput);
+      },
+    );
+  });
+}
+
+main()
+  .catch(console.error)
+  .finally(() => {
+    process.exit(1);
+  });

apps/simple/portkey.ts

@@ -1,23 +1,23 @@
 import { Portkey } from "llamaindex";
 
 (async () => {
-  const llms = [{
-
-  }]
+  const llms = [{}];
   const portkey = new Portkey({
     mode: "single",
-    llms: [{
-      provider:"anyscale",
-      virtual_key:"anyscale-3b3c04",
-      model: "meta-llama/Llama-2-13b-chat-hf",
-      max_tokens: 2000
-    }]
+    llms: [
+      {
+        provider: "anyscale",
+        virtual_key: "anyscale-3b3c04",
+        model: "meta-llama/Llama-2-13b-chat-hf",
+        max_tokens: 2000,
+      },
+    ],
   });
   const result = portkey.stream_chat([
     { role: "system", content: "You are a helpful assistant." },
-    { role: "user", content: "Tell me a joke." }
+    { role: "user", content: "Tell me a joke." },
   ]);
   for await (const res of result) {
-    process.stdout.write(res)
+    process.stdout.write(res);
   }
 })();

apps/simple/vectorIndexAnthropic.ts

@@ -2,7 +2,10 @@ import fs from "node:fs/promises";
 
 import {
   Anthropic,
+  anthropicTextQaPrompt,
+  CompactAndRefine,
   Document,
+  ResponseSynthesizer,
   serviceContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
@@ -18,12 +21,20 @@ async function main() {
 
   // Split text and create embeddings. Store them in a VectorStoreIndex
   const serviceContext = serviceContextFromDefaults({ llm: new Anthropic() });
+
+  const responseSynthesizer = new ResponseSynthesizer({
+    responseBuilder: new CompactAndRefine(
+      serviceContext,
+      anthropicTextQaPrompt,
+    ),
+  });
+
   const index = await VectorStoreIndex.fromDocuments([document], {
     serviceContext,
   });
 
   // Query the index
-  const queryEngine = index.asQueryEngine();
+  const queryEngine = index.asQueryEngine({ responseSynthesizer });
   const response = await queryEngine.query(
     "What did the author do in college?",
   );

apps/simple/vision.ts

@@ -0,0 +1,15 @@
+import { OpenAI } from "llamaindex";
+
+(async () => {
+  const llm = new OpenAI({ model: "gpt-4-vision-preview", temperature: 0.1 });
+
+  // complete api
+  const response1 = await llm.complete("How are you?");
+  console.log(response1.message.content);
+
+  // chat api
+  const response2 = await llm.chat([
+    { content: "Tell me a joke!", role: "user" },
+  ]);
+  console.log(response2.message.content);
+})();

examples/anthropic.ts

@@ -4,8 +4,6 @@ import { Anthropic } from "llamaindex";
   const anthropic = new Anthropic();
   const result = await anthropic.chat([
     { content: "You want to talk in rhymes.", role: "system" },
-    { content: "Hello, world!", role: "user" },
-    { content: "Hello!", role: "assistant" },
     {
       content:
         "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",

examples/assemblyai.ts

@@ -0,0 +1,60 @@
+import { program } from "commander";
+import { AudioTranscriptReader, CreateTranscriptParameters } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+import { VectorStoreIndex } from "../../packages/core/src/indices";
+
+program
+  .option(
+    "-a, --audio-url [string]",
+    "URL or path of the audio file to transcribe",
+  )
+  .option("-i, --transcript-id [string]", "ID of the AssemblyAI transcript")
+  .action(async (options) => {
+    if (!process.env.ASSEMBLYAI_API_KEY) {
+      console.log("No ASSEMBLYAI_API_KEY found in environment variables.");
+      return;
+    }
+
+    const reader = new AudioTranscriptReader();
+    let params: CreateTranscriptParameters | string;
+    console.log(options);
+    if (options.audioUrl) {
+      params = {
+        audio_url: options.audioUrl,
+      };
+    } else if (options.transcriptId) {
+      params = options.transcriptId;
+    } else {
+      console.log(
+        "You must provide either an --audio-url or a --transcript-id",
+      );
+      return;
+    }
+
+    const documents = await reader.loadData(params);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Ask a question: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

examples/clip.ts

@@ -0,0 +1,33 @@
+import { ClipEmbedding, similarity, SimilarityType } from "llamaindex";
+
+async function main() {
+  const clip = new ClipEmbedding();
+
+  // Get text embeddings
+  const text1 = "a car";
+  const textEmbedding1 = await clip.getTextEmbedding(text1);
+  const text2 = "a football match";
+  const textEmbedding2 = await clip.getTextEmbedding(text2);
+
+  // Get image embedding
+  const image =
+    "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg";
+  const imageEmbedding = await clip.getImageEmbedding(image);
+
+  // Calc similarity
+  const sim1 = similarity(
+    textEmbedding1,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+  const sim2 = similarity(
+    textEmbedding2,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+
+  console.log(`Similarity between "${text1}" and the image is ${sim1}`);
+  console.log(`Similarity between "${text2}" and the image is ${sim2}`);
+}
+
+main();

examples/directory.ts

@@ -0,0 +1,24 @@
+import { SimpleDirectoryReader } from "llamaindex";
+
+function callback(
+  category: string,
+  name: string,
+  status: any,
+  message?: string,
+): boolean {
+  console.log(category, name, status, message);
+  if (name.endsWith(".pdf")) {
+    console.log("I DON'T WANT PDF FILES!");
+    return false;
+  }
+  return true;
+}
+
+async function main() {
+  // Load page
+  const reader = new SimpleDirectoryReader(callback);
+  const params = { directoryPath: "./data" };
+  await reader.loadData(params);
+}
+
+main().catch(console.error);

examples/html.ts

@@ -0,0 +1,21 @@
+import { HTMLReader, VectorStoreIndex } from "llamaindex";
+
+async function main() {
+  // Load page
+  const reader = new HTMLReader();
+  const documents = await reader.loadData("data/18-1_Changelog.html");
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  const index = await VectorStoreIndex.fromDocuments(documents);
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query(
+    "What were the notable changes in 18.1?",
+  );
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);

examples/mongo.ts

@@ -0,0 +1,68 @@
+import { MongoClient } from "mongodb";
+import { Document } from "../../packages/core/src/Node";
+import { VectorStoreIndex } from "../../packages/core/src/indices";
+import { SimpleMongoReader } from "../../packages/core/src/readers/SimpleMongoReader";
+
+import { stdin as input, stdout as output } from "node:process";
+import readline from "node:readline/promises";
+
+async function main() {
+  //Dummy test code
+  const query: object = { _id: "waldo" };
+  const options: object = {};
+  const projections: object = { embedding: 0 };
+  const limit: number = Infinity;
+  const uri: string = process.env.MONGODB_URI ?? "fake_uri";
+  const client: MongoClient = new MongoClient(uri);
+
+  //Where the real code starts
+  const MR = new SimpleMongoReader(client);
+  const documents: Document[] = await MR.loadData(
+    "data",
+    "posts",
+    1,
+    {},
+    options,
+    projections,
+  );
+
+  //
+  //If you need to look at low-level details of
+  // a queryEngine (for example, needing to check each individual node)
+  //
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  // var storageContext = await storageContextFromDefaults({});
+  // var serviceContext = serviceContextFromDefaults({});
+  // const docStore = storageContext.docStore;
+
+  // for (const doc of documents) {
+  //   docStore.setDocumentHash(doc.id_, doc.hash);
+  // }
+  // const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
+  // console.log(nodes);
+
+  //
+  //Making Vector Store from documents
+  //
+
+  const index = await VectorStoreIndex.fromDocuments(documents);
+  // Create query engine
+  const queryEngine = index.asQueryEngine();
+
+  const rl = readline.createInterface({ input, output });
+  while (true) {
+    const query = await rl.question("Query: ");
+
+    if (!query) {
+      break;
+    }
+
+    const response = await queryEngine.query(query);
+
+    // Output response
+    console.log(response.toString());
+  }
+}
+
+main();

examples/openai.ts

@@ -1,7 +1,7 @@
 import { OpenAI } from "llamaindex";
 
 (async () => {
-  const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.0 });
+  const llm = new OpenAI({ model: "gpt-4-1106-preview", temperature: 0.1 });
 
   // complete api
   const response1 = await llm.complete("How are you?");
@@ -9,7 +9,7 @@ import { OpenAI } from "llamaindex";
 
   // chat api
   const response2 = await llm.chat([
-    { content: "Tell me a joke!", role: "user" },
+    { content: "Tell me a joke.", role: "user" },
   ]);
   console.log(response2.message.content);
 })();

examples/portkey.ts

@@ -1,23 +1,23 @@
 import { Portkey } from "llamaindex";
 
 (async () => {
-  const llms = [{
-
-  }]
+  const llms = [{}];
   const portkey = new Portkey({
     mode: "single",
-    llms: [{
-      provider:"anyscale",
-      virtual_key:"anyscale-3b3c04",
-      model: "meta-llama/Llama-2-13b-chat-hf",
-      max_tokens: 2000
-    }]
+    llms: [
+      {
+        provider: "anyscale",
+        virtual_key: "anyscale-3b3c04",
+        model: "meta-llama/Llama-2-13b-chat-hf",
+        max_tokens: 2000,
+      },
+    ],
   });
   const result = portkey.stream_chat([
     { role: "system", content: "You are a helpful assistant." },
-    { role: "user", content: "Tell me a joke." }
+    { role: "user", content: "Tell me a joke." },
   ]);
   for await (const res of result) {
-    process.stdout.write(res)
+    process.stdout.write(res);
   }
 })();

examples/vectorIndexAnthropic.ts

@@ -2,7 +2,10 @@ import fs from "node:fs/promises";
 
 import {
   Anthropic,
+  anthropicTextQaPrompt,
+  CompactAndRefine,
   Document,
+  ResponseSynthesizer,
   serviceContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
@@ -18,12 +21,20 @@ async function main() {
 
   // Split text and create embeddings. Store them in a VectorStoreIndex
   const serviceContext = serviceContextFromDefaults({ llm: new Anthropic() });
+
+  const responseSynthesizer = new ResponseSynthesizer({
+    responseBuilder: new CompactAndRefine(
+      serviceContext,
+      anthropicTextQaPrompt,
+    ),
+  });
+
   const index = await VectorStoreIndex.fromDocuments([document], {
     serviceContext,
   });
 
   // Query the index
-  const queryEngine = index.asQueryEngine();
+  const queryEngine = index.asQueryEngine({ responseSynthesizer });
   const response = await queryEngine.query(
     "What did the author do in college?",
   );

examples/vectorIndexCustomize.ts

@@ -3,6 +3,7 @@ import {
   OpenAI,
   RetrieverQueryEngine,
   serviceContextFromDefaults,
+  SimilarityPostprocessor,
   VectorStoreIndex,
 } from "llamaindex";
 import essay from "./essay";
@@ -21,8 +22,16 @@ async function main() {
 
   const retriever = index.asRetriever();
   retriever.similarityTopK = 5;
+  const nodePostprocessor = new SimilarityPostprocessor({
+    similarityCutoff: 0.7,
+  });
   // TODO: cannot pass responseSynthesizer into retriever query engine
-  const queryEngine = new RetrieverQueryEngine(retriever);
+  const queryEngine = new RetrieverQueryEngine(
+    retriever,
+    undefined,
+    undefined,
+    [nodePostprocessor],
+  );
 
   const response = await queryEngine.query(
     "What did the author do growing up?",

examples/vectorIndexFromVectorStore.ts

@@ -0,0 +1,197 @@
+import {
+  OpenAI,
+  ResponseSynthesizer,
+  RetrieverQueryEngine,
+  serviceContextFromDefaults,
+  TextNode,
+  TreeSummarize,
+  VectorIndexRetriever,
+  VectorStore,
+  VectorStoreIndex,
+  VectorStoreQuery,
+  VectorStoreQueryResult,
+} from "llamaindex";
+
+import { Index, Pinecone, RecordMetadata } from "@pinecone-database/pinecone";
+
+/**
+ * Please do not use this class in production; it's only for demonstration purposes.
+ */
+class PineconeVectorStore<T extends RecordMetadata = RecordMetadata>
+  implements VectorStore
+{
+  storesText = true;
+  isEmbeddingQuery = false;
+
+  indexName!: string;
+  pineconeClient!: Pinecone;
+  index!: Index<T>;
+
+  constructor({ indexName, client }: { indexName: string; client: Pinecone }) {
+    this.indexName = indexName;
+    this.pineconeClient = client;
+    this.index = client.index<T>(indexName);
+  }
+
+  client() {
+    return this.pineconeClient;
+  }
+
+  async query(
+    query: VectorStoreQuery,
+    kwargs?: any,
+  ): Promise<VectorStoreQueryResult> {
+    let queryEmbedding: number[] = [];
+    if (query.queryEmbedding) {
+      if (typeof query.alpha === "number") {
+        const alpha = query.alpha;
+        queryEmbedding = query.queryEmbedding.map((v) => v * alpha);
+      } else {
+        queryEmbedding = query.queryEmbedding;
+      }
+    }
+
+    // Current LlamaIndexTS implementation only support exact match filter, so we use kwargs instead.
+    const filter = kwargs?.filter || {};
+
+    const response = await this.index.query({
+      filter,
+      vector: queryEmbedding,
+      topK: query.similarityTopK,
+      includeValues: true,
+      includeMetadata: true,
+    });
+
+    console.log(
+      `Numbers of vectors returned by Pinecone after preFilters are applied: ${
+        response?.matches?.length || 0
+      }.`,
+    );
+
+    const topKIds: string[] = [];
+    const topKNodes: TextNode[] = [];
+    const topKScores: number[] = [];
+
+    const metadataToNode = (metadata?: T): Partial<TextNode> => {
+      if (!metadata) {
+        throw new Error("metadata is undefined.");
+      }
+
+      const nodeContent = metadata["_node_content"];
+      if (!nodeContent) {
+        throw new Error("nodeContent is undefined.");
+      }
+
+      if (typeof nodeContent !== "string") {
+        throw new Error("nodeContent is not a string.");
+      }
+
+      return JSON.parse(nodeContent);
+    };
+
+    if (response.matches) {
+      for (const match of response.matches) {
+        const node = new TextNode({
+          ...metadataToNode(match.metadata),
+          embedding: match.values,
+        });
+
+        topKIds.push(match.id);
+        topKNodes.push(node);
+        topKScores.push(match.score ?? 0);
+      }
+    }
+
+    const result = {
+      ids: topKIds,
+      nodes: topKNodes,
+      similarities: topKScores,
+    };
+
+    return result;
+  }
+
+  add(): Promise<string[]> {
+    return Promise.resolve([]);
+  }
+
+  delete(): Promise<void> {
+    throw new Error("Method `delete` not implemented.");
+  }
+
+  persist(): Promise<void> {
+    throw new Error("Method `persist` not implemented.");
+  }
+}
+
+/**
+ * The goal of this example is to show how to use Pinecone as a vector store
+ * for LlamaIndexTS with(out) preFilters.
+ *
+ * It should not be used in production like that,
+ * as you might want to find a proper PineconeVectorStore implementation.
+ */
+async function main() {
+  process.env.PINECONE_API_KEY = "Your Pinecone API Key.";
+  process.env.PINECONE_ENVIRONMENT = "Your Pinecone Environment.";
+  process.env.PINECONE_PROJECT_ID = "Your Pinecone Project ID.";
+  process.env.PINECONE_INDEX_NAME = "Your Pinecone Index Name.";
+  process.env.OPENAI_API_KEY = "Your OpenAI API Key.";
+  process.env.OPENAI_API_ORGANIZATION = "Your OpenAI API Organization.";
+
+  const getPineconeVectorStore = async () => {
+    return new PineconeVectorStore({
+      indexName: process.env.PINECONE_INDEX_NAME || "index-name",
+      client: new Pinecone(),
+    });
+  };
+
+  const getServiceContext = () => {
+    const openAI = new OpenAI({
+      model: "gpt-4",
+      apiKey: process.env.OPENAI_API_KEY,
+    });
+
+    return serviceContextFromDefaults({
+      llm: openAI,
+    });
+  };
+
+  const getQueryEngine = async (filter: unknown) => {
+    const vectorStore = await getPineconeVectorStore();
+    const serviceContext = getServiceContext();
+
+    const vectorStoreIndex = await VectorStoreIndex.fromVectorStore(
+      vectorStore,
+      serviceContext,
+    );
+
+    const retriever = new VectorIndexRetriever({
+      index: vectorStoreIndex,
+      similarityTopK: 500,
+    });
+
+    const responseSynthesizer = new ResponseSynthesizer({
+      serviceContext,
+      responseBuilder: new TreeSummarize(serviceContext),
+    });
+
+    return new RetrieverQueryEngine(retriever, responseSynthesizer, {
+      filter,
+    });
+  };
+
+  // whatever is a key from your metadata
+  const queryEngine = await getQueryEngine({
+    whatever: {
+      $gte: 1,
+      $lte: 100,
+    },
+  });
+
+  const response = await queryEngine.query("How many results do you have?");
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);

examples/vision.ts

@@ -0,0 +1,15 @@
+import { OpenAI } from "llamaindex";
+
+(async () => {
+  const llm = new OpenAI({ model: "gpt-4-vision-preview", temperature: 0.1 });
+
+  // complete api
+  const response1 = await llm.complete("How are you?");
+  console.log(response1.message.content);
+
+  // chat api
+  const response2 = await llm.chat([
+    { content: "Tell me a joke!", role: "user" },
+  ]);
+  console.log(response2.message.content);
+})();

package.json

@@ -3,7 +3,7 @@
   "scripts": {
     "build": "turbo run build",
     "dev": "turbo run dev",
-    "format": "prettier --write \"**/*.{ts,tsx,md}\"",
+    "format": "prettier --write \"**/*.{js,jsx,ts,tsx,md}\"",
     "lint": "turbo run lint",
     "prepare": "husky install",
     "test": "turbo run test",
@@ -11,24 +11,27 @@
     "publish-snapshot": "turbo run build lint test && changeset version --snapshot && changeset publish"
   },
   "devDependencies": {
+    "@changesets/cli": "^2.26.2",
     "@turbo/gen": "^1.10.16",
-    "@types/jest": "^29.5.6",
-    "eslint": "^8.52.0",
+    "@types/jest": "^29.5.10",
+    "eslint": "^8.54.0",
     "eslint-config-custom": "workspace:*",
     "husky": "^8.0.3",
     "jest": "^29.7.0",
-    "prettier": "^3.0.3",
-    "prettier-plugin-organize-imports": "^3.2.3",
+    "lint-staged": "^15.1.0",
+    "prettier": "^3.1.0",
+    "prettier-plugin-organize-imports": "^3.2.4",
     "ts-jest": "^29.1.1",
     "turbo": "^1.10.16"
   },
-  "packageManager": "pnpm@7.15.0",
-  "dependencies": {
-    "@changesets/cli": "^2.26.2"
-  },
+  "packageManager": "pnpm@8.10.5+sha256.a4bd9bb7b48214bbfcd95f264bd75bb70d100e5d4b58808f5cd6ab40c6ac21c5",
   "pnpm": {
     "overrides": {
-      "trim": "1.0.1"
+      "trim": "1.0.1",
+      "@babel/traverse": "7.23.2"
     }
+  },
+  "lint-staged": {
+    "*.{js,jsx,ts,tsx,md}": "prettier --write"
   }
 }

packages/core/CHANGELOG.md

@@ -1,5 +1,38 @@
 # llamaindex
 
+## 0.0.35
+
+### Patch Changes
+
+- 63f2108: Add multimodal support (thanks @marcusschiesser)
+
+## 0.0.34
+
+### Patch Changes
+
+- 2a27e21: Add support for gpt-3.5-turbo-1106
+
+## 0.0.33
+
+### Patch Changes
+
+- 5e2e92c: gpt-4-1106-preview and gpt-4-vision-preview from OpenAI dev day
+
+## 0.0.32
+
+### Patch Changes
+
+- 90c0b83: Add HTMLReader (thanks @mtutty)
+- dfd22aa: Add observer/filter to the SimpleDirectoryReader (thanks @mtutty)
+
+## 0.0.31
+
+### Patch Changes
+
+- 6c55b2d: Give HistoryChatEngine pluggable options (thanks @marcusschiesser)
+- 8aa8c65: Add SimilarityPostProcessor (thanks @TomPenguin)
+- 6c55b2d: Added LLMMetadata (thanks @marcusschiesser)
+
 ## 0.0.30
 
 ### Patch Changes

packages/core/package.json

@@ -1,34 +1,42 @@
 {
   "name": "llamaindex",
-  "version": "0.0.30",
+  "version": "0.0.35",
   "license": "MIT",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.8.0",
+    "@anthropic-ai/sdk": "^0.9.1",
     "@notionhq/client": "^2.2.13",
+    "@xenova/transformers": "^2.8.0",
+    "assemblyai": "^3.0.1",
+    "crypto-js": "^4.2.0",
+    "js-tiktoken": "^1.0.8",
     "lodash": "^4.17.21",
     "mammoth": "^1.6.0",
     "md-utils-ts": "^2.0.0",
-    "mongodb": "^6.2.0",
+    "mongodb": "^6.3.0",
     "notion-md-crawler": "^0.0.2",
-    "openai": "^4.13.0",
+    "openai": "^4.19.1",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
-    "portkey-ai": "^0.1.13",
+    "pg": "^8.11.3",
+    "pgvector": "^0.1.5",
+    "portkey-ai": "^0.1.16",
     "rake-modified": "^1.0.8",
-    "replicate": "^0.20.1",
-    "tiktoken": "^1.0.10",
+    "replicate": "^0.21.1",
+    "string-strip-html": "^13.4.3",
     "uuid": "^9.0.1",
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
-    "@types/lodash": "^4.14.200",
-    "@types/node": "^18.18.6",
-    "@types/papaparse": "^5.3.10",
-    "@types/pdf-parse": "^1.1.3",
-    "@types/uuid": "^9.0.6",
+    "@types/crypto-js": "^4.2.1",
+    "@types/lodash": "^4.14.202",
+    "@types/node": "^18.18.12",
+    "@types/papaparse": "^5.3.13",
+    "@types/pdf-parse": "^1.1.4",
+    "@types/pg": "^8.10.7",
+    "@types/uuid": "^9.0.7",
     "node-stdlib-browser": "^1.2.0",
     "tsup": "^7.2.0",
-    "typescript": "^4.9.5"
+    "typescript": "^5.3.2"
   },
   "engines": {
     "node": ">=18.0.0"
@@ -42,5 +50,12 @@
     "test": "jest",
     "build": "tsup src/index.ts --format esm,cjs --dts",
     "dev": "tsup src/index.ts --format esm,cjs --dts --watch"
+  },
+  "exports": {
+    ".": {
+      "require": "./dist/index.js",
+      "import": "./dist/index.mjs",
+      "types": "./dist/index.d.ts"
+    }
   }
-}
+}

packages/core/src/ChatEngine.ts

@@ -1,8 +1,5 @@
 import { v4 as uuidv4 } from "uuid";
-import { Event } from "./callbacks/CallbackManager";
 import { ChatHistory } from "./ChatHistory";
-import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
-import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   CondenseQuestionPrompt,
@@ -15,6 +12,9 @@ import { BaseQueryEngine } from "./QueryEngine";
 import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
+import { Event } from "./callbacks/CallbackManager";
+import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
+import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 
 /**
  * A ChatEngine is used to handle back and forth chats between the application and the LLM.
@@ -328,6 +328,17 @@ export class ContextChatEngine implements ChatEngine {
   }
 }
 
+export interface MessageContentDetail {
+  type: "text" | "image_url";
+  text: string;
+  image_url: { url: string };
+}
+
+/**
+ * Extended type for the content of a message that allows for multi-modal messages.
+ */
+export type MessageContent = string | MessageContentDetail[];
+
 /**
  * HistoryChatEngine is a ChatEngine that uses a `ChatHistory` object
  * to keeps track of chat's message history.
@@ -347,38 +358,34 @@ export class HistoryChatEngine {
   async chat<
     T extends boolean | undefined = undefined,
     R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(message: string, chatHistory: ChatHistory, streaming?: T): Promise<R> {
+  >(
+    message: MessageContent,
+    chatHistory: ChatHistory,
+    streaming?: T,
+  ): Promise<R> {
     //Streaming option
     if (streaming) {
       return this.streamChat(message, chatHistory) as R;
     }
-    const context = await this.contextGenerator?.generate(message);
-    chatHistory.addMessage({
-      content: message,
-      role: "user",
-    });
-    const response = await this.llm.chat(
-      await chatHistory.requestMessages(
-        context ? [context.message] : undefined,
-      ),
+    const requestMessages = await this.prepareRequestMessages(
+      message,
+      chatHistory,
     );
+    const response = await this.llm.chat(requestMessages);
     chatHistory.addMessage(response.message);
     return new Response(response.message.content) as R;
   }
 
   protected async *streamChat(
-    message: string,
+    message: MessageContent,
     chatHistory: ChatHistory,
   ): AsyncGenerator<string, void, unknown> {
-    const context = await this.contextGenerator?.generate(message);
-    chatHistory.addMessage({
-      content: message,
-      role: "user",
-    });
+    const requestMessages = await this.prepareRequestMessages(
+      message,
+      chatHistory,
+    );
     const response_stream = await this.llm.chat(
-      await chatHistory.requestMessages(
-        context ? [context.message] : undefined,
-      ),
+      requestMessages,
       undefined,
       true,
     );
@@ -394,4 +401,31 @@ export class HistoryChatEngine {
     });
     return;
   }
+
+  private async prepareRequestMessages(
+    message: MessageContent,
+    chatHistory: ChatHistory,
+  ) {
+    chatHistory.addMessage({
+      content: message,
+      role: "user",
+    });
+    let requestMessages;
+    let context;
+    if (this.contextGenerator) {
+      if (Array.isArray(message)) {
+        // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
+        // so we can pass them to the context generator
+        message = (message as MessageContentDetail[])
+          .filter((c) => c.type === "text")
+          .map((c) => c.text)
+          .join("\n\n");
+      }
+      context = await this.contextGenerator.generate(message);
+    }
+    requestMessages = await chatHistory.requestMessages(
+      context ? [context.message] : undefined,
+    );
+    return requestMessages;
+  }
 }

packages/core/src/GlobalsHelper.ts

@@ -1,5 +1,4 @@
-import cl100k_base from "tiktoken/encoders/cl100k_base.json";
-import { Tiktoken } from "tiktoken/lite";
+import { encodingForModel } from "js-tiktoken";
 
 import { v4 as uuidv4 } from "uuid";
 import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
@@ -18,18 +17,17 @@ class GlobalsHelper {
   } | null = null;
 
   private initDefaultTokenizer() {
-    const encoding = new Tiktoken(
-      cl100k_base.bpe_ranks,
-      cl100k_base.special_tokens,
-      cl100k_base.pat_str,
-    );
+    const encoding = encodingForModel("text-embedding-ada-002"); // cl100k_base
 
     this.defaultTokenizer = {
       encode: (text: string) => {
-        return encoding.encode(text);
+        return new Uint32Array(encoding.encode(text));
       },
       decode: (tokens: Uint32Array) => {
-        return new TextDecoder().decode(encoding.decode(tokens));
+        const numberArray = Array.from(tokens);
+        const text = encoding.decode(numberArray);
+        const uint8Array = new TextEncoder().encode(text);
+        return new TextDecoder().decode(uint8Array);
       },
     };
   }

packages/core/src/Node.ts

@@ -1,4 +1,4 @@
-import crypto from "crypto"; // TODO Node dependency
+import CryptoJS from "crypto-js";
 import { v4 as uuidv4 } from "uuid";
 
 export enum NodeRelationship {
@@ -175,13 +175,13 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
    * @returns
    */
   generateHash() {
-    const hashFunction = crypto.createHash("sha256");
+    const hashFunction = CryptoJS.algo.SHA256.create();
     hashFunction.update(`type=${this.getType()}`);
     hashFunction.update(
       `startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`,
     );
     hashFunction.update(this.getContent(MetadataMode.ALL));
-    return hashFunction.digest("base64");
+    return hashFunction.finalize().toString(CryptoJS.enc.Base64);
   }
 
   getType(): ObjectType {
@@ -272,12 +272,13 @@ export class Document<T extends Metadata = Metadata> extends TextNode<T> {
   }
 }
 
-export function jsonToNode(json: any) {
-  if (!json.type) {
+export function jsonToNode(json: any, type?: ObjectType) {
+  if (!json.type && !type) {
     throw new Error("Node type not found");
   }
+  const nodeType = type || json.type;
 
-  switch (json.type) {
+  switch (nodeType) {
     case ObjectType.TEXT:
       return new TextNode(json);
     case ObjectType.INDEX:
@@ -285,7 +286,7 @@ export function jsonToNode(json: any) {
     case ObjectType.DOCUMENT:
       return new Document(json);
     default:
-      throw new Error(`Invalid node type: ${json.type}`);
+      throw new Error(`Invalid node type: ${nodeType}`);
   }
 }
 

packages/core/src/Prompt.ts

@@ -36,6 +36,15 @@ Answer:`;
 
 export type TextQaPrompt = typeof defaultTextQaPrompt;
 
+export const anthropicTextQaPrompt = ({ context = "", query = "" }) => {
+  return `Context information:
+<context>
+${context}
+</context>
+Given the context information and not prior knowledge, answer the query.
+Query: ${query}`;
+};
+
 /*
 DEFAULT_SUMMARY_PROMPT_TMPL = (
     "Write a summary of the following. Try to use only the "

packages/core/src/ServiceContext.ts

@@ -1,8 +1,8 @@
-import { BaseEmbedding, OpenAIEmbedding } from "./Embedding";
+import { CallbackManager } from "./callbacks/CallbackManager";
+import { BaseEmbedding, OpenAIEmbedding } from "./embeddings";
+import { LLM, OpenAI } from "./llm/LLM";
 import { NodeParser, SimpleNodeParser } from "./NodeParser";
 import { PromptHelper } from "./PromptHelper";
-import { CallbackManager } from "./callbacks/CallbackManager";
-import { LLM, OpenAI } from "./llm/LLM";
 
 /**
  * The ServiceContext is a collection of components that are used in different parts of the application.

packages/core/src/callbacks/CallbackManager.ts

@@ -30,7 +30,7 @@ export interface DefaultStreamToken {
     index: number;
     delta: {
       content?: string | null;
-      role?: "user" | "assistant" | "system" | "function";
+      role?: "user" | "assistant" | "system" | "function" | "tool";
     };
     finish_reason: string | null;
   }[];

packages/core/src/embeddings/ClipEmbedding.ts

@@ -0,0 +1,78 @@
+import { MultiModalEmbedding } from "./MultiModalEmbedding";
+import { ImageType, readImage } from "./utils";
+
+export enum ClipEmbeddingModelType {
+  XENOVA_CLIP_VIT_BASE_PATCH32 = "Xenova/clip-vit-base-patch32",
+  XENOVA_CLIP_VIT_BASE_PATCH16 = "Xenova/clip-vit-base-patch16",
+}
+
+export class ClipEmbedding extends MultiModalEmbedding {
+  modelType: ClipEmbeddingModelType =
+    ClipEmbeddingModelType.XENOVA_CLIP_VIT_BASE_PATCH16;
+
+  private tokenizer: any;
+  private processor: any;
+  private visionModel: any;
+  private textModel: any;
+
+  async getTokenizer() {
+    if (!this.tokenizer) {
+      const { AutoTokenizer } = await import("@xenova/transformers");
+      this.tokenizer = await AutoTokenizer.from_pretrained(this.modelType);
+    }
+    return this.tokenizer;
+  }
+
+  async getProcessor() {
+    if (!this.processor) {
+      const { AutoProcessor } = await import("@xenova/transformers");
+      this.processor = await AutoProcessor.from_pretrained(this.modelType);
+    }
+    return this.processor;
+  }
+
+  async getVisionModel() {
+    if (!this.visionModel) {
+      const { CLIPVisionModelWithProjection } = await import(
+        "@xenova/transformers"
+      );
+      this.visionModel = await CLIPVisionModelWithProjection.from_pretrained(
+        this.modelType,
+      );
+    }
+
+    return this.visionModel;
+  }
+
+  async getTextModel() {
+    if (!this.textModel) {
+      const { CLIPTextModelWithProjection } = await import(
+        "@xenova/transformers"
+      );
+      this.textModel = await CLIPTextModelWithProjection.from_pretrained(
+        this.modelType,
+      );
+    }
+
+    return this.textModel;
+  }
+
+  async getImageEmbedding(image: ImageType): Promise<number[]> {
+    const loadedImage = await readImage(image);
+    const imageInputs = await (await this.getProcessor())(loadedImage);
+    const { image_embeds } = await (await this.getVisionModel())(imageInputs);
+    return image_embeds.data;
+  }
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    const textInputs = await (
+      await this.getTokenizer()
+    )([text], { padding: true, truncation: true });
+    const { text_embeds } = await (await this.getTextModel())(textInputs);
+    return text_embeds.data;
+  }
+
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getTextEmbedding(query);
+  }
+}

packages/core/src/embeddings/MultiModalEmbedding.ts

@@ -0,0 +1,17 @@
+import { BaseEmbedding } from "./types";
+import { ImageType } from "./utils";
+
+/*
+ * Base class for Multi Modal embeddings.
+ */
+
+export abstract class MultiModalEmbedding extends BaseEmbedding {
+  abstract getImageEmbedding(images: ImageType): Promise<number[]>;
+
+  async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
+    // Embed the input sequence of images asynchronously.
+    return Promise.all(
+      images.map((imgFilePath) => this.getImageEmbedding(imgFilePath)),
+    );
+  }
+}

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -0,0 +1,92 @@
+import { ClientOptions as OpenAIClientOptions } from "openai";
+import {
+  AzureOpenAIConfig,
+  getAzureBaseUrl,
+  getAzureConfigFromEnv,
+  getAzureModel,
+  shouldUseAzure,
+} from "../llm/azure";
+import { OpenAISession, getOpenAISession } from "../llm/openai";
+import { BaseEmbedding } from "./types";
+
+export enum OpenAIEmbeddingModelType {
+  TEXT_EMBED_ADA_002 = "text-embedding-ada-002",
+}
+
+export class OpenAIEmbedding extends BaseEmbedding {
+  model: OpenAIEmbeddingModelType;
+
+  // OpenAI session params
+  apiKey?: string = undefined;
+  maxRetries: number;
+  timeout?: number;
+  additionalSessionOptions?: Omit<
+    Partial<OpenAIClientOptions>,
+    "apiKey" | "maxRetries" | "timeout"
+  >;
+
+  session: OpenAISession;
+
+  constructor(init?: Partial<OpenAIEmbedding> & { azure?: AzureOpenAIConfig }) {
+    super();
+
+    this.model = OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002;
+
+    this.maxRetries = init?.maxRetries ?? 10;
+    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
+    this.additionalSessionOptions = init?.additionalSessionOptions;
+
+    if (init?.azure || shouldUseAzure()) {
+      const azureConfig = getAzureConfigFromEnv({
+        ...init?.azure,
+        model: getAzureModel(this.model),
+      });
+
+      if (!azureConfig.apiKey) {
+        throw new Error(
+          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable.",
+        );
+      }
+
+      this.apiKey = azureConfig.apiKey;
+      this.session =
+        init?.session ??
+        getOpenAISession({
+          azure: true,
+          apiKey: this.apiKey,
+          baseURL: getAzureBaseUrl(azureConfig),
+          maxRetries: this.maxRetries,
+          timeout: this.timeout,
+          defaultQuery: { "api-version": azureConfig.apiVersion },
+          ...this.additionalSessionOptions,
+        });
+    } else {
+      this.apiKey = init?.apiKey ?? undefined;
+      this.session =
+        init?.session ??
+        getOpenAISession({
+          apiKey: this.apiKey,
+          maxRetries: this.maxRetries,
+          timeout: this.timeout,
+          ...this.additionalSessionOptions,
+        });
+    }
+  }
+
+  private async getOpenAIEmbedding(input: string) {
+    const { data } = await this.session.openai.embeddings.create({
+      model: this.model,
+      input,
+    });
+
+    return data[0].embedding;
+  }
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    return this.getOpenAIEmbedding(text);
+  }
+
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getOpenAIEmbedding(query);
+  }
+}

packages/core/src/embeddings/index.ts

@@ -0,0 +1,5 @@
+export * from "./ClipEmbedding";
+export * from "./MultiModalEmbedding";
+export * from "./OpenAIEmbedding";
+export * from "./types";
+export * from "./utils";

packages/core/src/embeddings/types.ts

@@ -0,0 +1,24 @@
+import { similarity } from "./utils";
+
+/**
+ * Similarity type
+ * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
+ */
+export enum SimilarityType {
+  DEFAULT = "cosine",
+  DOT_PRODUCT = "dot_product",
+  EUCLIDEAN = "euclidean",
+}
+
+export abstract class BaseEmbedding {
+  similarity(
+    embedding1: number[],
+    embedding2: number[],
+    mode: SimilarityType = SimilarityType.DEFAULT,
+  ): number {
+    return similarity(embedding1, embedding2, mode);
+  }
+
+  abstract getTextEmbedding(text: string): Promise<number[]>;
+  abstract getQueryEmbedding(query: string): Promise<number[]>;
+}

packages/core/src/embeddings/utils.ts

@@ -1,33 +1,16 @@
-import { ClientOptions as OpenAIClientOptions } from "openai";
-
-import { DEFAULT_SIMILARITY_TOP_K } from "./constants";
-import {
-  AzureOpenAIConfig,
-  getAzureBaseUrl,
-  getAzureConfigFromEnv,
-  getAzureModel,
-  shouldUseAzure,
-} from "./llm/azure";
-import { OpenAISession, getOpenAISession } from "./llm/openai";
-import { VectorStoreQueryMode } from "./storage/vectorStore/types";
-
-/**
- * Similarity type
- * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
- */
-export enum SimilarityType {
-  DEFAULT = "cosine",
-  DOT_PRODUCT = "dot_product",
-  EUCLIDEAN = "euclidean",
-}
+import _ from "lodash";
+import { DEFAULT_SIMILARITY_TOP_K } from "../constants";
+import { VectorStoreQueryMode } from "../storage";
+import { SimilarityType } from "./types";
 
 /**
  * The similarity between two embeddings.
  * @param embedding1
  * @param embedding2
  * @param mode
- * @returns similartiy score with higher numbers meaning the two embeddings are more similar
+ * @returns similarity score with higher numbers meaning the two embeddings are more similar
  */
+
 export function similarity(
   embedding1: number[],
   embedding2: number[],
@@ -42,7 +25,6 @@ export function similarity(
   // will probably cause some avoidable loss of floating point precision
   // ml-distance is worth watching although they currently also use the naive
   // formulas
-
   function norm(x: number[]): number {
     let result = 0;
     for (let i = 0; i < x.length; i++) {
@@ -201,98 +183,14 @@ export function getTopKMMREmbeddings(
 
   return [resultSimilarities, resultIds];
 }
-
-export abstract class BaseEmbedding {
-  similarity(
-    embedding1: number[],
-    embedding2: number[],
-    mode: SimilarityType = SimilarityType.DEFAULT,
-  ): number {
-    return similarity(embedding1, embedding2, mode);
-  }
-
-  abstract getTextEmbedding(text: string): Promise<number[]>;
-  abstract getQueryEmbedding(query: string): Promise<number[]>;
-}
-
-enum OpenAIEmbeddingModelType {
-  TEXT_EMBED_ADA_002 = "text-embedding-ada-002",
-}
-
-export class OpenAIEmbedding extends BaseEmbedding {
-  model: OpenAIEmbeddingModelType;
-
-  // OpenAI session params
-  apiKey?: string = undefined;
-  maxRetries: number;
-  timeout?: number;
-  additionalSessionOptions?: Omit<
-    Partial<OpenAIClientOptions>,
-    "apiKey" | "maxRetries" | "timeout"
-  >;
-
-  session: OpenAISession;
-
-  constructor(init?: Partial<OpenAIEmbedding> & { azure?: AzureOpenAIConfig }) {
-    super();
-
-    this.model = OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002;
-
-    this.maxRetries = init?.maxRetries ?? 10;
-    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
-    this.additionalSessionOptions = init?.additionalSessionOptions;
-
-    if (init?.azure || shouldUseAzure()) {
-      const azureConfig = getAzureConfigFromEnv({
-        ...init?.azure,
-        model: getAzureModel(this.model),
-      });
-
-      if (!azureConfig.apiKey) {
-        throw new Error(
-          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable.",
-        );
-      }
-
-      this.apiKey = azureConfig.apiKey;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          azure: true,
-          apiKey: this.apiKey,
-          baseURL: getAzureBaseUrl(azureConfig),
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          defaultQuery: { "api-version": azureConfig.apiVersion },
-          ...this.additionalSessionOptions,
-        });
-    } else {
-      this.apiKey = init?.apiKey ?? undefined;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          apiKey: this.apiKey,
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          ...this.additionalSessionOptions,
-        });
-    }
-  }
-
-  private async getOpenAIEmbedding(input: string) {
-    const { data } = await this.session.openai.embeddings.create({
-      model: this.model,
-      input,
-    });
-
-    return data[0].embedding;
-  }
-
-  async getTextEmbedding(text: string): Promise<number[]> {
-    return this.getOpenAIEmbedding(text);
-  }
-
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getOpenAIEmbedding(query);
+export async function readImage(input: ImageType) {
+  const { RawImage } = await import("@xenova/transformers");
+  if (input instanceof Blob) {
+    return await RawImage.fromBlob(input);
+  } else if (_.isString(input) || input instanceof URL) {
+    return await RawImage.fromURL(input);
+  } else {
+    throw new Error(`Unsupported input type: ${typeof input}`);
   }
 }
+export type ImageType = string | Blob | URL;

packages/core/src/index.ts

@@ -1,11 +1,6 @@
-export * from "./callbacks/CallbackManager";
 export * from "./ChatEngine";
 export * from "./ChatHistory";
-export * from "./constants";
-export * from "./Embedding";
 export * from "./GlobalsHelper";
-export * from "./indices";
-export * from "./llm/LLM";
 export * from "./Node";
 export * from "./NodeParser";
 export * from "./OutputParser";
@@ -13,16 +8,24 @@ export * from "./Prompt";
 export * from "./PromptHelper";
 export * from "./QueryEngine";
 export * from "./QuestionGenerator";
-export * from "./readers/base";
-export * from "./readers/CSVReader";
-export * from "./readers/MarkdownReader";
-export * from "./readers/NotionReader";
-export * from "./readers/PDFReader";
-export * from "./readers/SimpleDirectoryReader";
 export * from "./Response";
 export * from "./ResponseSynthesizer";
 export * from "./Retriever";
 export * from "./ServiceContext";
-export * from "./storage";
 export * from "./TextSplitter";
 export * from "./Tool";
+export * from "./callbacks/CallbackManager";
+export * from "./constants";
+export * from "./embeddings";
+export * from "./indices";
+export * from "./llm/LLM";
+export * from "./readers/AssemblyAI";
+export * from "./readers/CSVReader";
+export * from "./readers/HTMLReader";
+export * from "./readers/MarkdownReader";
+export * from "./readers/NotionReader";
+export * from "./readers/PDFReader";
+export * from "./readers/SimpleDirectoryReader";
+export * from "./readers/SimpleMongoReader";
+export * from "./readers/base";
+export * from "./storage";

packages/core/src/indices/vectorStore/VectorIndexRetriever.ts

@@ -32,7 +32,11 @@ export class VectorIndexRetriever implements BaseRetriever {
     this.similarityTopK = similarityTopK ?? DEFAULT_SIMILARITY_TOP_K;
   }
 
-  async retrieve(query: string, parentEvent?: Event, preFilters?: unknown): Promise<NodeWithScore[]> {
+  async retrieve(
+    query: string,
+    parentEvent?: Event,
+    preFilters?: unknown,
+  ): Promise<NodeWithScore[]> {
     const queryEmbedding =
       await this.serviceContext.embedModel.getQueryEmbedding(query);
 

packages/core/src/llm/LLM.ts

@@ -8,12 +8,13 @@ import {
   StreamCallbackResponse,
 } from "../callbacks/CallbackManager";
 
+import { ChatCompletionMessageParam } from "openai/resources";
 import { LLMOptions } from "portkey-ai";
 import { globalsHelper, Tokenizers } from "../GlobalsHelper";
 import {
-  AnthropicSession,
   ANTHROPIC_AI_PROMPT,
   ANTHROPIC_HUMAN_PROMPT,
+  AnthropicSession,
   getAnthropicSession,
 } from "./anthropic";
 import {
@@ -36,7 +37,7 @@ export type MessageType =
   | "memory";
 
 export interface ChatMessage {
-  content: string;
+  content: any;
   role: MessageType;
 }
 
@@ -102,11 +103,14 @@ export interface LLM {
 export const GPT4_MODELS = {
   "gpt-4": { contextWindow: 8192 },
   "gpt-4-32k": { contextWindow: 32768 },
+  "gpt-4-1106-preview": { contextWindow: 128000 },
+  "gpt-4-vision-preview": { contextWindow: 8192 },
 };
 
-export const TURBO_MODELS = {
+export const GPT35_MODELS = {
   "gpt-3.5-turbo": { contextWindow: 4096 },
   "gpt-3.5-turbo-16k": { contextWindow: 16384 },
+  "gpt-3.5-turbo-1106": { contextWindow: 16384 },
 };
 
 /**
@@ -114,7 +118,7 @@ export const TURBO_MODELS = {
  */
 export const ALL_AVAILABLE_OPENAI_MODELS = {
   ...GPT4_MODELS,
-  ...TURBO_MODELS,
+  ...GPT35_MODELS,
 };
 
 /**
@@ -250,10 +254,13 @@ export class OpenAI implements LLM {
       model: this.model,
       temperature: this.temperature,
       max_tokens: this.maxTokens,
-      messages: messages.map((message) => ({
-        role: this.mapMessageType(message.role),
-        content: message.content,
-      })),
+      messages: messages.map(
+        (message) =>
+          ({
+            role: this.mapMessageType(message.role),
+            content: message.content,
+          }) as ChatCompletionMessageParam,
+      ),
       top_p: this.topP,
       ...this.additionalChatOptions,
     };
@@ -298,10 +305,13 @@ export class OpenAI implements LLM {
       model: this.model,
       temperature: this.temperature,
       max_tokens: this.maxTokens,
-      messages: messages.map((message) => ({
-        role: this.mapMessageType(message.role),
-        content: message.content,
-      })),
+      messages: messages.map(
+        (message) =>
+          ({
+            role: this.mapMessageType(message.role),
+            content: message.content,
+          }) as ChatCompletionMessageParam,
+      ),
       top_p: this.topP,
       ...this.additionalChatOptions,
     };
@@ -367,10 +377,10 @@ export const ALL_AVAILABLE_LLAMADEUCE_MODELS = {
   "Llama-2-70b-chat-4bit": {
     contextWindow: 4096,
     replicateApi:
-      "replicate/llama70b-v2-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1",
+      "meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
     //^ Model is based off of exllama 4bit.
   },
-  "Llama-2-13b-chat": {
+  "Llama-2-13b-chat-old": {
     contextWindow: 4096,
     replicateApi:
       "a16z-infra/llama13b-v2-chat:df7690f1994d94e96ad9d568eac121aecf50684a0b0963b25a41cc40061269e5",
@@ -379,9 +389,9 @@ export const ALL_AVAILABLE_LLAMADEUCE_MODELS = {
   "Llama-2-13b-chat-4bit": {
     contextWindow: 4096,
     replicateApi:
-      "a16z-infra/llama13b-v2-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
+      "meta/llama-2-13b-chat:f4e2de70d66816a838a89eeeb621910adffb0dd0baba3976c96980970978018d",
   },
-  "Llama-2-7b-chat": {
+  "Llama-2-7b-chat-old": {
     contextWindow: 4096,
     replicateApi:
       "a16z-infra/llama7b-v2-chat:4f0a4744c7295c024a1de15e1a63c880d3da035fa1f49bfd344fe076074c8eea",
@@ -393,7 +403,7 @@ export const ALL_AVAILABLE_LLAMADEUCE_MODELS = {
   "Llama-2-7b-chat-4bit": {
     contextWindow: 4096,
     replicateApi:
-      "a16z-infra/llama7b-v2-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc",
+      "meta/llama-2-7b-chat:13c3cdee13ee059ab779f0291d29054dab00a47dad8261375654de5540165fb0",
   },
 };
 
@@ -405,6 +415,8 @@ export enum DeuceChatStrategy {
   // Unfortunately any string only API won't support these properly.
   REPLICATE4BIT = "replicate4bit",
   //^ To satisfy Replicate's 4 bit models' requirements where they also insert some INST tags
+  REPLICATE4BITWNEWLINES = "replicate4bitwnewlines",
+  //^ Replicate's documentation recommends using newlines: https://replicate.com/blog/how-to-prompt-llama
 }
 
 /**
@@ -424,7 +436,7 @@ export class LlamaDeuce implements LLM {
     this.chatStrategy =
       init?.chatStrategy ??
       (this.model.endsWith("4bit")
-        ? DeuceChatStrategy.REPLICATE4BIT // With the newer A16Z/Replicate models they do the system message themselves.
+        ? DeuceChatStrategy.REPLICATE4BITWNEWLINES // With the newer Replicate models they do the system message themselves.
         : DeuceChatStrategy.METAWBOS); // With BOS and EOS seems to work best, although they all have problems past a certain point
     this.temperature = init?.temperature ?? 0.1; // minimum temperature is 0.01 for Replicate endpoint
     this.topP = init?.topP ?? 1;
@@ -458,7 +470,15 @@ export class LlamaDeuce implements LLM {
     } else if (this.chatStrategy === DeuceChatStrategy.METAWBOS) {
       return this.mapMessagesToPromptMeta(messages, { withBos: true });
     } else if (this.chatStrategy === DeuceChatStrategy.REPLICATE4BIT) {
-      return this.mapMessagesToPromptMeta(messages, { replicate4Bit: true });
+      return this.mapMessagesToPromptMeta(messages, {
+        replicate4Bit: true,
+        withNewlines: true,
+      });
+    } else if (this.chatStrategy === DeuceChatStrategy.REPLICATE4BITWNEWLINES) {
+      return this.mapMessagesToPromptMeta(messages, {
+        replicate4Bit: true,
+        withNewlines: true,
+      });
     } else {
       return this.mapMessagesToPromptMeta(messages);
     }
@@ -493,9 +513,17 @@ export class LlamaDeuce implements LLM {
 
   mapMessagesToPromptMeta(
     messages: ChatMessage[],
-    opts?: { withBos?: boolean; replicate4Bit?: boolean },
+    opts?: {
+      withBos?: boolean;
+      replicate4Bit?: boolean;
+      withNewlines?: boolean;
+    },
   ) {
-    const { withBos = false, replicate4Bit = false } = opts ?? {};
+    const {
+      withBos = false,
+      replicate4Bit = false,
+      withNewlines = false,
+    } = opts ?? {};
     const DEFAULT_SYSTEM_PROMPT = `You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
 
 If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.`;
@@ -543,11 +571,18 @@ If a question does not make any sense, or is not factually coherent, explain why
     return {
       prompt: messages.reduce((acc, message, index) => {
         if (index % 2 === 0) {
-          return `${acc}${
-            withBos ? BOS : ""
-          }${B_INST} ${message.content.trim()} ${E_INST}`;
+          return (
+            `${acc}${
+              withBos ? BOS : ""
+            }${B_INST} ${message.content.trim()} ${E_INST}` +
+            (withNewlines ? "\n" : "")
+          );
         } else {
-          return `${acc} ${message.content.trim()} ` + (withBos ? EOS : ""); // Yes, the EOS comes after the space. This is not a mistake.
+          return (
+            `${acc} ${message.content.trim()}` +
+            (withNewlines ? "\n" : " ") +
+            (withBos ? EOS : "")
+          ); // Yes, the EOS comes after the space. This is not a mistake.
         }
       }, ""),
       systemPrompt,
@@ -604,7 +639,7 @@ If a question does not make any sense, or is not factually coherent, explain why
 
 export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
   // both models have 100k context window, see https://docs.anthropic.com/claude/reference/selecting-a-model
-  "claude-2": { contextWindow: 100000 },
+  "claude-2": { contextWindow: 200000 },
   "claude-instant-1": { contextWindow: 100000 },
 };
 
@@ -648,7 +683,7 @@ export class Anthropic implements LLM {
 
     this.callbackManager = init?.callbackManager;
   }
-  
+
   tokens(messages: ChatMessage[]): number {
     throw new Error("Method not implemented.");
   }
@@ -670,10 +705,12 @@ export class Anthropic implements LLM {
         return (
           acc +
           `${
-            message.role === "assistant"
-              ? ANTHROPIC_AI_PROMPT
-              : ANTHROPIC_HUMAN_PROMPT
-          } ${message.content} `
+            message.role === "system"
+              ? ""
+              : message.role === "assistant"
+                ? ANTHROPIC_AI_PROMPT + " "
+                : ANTHROPIC_HUMAN_PROMPT + " "
+          }${message.content.trim()}`
         );
       }, "") + ANTHROPIC_AI_PROMPT
     );
@@ -694,6 +731,7 @@ export class Anthropic implements LLM {
       }
       return this.streamChat(messages, parentEvent) as R;
     }
+
     //Non-streaming
     const response = await this.session.anthropic.completions.create({
       model: this.model,

packages/core/src/llm/openai.ts

@@ -24,7 +24,10 @@ export class OpenAISession {
     if (options.azure) {
       this.openai = new AzureOpenAI(options);
     } else {
-      this.openai = new OpenAI(options);
+      this.openai = new OpenAI({
+        ...options,
+        // defaultHeaders: { "OpenAI-Beta": "assistants=v1" },
+      });
     }
   }
 }

packages/core/src/llm/portkey.ts

@@ -1,9 +1,12 @@
 import _ from "lodash";
 import { LLMOptions, Portkey } from "portkey-ai";
 
-export const readEnv = (env: string, default_val?: string): string | undefined => {
-  if (typeof process !== 'undefined') {
-      return process.env?.[env] ?? default_val;
+export const readEnv = (
+  env: string,
+  default_val?: string,
+): string | undefined => {
+  if (typeof process !== "undefined") {
+    return process.env?.[env] ?? default_val;
   }
   return default_val;
 };
@@ -12,23 +15,23 @@ interface PortkeyOptions {
   apiKey?: string;
   baseURL?: string;
   mode?: string;
-  llms?: [LLMOptions] | null
+  llms?: [LLMOptions] | null;
 }
 
 export class PortkeySession {
   portkey: Portkey;
 
-  constructor(options:PortkeyOptions = {}) {
+  constructor(options: PortkeyOptions = {}) {
     if (!options.apiKey) {
-      options.apiKey = readEnv('PORTKEY_API_KEY')
+      options.apiKey = readEnv("PORTKEY_API_KEY");
     }
 
     if (!options.baseURL) {
-      options.baseURL = readEnv('PORTKEY_BASE_URL', "https://api.portkey.ai")
+      options.baseURL = readEnv("PORTKEY_BASE_URL", "https://api.portkey.ai");
     }
 
     this.portkey = new Portkey({});
-    this.portkey.llms = [{}]
+    this.portkey.llms = [{}];
     if (!options.apiKey) {
       throw new Error("Set Portkey ApiKey in PORTKEY_API_KEY env variable");
     }
@@ -59,4 +62,3 @@ export function getPortkeySession(options: PortkeyOptions = {}) {
   }
   return session;
 }
-

packages/core/src/readers/AssemblyAI.ts

@@ -0,0 +1,148 @@
+import {
+  AssemblyAI,
+  BaseServiceParams,
+  CreateTranscriptParameters,
+  SubtitleFormat,
+  TranscriptParagraph,
+  TranscriptSentence,
+} from "assemblyai";
+import { Document } from "../Node";
+import { BaseReader } from "./base";
+
+type AssemblyAIOptions = Partial<BaseServiceParams>;
+
+/**
+ * Base class for AssemblyAI Readers.
+ */
+abstract class AssemblyAIReader implements BaseReader {
+  protected client: AssemblyAI;
+
+  /**
+   * Creates a new AssemblyAI Reader.
+   * @param assemblyAIOptions The options to configure the AssemblyAI Reader.
+   * Configure the `assemblyAIOptions.apiKey` with your AssemblyAI API key, or configure it as the `ASSEMBLYAI_API_KEY` environment variable.
+   */
+  constructor(assemblyAIOptions?: AssemblyAIOptions) {
+    let options = assemblyAIOptions;
+    if (!options) {
+      options = {};
+    }
+    if (!options.apiKey) {
+      options.apiKey = process.env.ASSEMBLYAI_API_KEY;
+    }
+    if (!options.apiKey) {
+      throw new Error("No AssemblyAI API key provided. Pass an `apiKey` option, or configure the `ASSEMBLYAI_API_KEY` environment variable.");
+    }
+
+    this.client = new AssemblyAI(options as BaseServiceParams);
+  }
+
+  abstract loadData(...args: any[]): Promise<Document[]>;
+
+  protected async getOrCreateTranscript(params: CreateTranscriptParameters | string) {
+    if (typeof params === "string") {
+      return await this.client.transcripts.get(params);
+    }
+    else {
+      return await this.client.transcripts.create(params);
+    }
+  }
+
+  protected async getTranscriptId(params: CreateTranscriptParameters | string) {
+    if (typeof params === "string") {
+      return params;
+    }
+    else {
+      return (await this.client.transcripts.create(params)).id;
+    }
+  }
+}
+
+/**
+ * Creates and reads the transcript as a document using AssemblyAI.
+ */
+class AudioTranscriptReader extends AssemblyAIReader {
+  /**
+   * Creates or gets a transcript and loads the transcript as a document using AssemblyAI.
+   * @param params The parameters to create or get the transcript.
+   * @returns A promise that resolves to a single document containing the transcript text.
+   */
+  async loadData(params: CreateTranscriptParameters | string): Promise<Document[]> {
+    const transcript = await this.getOrCreateTranscript(params);
+    return [
+      new Document({ text: transcript.text || undefined }),
+    ];
+  }
+}
+
+/**
+ * Creates a transcript and returns a document for each paragraph.
+ */
+class AudioTranscriptParagraphsReader extends AssemblyAIReader {
+  /**
+   * Creates or gets a transcript, and returns a document for each paragraph.
+   * @param params The parameters to create or get the transcript.
+   * @returns A promise that resolves to an array of documents, each containing a paragraph of the transcript.
+   */
+  async loadData(params: CreateTranscriptParameters | string): Promise<Document[]> {
+    let transcriptId = await this.getTranscriptId(params);
+    const paragraphsResponse = await this.client.transcripts.paragraphs(
+      transcriptId
+    );
+    return paragraphsResponse.paragraphs.map((p: TranscriptParagraph) =>
+      new Document({ text: p.text }),
+    );
+  }
+}
+
+/**
+ * Creates a transcript and returns a document for each sentence.
+ */
+class AudioTranscriptSentencesReader extends AssemblyAIReader {
+  /**
+   * Creates or gets a transcript, and returns a document for each sentence.
+   * @param params The parameters to create or get the transcript.
+   * @returns A promise that resolves to an array of documents, each containing a sentence of the transcript.
+   */
+  async loadData(params: CreateTranscriptParameters | string): Promise<Document[]> {
+    let transcriptId = await this.getTranscriptId(params);
+    const sentencesResponse = await this.client.transcripts.sentences(
+      transcriptId
+    );
+    return sentencesResponse.sentences.map((p: TranscriptSentence) =>
+      new Document({ text: p.text }),
+    );
+  }
+}
+
+/**
+ * Creates a transcript and reads subtitles for the transcript as `srt` or `vtt` format.
+ */
+class AudioSubtitlesReader extends AssemblyAIReader {
+  /**
+   * Creates or gets a transcript and reads subtitles for the transcript as `srt` or `vtt` format.
+   * @param params The parameters to create or get the transcript.
+   * @param subtitleFormat The format of the subtitles, either `srt` or `vtt`.
+   * @returns A promise that resolves a document containing the subtitles as the page content.
+   */
+  async loadData(
+    params: CreateTranscriptParameters | string,
+    subtitleFormat: SubtitleFormat = 'srt'
+  ): Promise<Document[]> {
+    let transcriptId = await this.getTranscriptId(params);
+    const subtitles = await this.client.transcripts.subtitles(transcriptId, subtitleFormat);
+    return [new Document({ text: subtitles })];
+  }
+}
+
+export {
+  AudioTranscriptReader,
+  AudioTranscriptParagraphsReader,
+  AudioTranscriptSentencesReader,
+  AudioSubtitlesReader,
+}
+export type {
+  AssemblyAIOptions,
+  CreateTranscriptParameters,
+  SubtitleFormat
+}

packages/core/src/readers/HTMLReader.ts

@@ -0,0 +1,77 @@
+import { Document } from "../Node";
+import { DEFAULT_FS } from "../storage/constants";
+import { GenericFileSystem } from "../storage/FileSystem";
+import { BaseReader } from "./base";
+
+/**
+ * Extract the significant text from an arbitrary HTML document.
+ * The contents of any head, script, style, and xml tags are removed completely.
+ * The URLs for a[href] tags are extracted, along with the inner text of the tag.
+ * All other tags are removed, and the inner text is kept intact.
+ * Html entities (e.g., &) are not decoded.
+ */
+export class HTMLReader implements BaseReader {
+  /**
+   * Public method for this reader.
+   * Required by BaseReader interface.
+   * @param file Path/name of the file to be loaded.
+   * @param fs fs wrapper interface for getting the file content.
+   * @returns Promise<Document[]> A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
+   */
+  async loadData(
+    file: string,
+    fs: GenericFileSystem = DEFAULT_FS,
+  ): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file, "utf-8");
+    const htmlOptions = this.getOptions();
+    const content = await this.parseContent(dataBuffer, htmlOptions);
+    return [new Document({ text: content, id_: file })];
+  }
+
+  /**
+   * Wrapper for string-strip-html usage.
+   * @param html Raw HTML content to be parsed.
+   * @param options An object of options for the underlying library
+   * @see getOptions
+   * @returns The HTML content, stripped of unwanted tags and attributes
+   */
+  async parseContent(html: string, options: any = {}): Promise<string> {
+    const { stripHtml } = await import("string-strip-html"); // ESM only
+    return stripHtml(html).result;
+  }
+
+  /**
+   * Wrapper for our configuration options passed to string-strip-html library
+   * @see https://codsen.com/os/string-strip-html/examples
+   * @returns An object of options for the underlying library
+   */
+  getOptions() {
+    return {
+      skipHtmlDecoding: true,
+      stripTogetherWithTheirContents: [
+        "script", // default
+        "style", // default
+        "xml", // default
+        "head", // <-- custom-added
+      ],
+      // Keep the URLs for embedded links
+      // cb: (tag: any, deleteFrom: number, deleteTo: number, insert: string, rangesArr: any, proposedReturn: string) => {
+      //   let temp;
+      //   if (
+      //     tag.name === "a" &&
+      //     tag.attributes &&
+      //     tag.attributes.some((attr: any) => {
+      //       if (attr.name === "href") {
+      //         temp = attr.value;
+      //         return true;
+      //       }
+      //     })
+      //   ) {
+      //     rangesArr.push([deleteFrom, deleteTo, `${temp} ${insert || ""}`]);
+      //   } else {
+      //     rangesArr.push(proposedReturn);
+      //   }
+      // },
+    };
+  }
+}

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -1,12 +1,25 @@
 import _ from "lodash";
 import { Document } from "../Node";
-import { DEFAULT_FS } from "../storage/constants";
 import { CompleteFileSystem, walk } from "../storage/FileSystem";
-import { BaseReader } from "./base";
+import { DEFAULT_FS } from "../storage/constants";
 import { PapaCSVReader } from "./CSVReader";
 import { DocxReader } from "./DocxReader";
+import { HTMLReader } from "./HTMLReader";
 import { MarkdownReader } from "./MarkdownReader";
 import { PDFReader } from "./PDFReader";
+import { BaseReader } from "./base";
+
+type ReaderCallback = (
+  category: "file" | "directory",
+  name: string,
+  status: ReaderStatus,
+  message?: string,
+) => boolean;
+enum ReaderStatus {
+  STARTED = 0,
+  COMPLETE,
+  ERROR,
+}
 
 /**
  * Read a .txt file
@@ -21,12 +34,14 @@ export class TextFileReader implements BaseReader {
   }
 }
 
-const FILE_EXT_TO_READER: Record<string, BaseReader> = {
+export const FILE_EXT_TO_READER: Record<string, BaseReader> = {
   txt: new TextFileReader(),
   pdf: new PDFReader(),
   csv: new PapaCSVReader(),
   md: new MarkdownReader(),
   docx: new DocxReader(),
+  htm: new HTMLReader(),
+  html: new HTMLReader(),
 };
 
 export type SimpleDirectoryReaderLoadDataProps = {
@@ -37,20 +52,37 @@ export type SimpleDirectoryReaderLoadDataProps = {
 };
 
 /**
- * Read all of the documents in a directory. Currently supports PDF and TXT files.
+ * Read all of the documents in a directory.
+ * By default, supports the list of file types
+ * in the FILE_EXIT_TO_READER map.
  */
 export class SimpleDirectoryReader implements BaseReader {
+  constructor(private observer?: ReaderCallback) {}
+
   async loadData({
     directoryPath,
     fs = DEFAULT_FS as CompleteFileSystem,
     defaultReader = new TextFileReader(),
     fileExtToReader = FILE_EXT_TO_READER,
   }: SimpleDirectoryReaderLoadDataProps): Promise<Document[]> {
+    // Observer can decide to skip the directory
+    if (
+      !this.doObserverCheck("directory", directoryPath, ReaderStatus.STARTED)
+    ) {
+      return [];
+    }
+
     let docs: Document[] = [];
     for await (const filePath of walk(fs, directoryPath)) {
       try {
         const fileExt = _.last(filePath.split(".")) || "";
 
+        // Observer can decide to skip each file
+        if (!this.doObserverCheck("file", filePath, ReaderStatus.STARTED)) {
+          // Skip this file
+          continue;
+        }
+
         let reader = null;
 
         if (fileExt in fileExtToReader) {
@@ -58,16 +90,52 @@ export class SimpleDirectoryReader implements BaseReader {
         } else if (!_.isNil(defaultReader)) {
           reader = defaultReader;
         } else {
-          console.warn(`No reader for file extension of ${filePath}`);
+          const msg = `No reader for file extension of ${filePath}`;
+          console.warn(msg);
+
+          // In an error condition, observer's false cancels the whole process.
+          if (
+            !this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)
+          ) {
+            return [];
+          }
+
           continue;
         }
 
         const fileDocs = await reader.loadData(filePath, fs);
-        docs.push(...fileDocs);
+
+        // Observer can still cancel addition of the resulting docs from this file
+        if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) {
+          docs.push(...fileDocs);
+        }
       } catch (e) {
-        console.error(`Error reading file ${filePath}: ${e}`);
+        const msg = `Error reading file ${filePath}: ${e}`;
+        console.error(msg);
+
+        // In an error condition, observer's false cancels the whole process.
+        if (!this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)) {
+          return [];
+        }
       }
     }
+
+    // After successful import of all files, directory completion
+    // is only a notification for observer, cannot be cancelled.
+    this.doObserverCheck("directory", directoryPath, ReaderStatus.COMPLETE);
+
     return docs;
   }
+
+  private doObserverCheck(
+    category: "file" | "directory",
+    name: string,
+    status: ReaderStatus,
+    message?: string,
+  ): boolean {
+    if (this.observer) {
+      return this.observer(category, name, status, message);
+    }
+    return true;
+  }
 }

packages/core/src/readers/SimpleMongoReader.ts

@@ -1,5 +1,5 @@
 import { MongoClient } from "mongodb";
-import { Document } from "../Node";
+import { Document, Metadata } from "../Node";
 import { BaseReader } from "./base";
 
 /**
@@ -13,39 +13,70 @@ export class SimpleMongoReader implements BaseReader {
   }
 
   /**
-   * Loads data from MongoDB collection
-   * @param {string} db_name - The name of the database to load.
-   * @param {string} collection_name - The name of the collection to load.
-   * @param {Number} [max_docs = 0] - Maximum number of documents to return. 0 means no limit.
-   * @param {Record<string, any>} [query_dict={}] - Specific query, as specified by MongoDB NodeJS documentation.
-   * @param {Record<string, any>} [query_options={}] - Specific query options, as specified by MongoDB NodeJS documentation.
-   * @param {Record<string, any>} [projection = {}] - Projection options, as specified by MongoDB NodeJS documentation.
-   * @returns {Promise<Document[]>}
+   * Flattens an array of strings or string arrays into a single-dimensional array of strings.
+   * @param texts - The array of strings or string arrays to flatten.
+   * @returns The flattened array of strings.
    */
-  async loadData(
-    db_name: string,
-    collection_name: string,
-    max_docs = 0,
-    //For later: Think about whether we want to pass generic objects in...
-    query_dict: Record<string, any> = {},
-    query_options: Record<string, any> = {},
-    projection: Record<string, any> = {},
-  ): Promise<Document[]> {
-    //Get items from collection using built-in functions
-    const cursor: Partial<Document>[] = await this.client
-      .db(db_name)
-      .collection(collection_name)
-      .find(query_dict, query_options)
-      .limit(max_docs)
-      .project(projection)
-      .toArray();
+  private flatten(texts: Array<string | string[]>): string[] {
+    return texts.reduce<string[]>(
+      (result, text) => result.concat(text instanceof Array ? text : [text]),
+      [],
+    );
+  }
+
+  /**
+   * Loads data from MongoDB collection
+   * @param {string} dbName - The name of the database to load.
+   * @param {string} collectionName - The name of the collection to load.
+   * @param {string[]} fieldNames - An array of field names to retrieve from each document. Defaults to ["text"].
+   * @param {string} separator - The separator to join multiple field values. Defaults to an empty string.
+   * @param {Record<string, any>} filterQuery - Specific query, as specified by MongoDB NodeJS documentation.
+   * @param {Number} maxDocs - The maximum number of documents to retrieve. Defaults to 0 (retrieve all documents).
+   * @param {string[]} metadataNames - An optional array of metadata field names. If specified extracts this information as metadata.
+   * @returns {Promise<Document[]>}
+   * @throws If a field specified in fieldNames or metadataNames is not found in a MongoDB document.
+   */
+  public async loadData(
+    dbName: string,
+    collectionName: string,
+    fieldNames: string[] = ["text"],
+    separator: string = "",
+    filterQuery: Record<string, any> = {},
+    maxDocs: number = 0,
+    metadataNames?: string[],
+  ): Promise<Document[]> {
+    const db = this.client.db(dbName);
+    // Get items from collection
+    const cursor = db
+      .collection(collectionName)
+      .find(filterQuery)
+      .limit(maxDocs);
 
-    //Aggregate results and return
     const documents: Document[] = [];
-    cursor.forEach((element: Partial<Document>) => {
-      //For later: Metadata filtering
-      documents.push(new Document({ text: JSON.stringify(element) }));
-    });
+
+    for await (const item of cursor) {
+      try {
+        const texts: Array<string | string[]> = fieldNames.map(
+          (name) => item[name],
+        );
+        const flattenedTexts = this.flatten(texts);
+        const text = flattenedTexts.join(separator);
+
+        let metadata: Metadata = {};
+        if (metadataNames) {
+          // extract metadata if fields are specified
+          metadata = Object.fromEntries(
+            metadataNames.map((name) => [name, item[name]]),
+          );
+        }
+
+        documents.push(new Document({ text, metadata }));
+      } catch (err) {
+        throw new Error(
+          `Field not found in Mongo document: ${(err as Error).message}`,
+        );
+      }
+    }
     return documents;
   }
 }

packages/core/src/storage/index.ts

@@ -7,5 +7,6 @@ export { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
 export * from "./indexStore/types";
 export { SimpleKVStore } from "./kvStore/SimpleKVStore";
 export * from "./kvStore/types";
+export { MongoDBAtlasVectorSearch } from "./vectorStore/MongoDBAtlasVectorStore";
 export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
 export * from "./vectorStore/types";

packages/core/src/storage/vectorStore/MongoDBAtlasVectorStore.ts

@@ -0,0 +1,164 @@
+import { BulkWriteOptions, Collection, MongoClient } from "mongodb";
+import { BaseNode, MetadataMode } from "../../Node";
+import {
+  MetadataFilters,
+  VectorStore,
+  VectorStoreQuery,
+  VectorStoreQueryResult,
+} from "./types";
+import { metadataDictToNode, nodeToMetadata } from "./utils";
+
+// Utility function to convert metadata filters to MongoDB filter
+function toMongoDBFilter(
+  standardFilters: MetadataFilters,
+): Record<string, any> {
+  const filters: Record<string, any> = {};
+  for (const filter of standardFilters.filters) {
+    filters[filter.key] = filter.value;
+  }
+  return filters;
+}
+
+// MongoDB Atlas Vector Store class implementing VectorStore
+export class MongoDBAtlasVectorSearch implements VectorStore {
+  storesText: boolean = true;
+  flatMetadata: boolean = true;
+
+  mongodbClient: MongoClient;
+  indexName: string;
+  embeddingKey: string;
+  idKey: string;
+  textKey: string;
+  metadataKey: string;
+  insertOptions?: BulkWriteOptions;
+  private collection: Collection;
+
+  constructor(
+    init: Partial<MongoDBAtlasVectorSearch> & {
+      dbName: string;
+      collectionName: string;
+    },
+  ) {
+    if (init.mongodbClient) {
+      this.mongodbClient = init.mongodbClient;
+    } else {
+      const mongoUri = process.env.MONGODB_URI;
+      if (!mongoUri) {
+        throw new Error(
+          "Must specify MONGODB_URI via env variable if not directly passing in client.",
+        );
+      }
+      this.mongodbClient = new MongoClient(mongoUri);
+    }
+
+    this.collection = this.mongodbClient
+      .db(init.dbName ?? "default_db")
+      .collection(init.collectionName ?? "default_collection");
+    this.indexName = init.indexName ?? "default";
+    this.embeddingKey = init.embeddingKey ?? "embedding";
+    this.idKey = init.idKey ?? "id";
+    this.textKey = init.textKey ?? "text";
+    this.metadataKey = init.metadataKey ?? "metadata";
+    this.insertOptions = init.insertOptions;
+  }
+
+  async add(nodes: BaseNode[]): Promise<string[]> {
+    if (!nodes || nodes.length === 0) {
+      return [];
+    }
+    const dataToInsert = nodes.map((node) => {
+      const metadata = nodeToMetadata(
+        node,
+        true,
+        this.textKey,
+        this.flatMetadata,
+      );
+
+      return {
+        [this.idKey]: node.id_,
+        [this.embeddingKey]: node.getEmbedding(),
+        [this.textKey]: node.getContent(MetadataMode.NONE) || "",
+        [this.metadataKey]: metadata,
+      };
+    });
+
+    console.debug("Inserting data into MongoDB: ", dataToInsert);
+    const insertResult = await this.collection.insertMany(
+      dataToInsert,
+      this.insertOptions,
+    );
+    console.debug("Result of insert: ", insertResult);
+    return nodes.map((node) => node.id_);
+  }
+
+  async delete(refDocId: string, deleteOptions?: any): Promise<void> {
+    await this.collection.deleteOne(
+      {
+        [`${this.metadataKey}.ref_doc_id`]: refDocId,
+      },
+      deleteOptions,
+    );
+  }
+
+  get client(): any {
+    return this.mongodbClient;
+  }
+
+  async query(
+    query: VectorStoreQuery,
+    options?: any,
+  ): Promise<VectorStoreQueryResult> {
+    const params: any = {
+      queryVector: query.queryEmbedding,
+      path: this.embeddingKey,
+      numCandidates: query.similarityTopK * 10,
+      limit: query.similarityTopK,
+      index: this.indexName,
+    };
+
+    if (query.filters) {
+      params.filter = toMongoDBFilter(query.filters);
+    }
+
+    const queryField = { $vectorSearch: params };
+    const pipeline = [
+      queryField,
+      {
+        $project: {
+          score: { $meta: "vectorSearchScore" },
+          [this.embeddingKey]: 0,
+        },
+      },
+    ];
+
+    console.debug("Running query pipeline: ", pipeline);
+    const cursor = await this.collection.aggregate(pipeline);
+
+    const nodes: BaseNode[] = [];
+    const ids: string[] = [];
+    const similarities: number[] = [];
+
+    for await (const res of await cursor) {
+      const text = res[this.textKey];
+      const score = res.score;
+      const id = res[this.idKey];
+      const metadata = res[this.metadataKey];
+
+      const node = metadataDictToNode(metadata);
+      node.setContent(text);
+
+      ids.push(id);
+      nodes.push(node);
+      similarities.push(score);
+    }
+
+    const result = {
+      nodes,
+      similarities,
+      ids,
+    };
+
+    console.debug("Result of query (ids):", ids);
+    return result;
+  }
+}

packages/core/src/storage/vectorStore/PGVectorStore.ts

@@ -0,0 +1,266 @@
+import pg from "pg";
+import pgvector from "pgvector/pg";
+
+import { VectorStore, VectorStoreQuery, VectorStoreQueryResult } from "./types";
+
+import { BaseNode, Document, Metadata, MetadataMode } from "../../Node";
+import { GenericFileSystem } from "../FileSystem";
+
+export const PGVECTOR_SCHEMA = "public";
+export const PGVECTOR_TABLE = "llamaindex_embedding";
+
+/**
+ * Provides support for writing and querying vector data in Postgres.
+ */
+export class PGVectorStore implements VectorStore {
+  storesText: boolean = true;
+
+  private collection: string = "";
+
+  /*
+    FROM pg LIBRARY:
+    type Config = {
+      user?: string, // default process.env.PGUSER || process.env.USER
+      password?: string or function, //default process.env.PGPASSWORD
+      host?: string, // default process.env.PGHOST
+      database?: string, // default process.env.PGDATABASE || user
+      port?: number, // default process.env.PGPORT
+      connectionString?: string, // e.g. postgres://user:password@host:5432/database
+      ssl?: any, // passed directly to node.TLSSocket, supports all tls.connect options
+      types?: any, // custom type parsers
+      statement_timeout?: number, // number of milliseconds before a statement in query will time out, default is no timeout
+      query_timeout?: number, // number of milliseconds before a query call will timeout, default is no timeout
+      application_name?: string, // The name of the application that created this Client instance
+      connectionTimeoutMillis?: number, // number of milliseconds to wait for connection, default is no timeout
+      idle_in_transaction_session_timeout?: number // number of milliseconds before terminating any session with an open idle transaction, default is no timeout
+    }  
+  */
+  db?: pg.Client;
+
+  constructor() {}
+
+  /**
+   * Setter for the collection property.
+   * Using a collection allows for simple segregation of vector data,
+   * e.g. by user, source, or access-level.
+   * Leave/set blank to ignore the collection value when querying.
+   * @param coll Name for the collection.
+   */
+  setCollection(coll: string) {
+    this.collection = coll;
+  }
+
+  /**
+   * Getter for the collection property.
+   * Using a collection allows for simple segregation of vector data,
+   * e.g. by user, source, or access-level.
+   * Leave/set blank to ignore the collection value when querying.
+   * @returns The currently-set collection value.  Default is empty string.
+   */
+  getCollection(): string {
+    return this.collection;
+  }
+
+  private async getDb(): Promise<pg.Client> {
+    if (!this.db) {
+      try {
+        // Create DB connection
+        // Read connection params from env - see comment block above
+        const db = new pg.Client();
+        await db.connect();
+
+        // Check vector extension
+        db.query("CREATE EXTENSION IF NOT EXISTS vector");
+        await pgvector.registerType(db);
+
+        // Check schema, table(s), index(es)
+        await this.checkSchema(db);
+
+        // All good?  Keep the connection reference
+        this.db = db;
+      } catch (err: any) {
+        console.error(err);
+        return Promise.reject(err);
+      }
+    }
+
+    return Promise.resolve(this.db);
+  }
+
+  private async checkSchema(db: pg.Client) {
+    await db.query(`CREATE SCHEMA IF NOT EXISTS ${PGVECTOR_SCHEMA}`);
+
+    const tbl = `CREATE TABLE IF NOT EXISTS ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE}(
+      id uuid DEFAULT gen_random_uuid() PRIMARY KEY,
+      external_id VARCHAR,
+      collection VARCHAR,
+      document TEXT,
+      metadata JSONB DEFAULT '{}',
+      embeddings VECTOR(1536)
+    )`;
+    await db.query(tbl);
+
+    const idxs = `CREATE INDEX IF NOT EXISTS idx_${PGVECTOR_TABLE}_external_id ON ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} (external_id);
+      CREATE INDEX IF NOT EXISTS idx_${PGVECTOR_TABLE}_collection ON ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} (collection);`;
+    await db.query(idxs);
+
+    // TODO add IVFFlat or HNSW indexing?
+    return db;
+  }
+
+  // isEmbeddingQuery?: boolean | undefined;
+
+  /**
+   * Connects to the database specified in environment vars.
+   * This method also checks and creates the vector extension,
+   * the destination table and indexes if not found.
+   * @returns A connection to the database, or the error encountered while connecting/setting up.
+   */
+  client() {
+    return this.getDb();
+  }
+
+  /**
+   * Delete all vector records for the specified collection.
+   * NOTE: Uses the collection property controlled by setCollection/getCollection.
+   * @returns The result of the delete query.
+   */
+  async clearCollection() {
+    const sql: string = `DELETE FROM ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} 
+      WHERE collection = $1`;
+
+    const db = (await this.getDb()) as pg.Client;
+    const ret = await db.query(sql, [this.collection]);
+
+    return ret;
+  }
+
+  /**
+   * Adds vector record(s) to the table.
+   * NOTE: Uses the collection property controlled by setCollection/getCollection.
+   * @param embeddingResults The Nodes to be inserted, optionally including metadata tuples.
+   * @returns A list of zero or more id values for the created records.
+   */
+  async add(embeddingResults: BaseNode<Metadata>[]): Promise<string[]> {
+    const sql: string = `INSERT INTO ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} 
+      (id, external_id, collection, document, metadata, embeddings) 
+      VALUES ($1, $2, $3, $4, $5, $6)`;
+
+    const db = (await this.getDb()) as pg.Client;
+
+    let ret: string[] = [];
+    for (let index = 0; index < embeddingResults.length; index++) {
+      const row = embeddingResults[index];
+
+      let id: any = row.id_.length ? row.id_ : null;
+      let meta = row.metadata || {};
+      meta.create_date = new Date();
+
+      const params = [
+        id,
+        "",
+        this.collection,
+        row.getContent(MetadataMode.EMBED),
+        meta,
+        "[" + row.getEmbedding().join(",") + "]",
+      ];
+
+      try {
+        const result = await db.query(sql, params);
+
+        if (result.rows.length) {
+          id = result.rows[0].id as string;
+          ret.push(id);
+        }
+      } catch (err) {
+        const msg = `${err}`;
+        console.log(msg, err);
+      }
+    }
+
+    return Promise.resolve(ret);
+  }
+
+  /**
+   * Deletes a single record from the database by id.
+   * NOTE: Uses the collection property controlled by setCollection/getCollection.
+   * @param refDocId Unique identifier for the record to delete.
+   * @param deleteKwargs Required by VectorStore interface.  Currently ignored.
+   * @returns Promise that resolves if the delete query did not throw an error.
+   */
+  async delete(refDocId: string, deleteKwargs?: any): Promise<void> {
+    const collectionCriteria = this.collection.length
+      ? "AND collection = $2"
+      : "";
+    const sql: string = `DELETE FROM ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} 
+      WHERE id = $1 ${collectionCriteria}`;
+
+    const db = (await this.getDb()) as pg.Client;
+    const params = this.collection.length
+      ? [refDocId, this.collection]
+      : [refDocId];
+    await db.query(sql, params);
+    return Promise.resolve();
+  }
+
+  /**
+   * Query the vector store for the closest matching data to the query embeddings
+   * @param query The VectorStoreQuery to be used
+   * @param options Required by VectorStore interface.  Currently ignored.
+   * @returns Zero or more Document instances with data from the vector store.
+   */
+  async query(
+    query: VectorStoreQuery,
+    options?: any,
+  ): Promise<VectorStoreQueryResult> {
+    // TODO QUERY TYPES:
+    //    Distance:       SELECT embedding <-> $1 AS distance FROM items;
+    //    Inner Product:  SELECT (embedding <#> $1) * -1 AS inner_product FROM items;
+    //    Cosine Sim:     SELECT 1 - (embedding <=> $1) AS cosine_similarity FROM items;
+
+    const embedding = "[" + query.queryEmbedding?.join(",") + "]";
+    const max = query.similarityTopK ?? 2;
+    const where = this.collection.length ? "WHERE collection = $2" : "";
+    // TODO Add collection filter if set
+    const sql = `SELECT * FROM ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE}
+      ${where}
+      ORDER BY embeddings <-> $1 LIMIT ${max}
+    `;
+
+    const db = (await this.getDb()) as pg.Client;
+    const params = this.collection.length
+      ? [embedding, this.collection]
+      : [embedding];
+    const results = await db.query(sql, params);
+
+    const nodes = results.rows.map((row) => {
+      return new Document({
+        id_: row.id,
+        text: row.document,
+        metadata: row.metadata,
+        embedding: row.embeddings,
+      });
+    });
+
+    const ret = {
+      nodes: nodes,
+      similarities: results.rows.map((row) => row.embeddings),
+      ids: results.rows.map((row) => row.id),
+    };
+
+    return Promise.resolve(ret);
+  }
+
+  /**
+   * Required by VectorStore interface.  Currently ignored.
+   * @param persistPath
+   * @param fs
+   * @returns Resolved Promise.
+   */
+  persist(
+    persistPath: string,
+    fs?: GenericFileSystem | undefined,
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+}

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -1,11 +1,11 @@
 import _ from "lodash";
 import * as path from "path";
+import { BaseNode } from "../../Node";
 import {
   getTopKEmbeddings,
   getTopKEmbeddingsLearner,
   getTopKMMREmbeddings,
-} from "../../Embedding";
-import { BaseNode } from "../../Node";
+} from "../../embeddings";
 import { GenericFileSystem, exists } from "../FileSystem";
 import { DEFAULT_FS, DEFAULT_PERSIST_DIR } from "../constants";
 import {

packages/core/src/storage/vectorStore/types.ts

@@ -1,5 +1,4 @@
 import { BaseNode } from "../../Node";
-import { GenericFileSystem } from "../FileSystem";
 
 export interface VectorStoreQueryResult {
   nodes?: BaseNode[];
@@ -62,10 +61,9 @@ export interface VectorStore {
   isEmbeddingQuery?: boolean;
   client(): any;
   add(embeddingResults: BaseNode[]): Promise<string[]>;
-  delete(refDocId: string, deleteKwargs?: any): Promise<void>;
+  delete(refDocId: string, deleteOptions?: any): Promise<void>;
   query(
     query: VectorStoreQuery,
     options?: any,
   ): Promise<VectorStoreQueryResult>;
-  persist(persistPath: string, fs?: GenericFileSystem): Promise<void>;
 }

packages/core/src/storage/vectorStore/utils.ts

@@ -0,0 +1,59 @@
+import { BaseNode, jsonToNode, Metadata, ObjectType } from "../../Node";
+
+const DEFAULT_TEXT_KEY = "text";
+
+export function validateIsFlat(obj: { [key: string]: any }): void {
+  for (let key in obj) {
+    if (typeof obj[key] === "object" && obj[key] !== null) {
+      throw new Error(`Value for metadata ${key} must not be another object`);
+    }
+  }
+}
+
+export function nodeToMetadata(
+  node: BaseNode,
+  removeText: boolean = false,
+  textField: string = DEFAULT_TEXT_KEY,
+  flatMetadata: boolean = false,
+): Metadata {
+  const nodeObj = node.toJSON();
+  const metadata = node.metadata;
+
+  if (flatMetadata) {
+    validateIsFlat(node.metadata);
+  }
+
+  if (removeText) {
+    nodeObj[textField] = "";
+  }
+
+  nodeObj["embedding"] = null;
+
+  metadata["_node_content"] = JSON.stringify(nodeObj);
+  metadata["_node_type"] = node.constructor.name.replace("_", ""); // remove leading underscore to be compatible with Python
+
+  metadata["document_id"] = node.sourceNode?.nodeId || "None";
+  metadata["doc_id"] = node.sourceNode?.nodeId || "None";
+  metadata["ref_doc_id"] = node.sourceNode?.nodeId || "None";
+
+  return metadata;
+}
+
+export function metadataDictToNode(metadata: Metadata): BaseNode {
+  const nodeContent = metadata["_node_content"];
+  if (!nodeContent) {
+    throw new Error("Node content not found in metadata.");
+  }
+  const nodeObj = JSON.parse(nodeContent);
+
+  // Note: we're using the name of the class stored in `_node_type`
+  // and not the type attribute to reconstruct
+  // the node. This way we're compatible with LlamaIndex Python
+  const node_type = metadata["_node_type"];
+  switch (node_type) {
+    case "IndexNode":
+      return jsonToNode(nodeObj, ObjectType.INDEX);
+    default:
+      return jsonToNode(nodeObj, ObjectType.TEXT);
+  }
+}

packages/core/src/tests/CallbackManager.test.ts

@@ -1,18 +1,18 @@
-import { OpenAIEmbedding } from "../Embedding";
+import {
+  CallbackManager,
+  RetrievalCallbackResponse,
+  StreamCallbackResponse,
+} from "../callbacks/CallbackManager";
+import { OpenAIEmbedding } from "../embeddings";
+import { SummaryIndex } from "../indices/summary";
+import { VectorStoreIndex } from "../indices/vectorStore/VectorStoreIndex";
+import { OpenAI } from "../llm/LLM";
 import { Document } from "../Node";
 import {
   ResponseSynthesizer,
   SimpleResponseBuilder,
 } from "../ResponseSynthesizer";
 import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext";
-import {
-  CallbackManager,
-  RetrievalCallbackResponse,
-  StreamCallbackResponse,
-} from "../callbacks/CallbackManager";
-import { SummaryIndex } from "../indices/summary";
-import { VectorStoreIndex } from "../indices/vectorStore/VectorStoreIndex";
-import { OpenAI } from "../llm/LLM";
 import { mockEmbeddingModel, mockLlmGeneration } from "./utility/mockOpenAI";
 
 // Mock the OpenAI getOpenAISession function during testing

packages/core/src/tests/Embedding.test.ts

@@ -1,4 +1,4 @@
-import { SimilarityType, similarity } from "../Embedding";
+import { similarity, SimilarityType } from "../embeddings";
 
 describe("similarity", () => {
   test("throws error on mismatched lengths", () => {

packages/core/src/tests/Node.test.ts

@@ -0,0 +1,15 @@
+import { TextNode } from "../Node";
+
+describe("TextNode", () => {
+  let node: TextNode;
+
+  beforeEach(() => {
+    node = new TextNode({ text: "Hello World" });
+  });
+
+  describe("generateHash", () => {
+    it("should generate a hash", () => {
+      expect(node.hash).toBe("nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do=");
+    });
+  });
+});

packages/core/src/tests/TextSplitter.test.ts

@@ -73,6 +73,7 @@ describe("SentenceSplitter", () => {
     let splits = sentenceSplitter.splitText(
       "This is a sentence. This is another sentence. 1.0",
     );
+
     expect(splits).toEqual([
       "This is a sentence.",
       "This is another sentence.",

packages/core/src/tests/utility/mockOpenAI.ts

@@ -1,6 +1,6 @@
-import { OpenAIEmbedding } from "../../Embedding";
-import { globalsHelper } from "../../GlobalsHelper";
 import { CallbackManager, Event } from "../../callbacks/CallbackManager";
+import { OpenAIEmbedding } from "../../embeddings";
+import { globalsHelper } from "../../GlobalsHelper";
 import { ChatMessage, OpenAI } from "../../llm/LLM";
 
 export function mockLlmGeneration({

packages/core/tsconfig.json

@@ -3,6 +3,7 @@
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
     "isolatedModules": true,
+    "module": "esnext",
     "moduleResolution": "node",
     "preserveWatchOutput": true,
     "skipLibCheck": true,

packages/create-llama/CHANGELOG.md

@@ -0,0 +1,49 @@
+# create-llama
+
+## 0.0.9
+
+### Patch Changes
+
+- acfe232: Deployment fixes (thanks @seldo)
+
+## 0.0.8
+
+### Patch Changes
+
+- 8cdb07f: Fix Next deployment (thanks @seldo and @marcusschiesser)
+
+## 0.0.7
+
+### Patch Changes
+
+- 9f9f293: Added more to README and made it easier to switch models (thanks @seldo)
+
+## 0.0.6
+
+### Patch Changes
+
+- 4431ec7: Label bug fix (thanks @marcusschiesser)
+
+## 0.0.5
+
+### Patch Changes
+
+- 25257f4: Fix issue where it doesn't find OpenAI Key when running npm run generate (#182) (thanks @RayFernando1337)
+
+## 0.0.4
+
+### Patch Changes
+
+- 031e926: Update create-llama readme (thanks @logan-markewich)
+
+## 0.0.3
+
+### Patch Changes
+
+- 91b42a3: change version (thanks @marcusschiesser)
+
+## 0.0.2
+
+### Patch Changes
+
+- e2a6805: Hello Create Llama (thanks @marcusschiesser)

packages/create-llama/LICENSE.md

@@ -0,0 +1,9 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 LlamaIndex, Vercel, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

packages/create-llama/README.md

@@ -0,0 +1,126 @@
+# Create LlamaIndex App
+
+The easiest way to get started with [LlamaIndex](https://www.llamaindex.ai/) is by using `create-llama`. This CLI tool enables you to quickly start building a new LlamaIndex application, with everything set up for you.
+
+Just run
+
+```bash
+npx create-llama@latest
+```
+
+to get started, or see below for more options. Once your app is generated, run
+
+```bash
+npm run dev
+```
+
+to start the development server. You can then visit [http://localhost:3000](http://localhost:3000) to see your app.
+
+## What you'll get
+
+- A Next.js-powered front-end. The app is set up as a chat interface that can answer questions about your data (see below)
+  - You can style it with HTML and CSS, or you can optionally use components from [shadcn/ui](https://ui.shadcn.com/)
+- Your choice of 3 back-ends:
+  - **Next.js**: if you select this option, you’ll have a full stack Next.js application that you can deploy to a host like [Vercel](https://vercel.com/) in just a few clicks. This uses [LlamaIndex.TS](https://www.npmjs.com/package/llamaindex), our TypeScript library.
+  - **Express**: if you want a more traditional Node.js application you can generate an Express backend. This also uses LlamaIndex.TS.
+  - **Python FastAPI**: if you select this option you’ll get a backend powered by the [llama-index python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
+- The back-end has a single endpoint that allows you to send the state of your chat and receive additional responses
+- You can choose whether you want a streaming or non-streaming back-end (if you're not sure, we recommend streaming)
+- You can choose whether you want to use `ContextChatEngine` or `SimpleChatEngine`
+  - `SimpleChatEngine` will just talk to the LLM directly without using your data
+  - `ContextChatEngine` will use your data to answer questions (see below).
+- The app uses OpenAI by default, so you'll need an OpenAI API key, or you can customize it to use any of the dozens of LLMs we support.
+
+## Using your data
+
+If you've enabled `ContextChatEngine`, you can supply your own data and the app will index it and answer questions. Your generated app will have a folder called `data`:
+
+- With the Next.js backend this is `./data`
+- With the Express or Python backend this is in `./backend/data`
+
+The app will ingest any supported files you put in this directory. Your Next.js and Express apps use LlamaIndex.TS so they will be able to ingest any PDF, text, CSV, Markdown, Word and HTML files. The Python backend can read even more types, including video and audio files.
+
+Before you can use your data, you need to index it. If you're using the Next.js or Express apps, run:
+
+```bash
+npm run generate
+```
+
+Then re-start your app. Remember you'll need to re-run `generate` if you add new files to your `data` folder. If you're using the Python backend, you can trigger indexing of your data by deleting the `./storage` folder and re-starting the app.
+
+## Don't want a front-end?
+
+It's optional! If you've selected the Python or Express back-ends, just delete the `frontend` folder and you'll get an API without any front-end code.
+
+## Customizing the LLM
+
+By default the app will use OpenAI's gpt-3.5-turbo model. If you want to use GPT-4, you can modify this by editing a file:
+
+- In the Next.js backend, edit `./app/api/chat/route.ts` and replace `gpt-3.5-turbo` with `gpt-4`
+- In the Express backend, edit `./backend/src/controllers/chat.controller.ts` and likewise replace `gpt-3.5-turbo` with `gpt-4`
+- In the Python backend, edit `./backend/app/utils/index.py` and once again replace `gpt-3.5-turbo` with `gpt-4`
+
+You can also replace OpenAI with one of our [dozens of other supported LLMs](https://docs.llamaindex.ai/en/stable/module_guides/models/llms/modules.html).
+
+## Example
+
+The simplest thing to do is run `create-llama` in interactive mode:
+
+```bash
+npx create-llama@latest
+# or
+npm create llama@latest
+# or
+yarn create llama
+# or
+pnpm create llama@latest
+```
+
+You will be asked for the name of your project, along with other configuration options, something like this:
+
+```bash
+>> npm create llama@latest
+Need to install the following packages:
+  create-llama@latest
+Ok to proceed? (y) y
+✔ What is your project named? … my-app
+✔ Which template would you like to use? › Chat with streaming
+✔ Which framework would you like to use? › NextJS
+✔ Which UI would you like to use? › Just HTML
+✔ Which chat engine would you like to use? › ContextChatEngine
+✔ Please provide your OpenAI API key (leave blank to skip): …
+✔ Would you like to use ESLint? … No / Yes
+Creating a new LlamaIndex app in /home/my-app.
+```
+
+### Running non-interactively
+
+You can also pass command line arguments to set up a new project
+non-interactively. See `create-llama --help`:
+
+```bash
+create-llama <project-directory> [options]
+
+Options:
+  -V, --version                      output the version number
+
+  --use-npm
+
+    Explicitly tell the CLI to bootstrap the app using npm
+
+  --use-pnpm
+
+    Explicitly tell the CLI to bootstrap the app using pnpm
+
+  --use-yarn
+
+    Explicitly tell the CLI to bootstrap the app using Yarn
+
+```
+
+## LlamaIndex Documentation
+
+- [TS/JS docs](https://ts.llamaindex.ai/)
+- [Python docs](https://docs.llamaindex.ai/en/stable/)
+
+Inspired by and adapted from [create-next-app](https://github.com/vercel/next.js/tree/canary/packages/create-next-app)

packages/create-llama/create-app.ts

@@ -0,0 +1,109 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import path from "path";
+import { green } from "picocolors";
+import { tryGitInit } from "./helpers/git";
+import { isFolderEmpty } from "./helpers/is-folder-empty";
+import { getOnline } from "./helpers/is-online";
+import { isWriteable } from "./helpers/is-writeable";
+import { makeDir } from "./helpers/make-dir";
+
+import fs from "fs";
+import terminalLink from "terminal-link";
+import type { InstallTemplateArgs } from "./templates";
+import { installTemplate } from "./templates";
+
+export async function createApp({
+  template,
+  framework,
+  engine,
+  ui,
+  appPath,
+  packageManager,
+  eslint,
+  frontend,
+  openAIKey,
+}: Omit<
+  InstallTemplateArgs,
+  "appName" | "root" | "isOnline" | "customApiPath"
+> & {
+  appPath: string;
+  frontend: boolean;
+}): Promise<void> {
+  const root = path.resolve(appPath);
+
+  if (!(await isWriteable(path.dirname(root)))) {
+    console.error(
+      "The application path is not writable, please check folder permissions and try again.",
+    );
+    console.error(
+      "It is likely you do not have write permissions for this folder.",
+    );
+    process.exit(1);
+  }
+
+  const appName = path.basename(root);
+
+  await makeDir(root);
+  if (!isFolderEmpty(root, appName)) {
+    process.exit(1);
+  }
+
+  const useYarn = packageManager === "yarn";
+  const isOnline = !useYarn || (await getOnline());
+
+  console.log(`Creating a new LlamaIndex app in ${green(root)}.`);
+  console.log();
+
+  const args = {
+    appName,
+    root,
+    template,
+    framework,
+    engine,
+    ui,
+    packageManager,
+    isOnline,
+    eslint,
+    openAIKey,
+  };
+
+  if (frontend) {
+    // install backend
+    const backendRoot = path.join(root, "backend");
+    await makeDir(backendRoot);
+    await installTemplate({ ...args, root: backendRoot, backend: true });
+    // install frontend
+    const frontendRoot = path.join(root, "frontend");
+    await makeDir(frontendRoot);
+    await installTemplate({
+      ...args,
+      root: frontendRoot,
+      framework: "nextjs",
+      customApiPath: "http://localhost:8000/api/chat",
+      backend: false,
+    });
+    // copy readme for fullstack
+    await fs.promises.copyFile(
+      path.join(__dirname, "templates", "README-fullstack.md"),
+      path.join(root, "README.md"),
+    );
+  } else {
+    await installTemplate({ ...args, backend: true, forBackend: framework });
+  }
+
+  process.chdir(root);
+  if (tryGitInit(root)) {
+    console.log("Initialized a git repository.");
+    console.log();
+  }
+
+  console.log(`${green("Success!")} Created ${appName} at ${appPath}`);
+
+  console.log(
+    `Now have a look at the ${terminalLink(
+      "README.md",
+      `file://${appName}/README.md`,
+    )} and learn how to get started.`,
+  );
+  console.log();
+}

packages/create-llama/helpers/copy.ts

@@ -0,0 +1,50 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { async as glob } from "fast-glob";
+import fs from "fs";
+import path from "path";
+
+interface CopyOption {
+  cwd?: string;
+  rename?: (basename: string) => string;
+  parents?: boolean;
+}
+
+const identity = (x: string) => x;
+
+export const copy = async (
+  src: string | string[],
+  dest: string,
+  { cwd, rename = identity, parents = true }: CopyOption = {},
+) => {
+  const source = typeof src === "string" ? [src] : src;
+
+  if (source.length === 0 || !dest) {
+    throw new TypeError("`src` and `dest` are required");
+  }
+
+  const sourceFiles = await glob(source, {
+    cwd,
+    dot: true,
+    absolute: false,
+    stats: false,
+  });
+
+  const destRelativeToCwd = cwd ? path.resolve(cwd, dest) : dest;
+
+  return Promise.all(
+    sourceFiles.map(async (p) => {
+      const dirname = path.dirname(p);
+      const basename = rename(path.basename(p));
+
+      const from = cwd ? path.resolve(cwd, p) : p;
+      const to = parents
+        ? path.join(destRelativeToCwd, dirname, basename)
+        : path.join(destRelativeToCwd, basename);
+
+      // Ensure the destination directory exists
+      await fs.promises.mkdir(path.dirname(to), { recursive: true });
+
+      return fs.promises.copyFile(from, to);
+    }),
+  );
+};

packages/create-llama/helpers/get-pkg-manager.ts

@@ -0,0 +1,15 @@
+export type PackageManager = "npm" | "pnpm" | "yarn";
+
+export function getPkgManager(): PackageManager {
+  const userAgent = process.env.npm_config_user_agent || "";
+
+  if (userAgent.startsWith("yarn")) {
+    return "yarn";
+  }
+
+  if (userAgent.startsWith("pnpm")) {
+    return "pnpm";
+  }
+
+  return "npm";
+}

packages/create-llama/helpers/git.ts

@@ -0,0 +1,58 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { execSync } from "child_process";
+import fs from "fs";
+import path from "path";
+
+function isInGitRepository(): boolean {
+  try {
+    execSync("git rev-parse --is-inside-work-tree", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+function isInMercurialRepository(): boolean {
+  try {
+    execSync("hg --cwd . root", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+function isDefaultBranchSet(): boolean {
+  try {
+    execSync("git config init.defaultBranch", { stdio: "ignore" });
+    return true;
+  } catch (_) {}
+  return false;
+}
+
+export function tryGitInit(root: string): boolean {
+  let didInit = false;
+  try {
+    execSync("git --version", { stdio: "ignore" });
+    if (isInGitRepository() || isInMercurialRepository()) {
+      return false;
+    }
+
+    execSync("git init", { stdio: "ignore" });
+    didInit = true;
+
+    if (!isDefaultBranchSet()) {
+      execSync("git checkout -b main", { stdio: "ignore" });
+    }
+
+    execSync("git add -A", { stdio: "ignore" });
+    execSync('git commit -m "Initial commit from Create Llama"', {
+      stdio: "ignore",
+    });
+    return true;
+  } catch (e) {
+    if (didInit) {
+      try {
+        fs.rmSync(path.join(root, ".git"), { recursive: true, force: true });
+      } catch (_) {}
+    }
+    return false;
+  }
+}

packages/create-llama/helpers/install.ts

@@ -0,0 +1,50 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import spawn from "cross-spawn";
+import { yellow } from "picocolors";
+import type { PackageManager } from "./get-pkg-manager";
+
+/**
+ * Spawn a package manager installation based on user preference.
+ *
+ * @returns A Promise that resolves once the installation is finished.
+ */
+export async function callPackageManager(
+  /** Indicate which package manager to use. */
+  packageManager: PackageManager,
+  /** Indicate whether there is an active Internet connection.*/
+  isOnline: boolean,
+  args: string[] = ["install"],
+): Promise<void> {
+  if (!isOnline) {
+    console.log(
+      yellow("You appear to be offline.\nFalling back to the local cache."),
+    );
+    args.push("--offline");
+  }
+  /**
+   * Return a Promise that resolves once the installation is finished.
+   */
+  return new Promise((resolve, reject) => {
+    /**
+     * Spawn the installation process.
+     */
+    const child = spawn(packageManager, args, {
+      stdio: "inherit",
+      env: {
+        ...process.env,
+        ADBLOCK: "1",
+        // we set NODE_ENV to development as pnpm skips dev
+        // dependencies when production
+        NODE_ENV: "development",
+        DISABLE_OPENCOLLECTIVE: "1",
+      },
+    });
+    child.on("close", (code) => {
+      if (code !== 0) {
+        reject({ command: `${packageManager} ${args.join(" ")}` });
+        return;
+      }
+      resolve();
+    });
+  });
+}

packages/create-llama/helpers/is-folder-empty.ts

@@ -0,0 +1,62 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import fs from "fs";
+import path from "path";
+import { blue, green } from "picocolors";
+
+export function isFolderEmpty(root: string, name: string): boolean {
+  const validFiles = [
+    ".DS_Store",
+    ".git",
+    ".gitattributes",
+    ".gitignore",
+    ".gitlab-ci.yml",
+    ".hg",
+    ".hgcheck",
+    ".hgignore",
+    ".idea",
+    ".npmignore",
+    ".travis.yml",
+    "LICENSE",
+    "Thumbs.db",
+    "docs",
+    "mkdocs.yml",
+    "npm-debug.log",
+    "yarn-debug.log",
+    "yarn-error.log",
+    "yarnrc.yml",
+    ".yarn",
+  ];
+
+  const conflicts = fs
+    .readdirSync(root)
+    .filter((file) => !validFiles.includes(file))
+    // Support IntelliJ IDEA-based editors
+    .filter((file) => !/\.iml$/.test(file));
+
+  if (conflicts.length > 0) {
+    console.log(
+      `The directory ${green(name)} contains files that could conflict:`,
+    );
+    console.log();
+    for (const file of conflicts) {
+      try {
+        const stats = fs.lstatSync(path.join(root, file));
+        if (stats.isDirectory()) {
+          console.log(`  ${blue(file)}/`);
+        } else {
+          console.log(`  ${file}`);
+        }
+      } catch {
+        console.log(`  ${file}`);
+      }
+    }
+    console.log();
+    console.log(
+      "Either try using a new directory name, or remove the files listed above.",
+    );
+    console.log();
+    return false;
+  }
+
+  return true;
+}

packages/create-llama/helpers/is-online.ts

@@ -0,0 +1,40 @@
+import { execSync } from "child_process";
+import dns from "dns";
+import url from "url";
+
+function getProxy(): string | undefined {
+  if (process.env.https_proxy) {
+    return process.env.https_proxy;
+  }
+
+  try {
+    const httpsProxy = execSync("npm config get https-proxy").toString().trim();
+    return httpsProxy !== "null" ? httpsProxy : undefined;
+  } catch (e) {
+    return;
+  }
+}
+
+export function getOnline(): Promise<boolean> {
+  return new Promise((resolve) => {
+    dns.lookup("registry.yarnpkg.com", (registryErr) => {
+      if (!registryErr) {
+        return resolve(true);
+      }
+
+      const proxy = getProxy();
+      if (!proxy) {
+        return resolve(false);
+      }
+
+      const { hostname } = url.parse(proxy);
+      if (!hostname) {
+        return resolve(false);
+      }
+
+      dns.lookup(hostname, (proxyErr) => {
+        resolve(proxyErr == null);
+      });
+    });
+  });
+}

packages/create-llama/helpers/is-url.ts

@@ -0,0 +1,8 @@
+export function isUrl(url: string): boolean {
+  try {
+    new URL(url);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}

packages/create-llama/helpers/is-writeable.ts

@@ -0,0 +1,10 @@
+import fs from "fs";
+
+export async function isWriteable(directory: string): Promise<boolean> {
+  try {
+    await fs.promises.access(directory, (fs.constants || fs).W_OK);
+    return true;
+  } catch (err) {
+    return false;
+  }
+}

packages/create-llama/helpers/make-dir.ts

@@ -0,0 +1,8 @@
+import fs from "fs";
+
+export function makeDir(
+  root: string,
+  options = { recursive: true },
+): Promise<string | undefined> {
+  return fs.promises.mkdir(root, options);
+}