update AssemblyAI reader to use new functions (#245)

This reverts commit c8bbc101cc.

.github/ISSUE_TEMPLATE/bugfix.yml

@@ -1,7 +1,6 @@
 name: Bugfix
-title: "Sweep: "
+title: ""
 description: Write something like "We notice ... behavior when ... happens instead of ...""
-labels: sweep
 body:
   - type: textarea
     id: description

.github/ISSUE_TEMPLATE/feature.yml

@@ -1,11 +1,10 @@
 name: Feature Request
-title: "Sweep: "
-description: Write something like "Write an api endpoint that does "..." in the "..." file"
-labels: sweep
+title: ""
+description: Write something like "Write an api endpoint that does "..." in the "..." file". If you would like to use sweep.dev prefix with "Sweep:"
 body:
   - type: textarea
     id: description
     attributes:
       label: Details
-      description: More details for Sweep
+      description: More details
       placeholder: The new endpoint should use the ... class from ... file because it contains ... logic

.github/ISSUE_TEMPLATE/refactor.yml

@@ -1,11 +1,10 @@
 name: Refactor
-title: "Sweep: "
-description: Write something like "Modify the ... api endpoint to use ... version and ... framework"
-labels: sweep
+title: ""
+description: Write something like "Modify the ... api endpoint to use ... version and ... framework" If you would like to use sweep.dev prefix with "Sweep:"
 body:
   - type: textarea
     id: description
     attributes:
       label: Details
-      description: More details for Sweep
+      description: More details
       placeholder: We are migrating this function to ... version because ...

.vscode/settings.json

@@ -4,5 +4,6 @@
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "[xml]": {
     "editor.defaultFormatter": "redhat.vscode-xml"
-  }
-}
+  },
+  "jest.rootPath": "./packages/core"
+}

apps/docs/docs/introduction.md

@@ -5,7 +5,7 @@ slug: /
 
 # What is LlamaIndex.TS?
 
-LlamaIndex.TS is a data framework for LLM applications to ingest, structure, and access private or domain-specific data. While a python package is also available (see [here](https://gpt-index.readthedocs.io/en/latest/)), LlamaIndex.TS offers core features in a simple package, optimized for usage with TypeScript.
+LlamaIndex.TS is a data framework for LLM applications to ingest, structure, and access private or domain-specific data. While a python package is also available (see [here](https://docs.llamaindex.ai/en/stable/)), LlamaIndex.TS offers core features in a simple package, optimized for usage with TypeScript.
 
 ## 🚀 Why LlamaIndex.TS?
 

apps/mongodb/1_import.ts

@@ -0,0 +1,34 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import * as dotenv from "dotenv";
+import * as fs from "fs";
+import { MongoClient } from "mongodb";
+
+// Load environment variables from local .env file
+dotenv.config();
+
+const jsonFile = "tinytweets.json";
+const mongoUri = process.env.MONGODB_URI!;
+const databaseName = process.env.MONGODB_DATABASE!;
+const collectionName = process.env.MONGODB_COLLECTION!;
+
+async function importJsonToMongo() {
+  // Load the tweets from a local file
+  const tweets = JSON.parse(fs.readFileSync(jsonFile, "utf-8"));
+
+  // Create a new client and connect to the server
+  const client = new MongoClient(mongoUri);
+
+  const db = client.db(databaseName);
+  const collection = db.collection(collectionName);
+
+  // Insert the tweets into mongo
+  await collection.insertMany(tweets);
+
+  console.log(
+    `Data imported successfully to the MongoDB collection ${collectionName}.`,
+  );
+  await client.close();
+}
+
+// Run the import function
+importJsonToMongo();

apps/mongodb/2_load_and_index.ts

@@ -0,0 +1,50 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import * as dotenv from "dotenv";
+import {
+  MongoDBAtlasVectorSearch,
+  SimpleMongoReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { MongoClient } from "mongodb";
+
+// Load environment variables from local .env file
+dotenv.config();
+
+const mongoUri = process.env.MONGODB_URI!;
+const databaseName = process.env.MONGODB_DATABASE!;
+const collectionName = process.env.MONGODB_COLLECTION!;
+const vectorCollectionName = process.env.MONGODB_VECTORS!;
+const indexName = process.env.MONGODB_VECTOR_INDEX!;
+
+async function loadAndIndex() {
+  // Create a new client and connect to the server
+  const client = new MongoClient(mongoUri);
+  // load objects from mongo and convert them into LlamaIndex Document objects
+  // llamaindex has a special class that does this for you
+  // it pulls every object in a given collection
+  const reader = new SimpleMongoReader(client);
+  const documents = await reader.loadData(databaseName, collectionName, [
+    "full_text",
+  ]);
+
+  // create Atlas as a vector store
+  const vectorStore = new MongoDBAtlasVectorSearch({
+    mongodbClient: client,
+    dbName: databaseName,
+    collectionName: vectorCollectionName, // this is where your embeddings will be stored
+    indexName: indexName, // this is the name of the index you will need to create
+  });
+
+  // now create an index from all the Documents and store them in Atlas
+  const storageContext = await storageContextFromDefaults({ vectorStore });
+  await VectorStoreIndex.fromDocuments(documents, { storageContext });
+  console.log(
+    `Successfully created embeddings in the MongoDB collection ${vectorCollectionName}.`,
+  );
+  await client.close();
+}
+
+loadAndIndex();
+
+// you can't query your index yet because you need to create a vector search index in mongodb's UI now

apps/mongodb/3_query.ts

@@ -0,0 +1,34 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import * as dotenv from "dotenv";
+import {
+  MongoDBAtlasVectorSearch,
+  serviceContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { MongoClient } from "mongodb";
+
+// Load environment variables from local .env file
+dotenv.config();
+
+async function query() {
+  const client = new MongoClient(process.env.MONGODB_URI!);
+  const serviceContext = serviceContextFromDefaults();
+  const store = new MongoDBAtlasVectorSearch({
+    mongodbClient: client,
+    dbName: process.env.MONGODB_DATABASE!,
+    collectionName: process.env.MONGODB_VECTORS!,
+    indexName: process.env.MONGODB_VECTOR_INDEX!,
+  });
+
+  const index = await VectorStoreIndex.fromVectorStore(store, serviceContext);
+
+  const retriever = index.asRetriever({ similarityTopK: 20 });
+  const queryEngine = index.asQueryEngine({ retriever });
+  const result = await queryEngine.query(
+    "What does the author think of web frameworks?",
+  );
+  console.log(result.response);
+  await client.close();
+}
+
+query();

apps/mongodb/CHANGELOG.md

@@ -0,0 +1,20 @@
+# mongodb-llamaindexts
+
+## 0.0.3
+
+### Patch Changes
+
+- Updated dependencies [3bab231]
+  - llamaindex@0.0.37
+
+## 0.0.2
+
+### Patch Changes
+
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+  - llamaindex@0.0.36

apps/mongodb/README.md

@@ -0,0 +1,127 @@
+# LlamaIndexTS retrieval augmented generation with MongoDB
+
+### Prepare Environment
+
+Make sure to run `pnpm install` and set your OpenAI environment variable before running these examples.
+
+```
+pnpm install
+export OPENAI_API_KEY="sk-..."
+```
+
+### Sign up for MongoDB Atlas
+
+We'll be using MongoDB's hosted database service, [MongoDB Atlas](https://www.mongodb.com/cloud/atlas/register). You can sign up for free and get a small hosted cluster for free:
+
+![MongoDB Atlas signup](./docs/1_signup.png)
+
+The signup process will walk you through the process of creating your cluster and ensuring it's configured for you to access. Once the cluster is created, choose "Connect" and then "Connect to your application". Choose Python, and you'll be presented with a connection string that looks like this:
+
+![MongoDB Atlas connection string](./docs/2_connection_string.png)
+
+### Set up environment variables
+
+Copy the connection string (make sure you include your password) and put it into a file called `.env` in the root of this repo. It should look like this:
+
+```
+MONGODB_URI=mongodb+srv://seldo:xxxxxxxxxxx@llamaindexdemocluster.xfrdhpz.mongodb.net/?retryWrites=true&w=majority
+```
+
+You will also need to choose a name for your database, and the collection where we will store the tweets, and also include them in .env. They can be any string, but this is what we used:
+
+```
+MONGODB_DATABASE=tiny_tweets_db
+MONGODB_COLLECTION=tiny_tweets_collection
+```
+
+### Import tweets into MongoDB
+
+You are now ready to import our ready-made data set into Mongo. This is the file `tinytweets.json`, a selection of approximately 1000 tweets from @seldo on Twitter in mid-2019. With your environment set up you can do this by running
+
+```
+pnpm ts-node 1_import.ts
+```
+
+If you don't want to use tweets, you can replace `json_file` with any other array of JSON objects, but you will need to modify some code later to make sure the correct field gets indexed. There is no LlamaIndex-specific code here; you can load your data into Mongo any way you want to.
+
+### Load and index your data
+
+Now we're ready to index our data. To do this, LlamaIndex will pull your text out of Mongo, split it into chunks, and then send those chunks to OpenAI to be turned into [vector embeddings](https://docs.llamaindex.ai/en/stable/understanding/indexing/indexing.html#what-is-an-embedding). The embeddings will then be stored in a new collection in Mongo. This will take a while depending how much text you have, but the good news is that once it's done you will be able to query quickly without needing to re-index.
+
+We'll be using OpenAI to do the embedding, so now is when you need to [generate an OpenAI API key](https://platform.openai.com/account/api-keys) if you haven't already and add it to your `.env` file like this:
+
+```
+OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+```
+
+You'll also need to pick a name for the new collection where the embeddings will be stored, and add it to `.env`, along with the name of a vector search index (we'll be creating this in the next step, after you've indexed your data):
+
+```
+MONGODB_VECTORS=tiny_tweets_vectors
+MONGODB_VECTOR_INDEX=tiny_tweets_vector_index
+```
+
+If the data you're indexing is the tweets we gave you, you're ready to go:
+
+```bash
+pnpm ts-node 2_load_and_index.ts
+```
+
+> Note: this script is running a couple of minutes and currently doesn't show any progress.
+
+What you're doing here is creating a Reader which loads the data out of Mongo in the collection and database specified. It looks for text in a set of specific keys in each object. In this case we've given it just one key, "full_text".
+
+Now you're creating a vector search client for Mongo. In addition to a MongoDB client object, you again tell it what database everything is in. This time you give it the name of the collection where you'll store the vector embeddings, and the name of the vector search index you'll create in the next step.
+
+### Create a vector search index
+
+Now if all has gone well you should be able to log in to the Mongo Atlas UI and see two collections in your database: the original data in `tiny_tweets_collection`, and the vector embeddings in `tiny_tweets_vectors`.
+
+![MongoDB Atlas collections](./docs/3_vectors_in_db.png)
+
+Now it's time to create the vector search index so that you can query the data.
+It's not yet possible to programmatically create a vector search index using the [`createIndex`](https://www.mongodb.com/docs/manual/reference/method/db.collection.createIndex/) function, therefore we have to create one manually in the UI.
+To do so, first, click the Search tab, and then click "Create Search Index":
+
+![MongoDB Atlas create search index](./docs/4_search_tab.png)
+
+We have to use the JSON editor, as the Visual Editor does not yet support to create a vector search index:
+
+![MongoDB Atlas JSON editor](./docs/5_json_editor.png)
+
+Now under "database and collection" select `tiny_tweets_db` and within that select `tiny_tweets_vectors`. Then under "Index name" enter `tiny_tweets_vector_index` (or whatever value you put for MONGODB_VECTOR_INDEX in `.env`). Under that, you'll want to enter this JSON object:
+
+```json
+{
+  "mappings": {
+    "dynamic": true,
+    "fields": {
+      "embedding": {
+        "dimensions": 1536,
+        "similarity": "cosine",
+        "type": "knnVector"
+      }
+    }
+  }
+}
+```
+
+This tells Mongo that the `embedding` field in each document (in the `tiny_tweets_vectors` collection) is a vector of 1536 dimensions (this is the size of embeddings used by OpenAI), and that we want to use cosine similarity to compare vectors. You don't need to worry too much about these values unless you want to use a different LLM to OpenAI entirely.
+
+The UI will ask you to review and confirm your choices, then you need to wait a minute or two while it generates the index. If all goes well, you should see something like this screen:
+
+![MongoDB Atlas index created](./docs/7_index_created.png)
+
+Now you're ready to query your data!
+
+### Run a test query
+
+You can do this by running
+
+```bash
+pnpm ts-node 3_query.ts
+```
+
+This sets up a connection to Atlas just like `2_load_and_index.ts` did, then it creates a [query engine](https://docs.llamaindex.ai/en/stable/understanding/querying/querying.html#getting-started) and runs a query against it.
+
+If all is well, you should get a nuanced opinion about web frameworks.

apps/mongodb/package.json

@@ -0,0 +1,17 @@
+{
+  "version": "0.0.3",
+  "private": true,
+  "name": "mongodb-llamaindexts",
+  "dependencies": {
+    "llamaindex": "workspace:*",
+    "dotenv": "^16.3.1",
+    "mongodb": "^6.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18.18.6",
+    "ts-node": "^10.9.1"
+  },
+  "scripts": {
+    "lint": "eslint ."
+  }
+}

apps/simple/CHANGELOG.md

@@ -1,5 +1,24 @@
 # simple
 
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [3bab231]
+  - llamaindex@0.0.37
+
+## 0.0.34
+
+### Patch Changes
+
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+  - llamaindex@0.0.36
+
 ## 0.0.33
 
 ### Patch Changes

apps/simple/anthropic.ts

@@ -4,8 +4,6 @@ import { Anthropic } from "llamaindex";
   const anthropic = new Anthropic();
   const result = await anthropic.chat([
     { content: "You want to talk in rhymes.", role: "system" },
-    { content: "Hello, world!", role: "user" },
-    { content: "Hello!", role: "assistant" },
     {
       content:
         "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",

apps/simple/assemblyai.ts

@@ -0,0 +1,59 @@
+import { program } from "commander";
+import { AudioTranscriptReader, TranscribeParams } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+import { VectorStoreIndex } from "../../packages/core/src/indices";
+
+program
+  .option(
+    "-a, --audio-url [string]",
+    "URL or path of the audio file to transcribe",
+  )
+  .option("-i, --transcript-id [string]", "ID of the AssemblyAI transcript")
+  .action(async (options) => {
+    if (!process.env.ASSEMBLYAI_API_KEY) {
+      console.log("No ASSEMBLYAI_API_KEY found in environment variables.");
+      return;
+    }
+
+    const reader = new AudioTranscriptReader();
+    let params: TranscribeParams | string;
+    if (options.audioUrl) {
+      params = {
+        audio: options.audioUrl,
+      };
+    } else if (options.transcriptId) {
+      params = options.transcriptId;
+    } else {
+      console.log(
+        "You must provide either an --audio-url or a --transcript-id",
+      );
+      return;
+    }
+
+    const documents = await reader.loadData(params);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Ask a question: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

apps/simple/llmStream.ts

@@ -1,47 +0,0 @@
-import { ChatMessage, SimpleChatEngine } from "llamaindex";
-import { stdin as input, stdout as output } from "node:process";
-import readline from "node:readline/promises";
-import { Anthropic } from "../../packages/core/src/llm/LLM";
-
-async function main() {
-  const query: string = `
-Where is Istanbul?
-  `;
-
-  // const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 });
-  const llm = new Anthropic();
-  const message: ChatMessage = { content: query, role: "user" };
-
-  //TODO: Add callbacks later
-
-  //Stream Complete
-  //Note: Setting streaming flag to true or false will auto-set your return type to
-  //either an AsyncGenerator or a Response.
-  // Omitting the streaming flag automatically sets streaming to false
-
-  const chatEngine: SimpleChatEngine = new SimpleChatEngine({
-    chatHistory: undefined,
-    llm: llm,
-  });
-
-  const rl = readline.createInterface({ input, output });
-  while (true) {
-    const query = await rl.question("Query: ");
-
-    if (!query) {
-      break;
-    }
-
-    //Case 1: .chat(query, undefined, true) => Stream
-    //Case 2: .chat(query, undefined, false) => Response object
-    //Case 3: .chat(query, undefined) => Response object
-    const chatStream = await chatEngine.chat(query, undefined, true);
-    var accumulated_result = "";
-    for await (const part of chatStream) {
-      accumulated_result += part;
-      process.stdout.write(part);
-    }
-  }
-}
-
-main();

apps/simple/mongo.ts

@@ -1,6 +1,6 @@
 import { MongoClient } from "mongodb";
-import { Document } from "../../packages/core/src/Node";
 import { VectorStoreIndex } from "../../packages/core/src/indices";
+import { Document } from "../../packages/core/src/Node";
 import { SimpleMongoReader } from "../../packages/core/src/readers/SimpleMongoReader";
 
 import { stdin as input, stdout as output } from "node:process";

apps/simple/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.33",
+  "version": "0.0.35",
   "private": true,
   "name": "simple",
   "dependencies": {

apps/simple/pg-vector-store/README.md

@@ -0,0 +1,33 @@
+# Postgres Vector Store
+
+There are two scripts available here: load-docs.ts and query.ts
+
+## Prerequisites
+
+You'll need a postgres database instance against which to run these scripts. A simple docker command would look like this:
+
+> `docker run -d --rm --name vector-db -p 5432:5432 -e "POSTGRES_HOST_AUTH_METHOD=trust" ankane/pgvector`
+
+Set the PGHOST and PGUSER (and PGPASSWORD) environment variables to match your database setup.
+
+You'll also need a value for OPENAI_API_KEY in your environment.
+
+**NOTE:** Using `--rm` in the example docker command above means that the vector store will be deleted every time the container is stopped. For production purposes, use a volume to ensure persistence across restarts.
+
+## Setup and Loading Docs
+
+Read and follow the instructions in the README.md file located one directory up to make sure your JS/TS dependencies are set up. The commands listed below are also run from that parent directory.
+
+To import documents and save the embedding vectors to your database:
+
+> `npx ts-node pg-vector-store/load-docs.ts data`
+
+where data is the directory containing your input files. Using the _data_ directory in the example above will read all of the files in that directory using the llamaindexTS default readers for each file type.
+
+## RAG Querying
+
+To query using the resulting vector store:
+
+> `npx ts-node pg-vector-store/query.ts`
+
+The script will prompt for a question, then process and present the answer using the PGVectorStore data and your OpenAI API key. It will continue to prompt until you enter `q`, `quit` or `exit` as the next query.

apps/simple/pg-vector-store/load-docs.ts

@@ -0,0 +1,68 @@
+// load-docs.ts
+import fs from "fs/promises";
+import {
+  SimpleDirectoryReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { PGVectorStore } from "../../../packages/core/src/storage/vectorStore/PGVectorStore";
+
+async function getSourceFilenames(sourceDir: string) {
+  return await fs
+    .readdir(sourceDir)
+    .then((fileNames) => fileNames.map((file) => sourceDir + "/" + file));
+}
+
+function callback(
+  category: string,
+  name: string,
+  status: any,
+  message: string = "",
+): boolean {
+  console.log(category, name, status, message);
+  return true;
+}
+
+async function main(args: any) {
+  const sourceDir: string = args.length > 2 ? args[2] : "../data";
+
+  console.log(`Finding documents in ${sourceDir}`);
+  const fileList = await getSourceFilenames(sourceDir);
+  const count = fileList.length;
+  console.log(`Found ${count} files`);
+
+  console.log(`Importing contents from ${count} files in ${sourceDir}`);
+  var fileName = "";
+  try {
+    // Passing callback fn to the ctor here
+    // will enable looging to console.
+    // See callback fn, defined above.
+    const rdr = new SimpleDirectoryReader(callback);
+    const docs = await rdr.loadData({ directoryPath: sourceDir });
+
+    const pgvs = new PGVectorStore();
+    pgvs.setCollection(sourceDir);
+    pgvs.clearCollection();
+
+    const ctx = await storageContextFromDefaults({ vectorStore: pgvs });
+
+    console.debug("  - creating vector store");
+    const index = await VectorStoreIndex.fromDocuments(docs, {
+      storageContext: ctx,
+    });
+    console.debug("  - done.");
+  } catch (err) {
+    console.error(fileName, err);
+    console.log(
+      "If your PGVectorStore init failed, make sure to set env vars for PGUSER or USER, PGHOST, PGPORT and PGPASSWORD as needed.",
+    );
+    process.exit(1);
+  }
+
+  console.log(
+    "Done. Try running query.ts to ask questions against the imported embeddings.",
+  );
+  process.exit(0);
+}
+
+main(process.argv).catch((err) => console.error(err));

apps/simple/pg-vector-store/query.ts

@@ -0,0 +1,67 @@
+import { VectorStoreIndex } from "../../../packages/core/src/indices/vectorStore/VectorStoreIndex";
+import { serviceContextFromDefaults } from "../../../packages/core/src/ServiceContext";
+import { PGVectorStore } from "../../../packages/core/src/storage/vectorStore/PGVectorStore";
+
+async function main() {
+  const readline = require("readline").createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  try {
+    const pgvs = new PGVectorStore();
+    // Optional - set your collection name, default is no filter on this field.
+    // pgvs.setCollection();
+
+    const ctx = serviceContextFromDefaults();
+    const index = await VectorStoreIndex.fromVectorStore(pgvs, ctx);
+
+    // Query the index
+    const queryEngine = await index.asQueryEngine();
+
+    let question = "";
+    while (!isQuit(question)) {
+      question = await getUserInput(readline);
+
+      if (isQuit(question)) {
+        readline.close();
+        process.exit(0);
+      }
+
+      try {
+        const answer = await queryEngine.query(question);
+        console.log(answer.response);
+      } catch (error) {
+        console.error("Error:", error);
+      }
+    }
+  } catch (err) {
+    console.error(err);
+    console.log(
+      "If your PGVectorStore init failed, make sure to set env vars for PGUSER or USER, PGHOST, PGPORT and PGPASSWORD as needed.",
+    );
+    process.exit(1);
+  }
+}
+
+function isQuit(question: string) {
+  return ["q", "quit", "exit"].includes(question.trim().toLowerCase());
+}
+
+// Function to get user input as a promise
+function getUserInput(readline: any): Promise<string> {
+  return new Promise((resolve) => {
+    readline.question(
+      "What would you like to know?\n>",
+      (userInput: string) => {
+        resolve(userInput);
+      },
+    );
+  });
+}
+
+main()
+  .catch(console.error)
+  .finally(() => {
+    process.exit(1);
+  });

apps/simple/vectorIndexAnthropic.ts

@@ -2,7 +2,10 @@ import fs from "node:fs/promises";
 
 import {
   Anthropic,
+  anthropicTextQaPrompt,
+  CompactAndRefine,
   Document,
+  ResponseSynthesizer,
   serviceContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
@@ -18,12 +21,20 @@ async function main() {
 
   // Split text and create embeddings. Store them in a VectorStoreIndex
   const serviceContext = serviceContextFromDefaults({ llm: new Anthropic() });
+
+  const responseSynthesizer = new ResponseSynthesizer({
+    responseBuilder: new CompactAndRefine(
+      serviceContext,
+      anthropicTextQaPrompt,
+    ),
+  });
+
   const index = await VectorStoreIndex.fromDocuments([document], {
     serviceContext,
   });
 
   // Query the index
-  const queryEngine = index.asQueryEngine();
+  const queryEngine = index.asQueryEngine({ responseSynthesizer });
   const response = await queryEngine.query(
     "What did the author do in college?",
   );

examples/anthropic.ts

@@ -4,8 +4,6 @@ import { Anthropic } from "llamaindex";
   const anthropic = new Anthropic();
   const result = await anthropic.chat([
     { content: "You want to talk in rhymes.", role: "system" },
-    { content: "Hello, world!", role: "user" },
-    { content: "Hello!", role: "assistant" },
     {
       content:
         "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",

examples/assemblyai.ts

@@ -0,0 +1,59 @@
+import { program } from "commander";
+import { AudioTranscriptReader, TranscribeParams } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+import { VectorStoreIndex } from "../../packages/core/src/indices";
+
+program
+  .option(
+    "-a, --audio-url [string]",
+    "URL or path of the audio file to transcribe",
+  )
+  .option("-i, --transcript-id [string]", "ID of the AssemblyAI transcript")
+  .action(async (options) => {
+    if (!process.env.ASSEMBLYAI_API_KEY) {
+      console.log("No ASSEMBLYAI_API_KEY found in environment variables.");
+      return;
+    }
+
+    const reader = new AudioTranscriptReader();
+    let params: TranscribeParams | string;
+    if (options.audioUrl) {
+      params = {
+        audio: options.audioUrl,
+      };
+    } else if (options.transcriptId) {
+      params = options.transcriptId;
+    } else {
+      console.log(
+        "You must provide either an --audio-url or a --transcript-id",
+      );
+      return;
+    }
+
+    const documents = await reader.loadData(params);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Ask a question: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

examples/clip.ts

@@ -0,0 +1,33 @@
+import { ClipEmbedding, similarity, SimilarityType } from "llamaindex";
+
+async function main() {
+  const clip = new ClipEmbedding();
+
+  // Get text embeddings
+  const text1 = "a car";
+  const textEmbedding1 = await clip.getTextEmbedding(text1);
+  const text2 = "a football match";
+  const textEmbedding2 = await clip.getTextEmbedding(text2);
+
+  // Get image embedding
+  const image =
+    "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg";
+  const imageEmbedding = await clip.getImageEmbedding(image);
+
+  // Calc similarity
+  const sim1 = similarity(
+    textEmbedding1,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+  const sim2 = similarity(
+    textEmbedding2,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+
+  console.log(`Similarity between "${text1}" and the image is ${sim1}`);
+  console.log(`Similarity between "${text2}" and the image is ${sim2}`);
+}
+
+main();

examples/llmStream.ts

@@ -1,47 +0,0 @@
-import { ChatMessage, SimpleChatEngine } from "llamaindex";
-import { stdin as input, stdout as output } from "node:process";
-import readline from "node:readline/promises";
-import { Anthropic } from "../../packages/core/src/llm/LLM";
-
-async function main() {
-  const query: string = `
-Where is Istanbul?
-  `;
-
-  // const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 });
-  const llm = new Anthropic();
-  const message: ChatMessage = { content: query, role: "user" };
-
-  //TODO: Add callbacks later
-
-  //Stream Complete
-  //Note: Setting streaming flag to true or false will auto-set your return type to
-  //either an AsyncGenerator or a Response.
-  // Omitting the streaming flag automatically sets streaming to false
-
-  const chatEngine: SimpleChatEngine = new SimpleChatEngine({
-    chatHistory: undefined,
-    llm: llm,
-  });
-
-  const rl = readline.createInterface({ input, output });
-  while (true) {
-    const query = await rl.question("Query: ");
-
-    if (!query) {
-      break;
-    }
-
-    //Case 1: .chat(query, undefined, true) => Stream
-    //Case 2: .chat(query, undefined, false) => Response object
-    //Case 3: .chat(query, undefined) => Response object
-    const chatStream = await chatEngine.chat(query, undefined, true);
-    var accumulated_result = "";
-    for await (const part of chatStream) {
-      accumulated_result += part;
-      process.stdout.write(part);
-    }
-  }
-}
-
-main();

examples/vectorIndexAnthropic.ts

@@ -2,7 +2,10 @@ import fs from "node:fs/promises";
 
 import {
   Anthropic,
+  anthropicTextQaPrompt,
+  CompactAndRefine,
   Document,
+  ResponseSynthesizer,
   serviceContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
@@ -18,12 +21,20 @@ async function main() {
 
   // Split text and create embeddings. Store them in a VectorStoreIndex
   const serviceContext = serviceContextFromDefaults({ llm: new Anthropic() });
+
+  const responseSynthesizer = new ResponseSynthesizer({
+    responseBuilder: new CompactAndRefine(
+      serviceContext,
+      anthropicTextQaPrompt,
+    ),
+  });
+
   const index = await VectorStoreIndex.fromDocuments([document], {
     serviceContext,
   });
 
   // Query the index
-  const queryEngine = index.asQueryEngine();
+  const queryEngine = index.asQueryEngine({ responseSynthesizer });
   const response = await queryEngine.query(
     "What did the author do in college?",
   );

package.json

@@ -13,8 +13,8 @@
   "devDependencies": {
     "@changesets/cli": "^2.26.2",
     "@turbo/gen": "^1.10.16",
-    "@types/jest": "^29.5.8",
-    "eslint": "^8.53.0",
+    "@types/jest": "^29.5.10",
+    "eslint": "^8.54.0",
     "eslint-config-custom": "workspace:*",
     "husky": "^8.0.3",
     "jest": "^29.7.0",

packages/core/CHANGELOG.md

@@ -1,5 +1,22 @@
 # llamaindex
 
+## 0.0.37
+
+### Patch Changes
+
+- 3bab231: Fixed errors (#225 and #226) Thanks @marcusschiesser
+
+## 0.0.36
+
+### Patch Changes
+
+- Support for Claude 2.1
+- Add AssemblyAI integration (thanks @Swimburger)
+- Use cryptoJS (thanks @marcusschiesser)
+- Add PGVectorStore (thanks @mtutty)
+- Add CLIP embeddings (thanks @marcusschiesser)
+- Add MongoDB support (thanks @marcusschiesser)
+
 ## 0.0.35
 
 ### Patch Changes

packages/core/package.json

@@ -1,19 +1,24 @@
 {
   "name": "llamaindex",
-  "version": "0.0.35",
+  "version": "0.0.37",
   "license": "MIT",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.9.0",
+    "@anthropic-ai/sdk": "^0.9.1",
     "@notionhq/client": "^2.2.13",
-    "js-tiktoken": "^1.0.7",
+    "@xenova/transformers": "^2.8.0",
+    "assemblyai": "^3.1.3",
+    "crypto-js": "^4.2.0",
+    "js-tiktoken": "^1.0.8",
     "lodash": "^4.17.21",
     "mammoth": "^1.6.0",
     "md-utils-ts": "^2.0.0",
-    "mongodb": "^6.2.0",
+    "mongodb": "^6.3.0",
     "notion-md-crawler": "^0.0.2",
-    "openai": "^4.16.1",
+    "openai": "^4.20.0",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
+    "pg": "^8.11.3",
+    "pgvector": "^0.1.5",
     "portkey-ai": "^0.1.16",
     "rake-modified": "^1.0.8",
     "replicate": "^0.21.1",
@@ -22,14 +27,16 @@
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
-    "@types/lodash": "^4.14.200",
-    "@types/node": "^18.18.8",
-    "@types/papaparse": "^5.3.10",
-    "@types/pdf-parse": "^1.1.3",
-    "@types/uuid": "^9.0.6",
+    "@types/crypto-js": "^4.2.1",
+    "@types/lodash": "^4.14.202",
+    "@types/node": "^18.18.12",
+    "@types/papaparse": "^5.3.13",
+    "@types/pdf-parse": "^1.1.4",
+    "@types/pg": "^8.10.7",
+    "@types/uuid": "^9.0.7",
     "node-stdlib-browser": "^1.2.0",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "^5.3.2"
   },
   "engines": {
     "node": ">=18.0.0"
@@ -44,4 +51,4 @@
     "build": "tsup src/index.ts --format esm,cjs --dts",
     "dev": "tsup src/index.ts --format esm,cjs --dts --watch"
   }
-}
+}

packages/core/src/ChatEngine.ts

@@ -30,7 +30,7 @@ export interface ChatEngine {
     T extends boolean | undefined = undefined,
     R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
   >(
-    message: string,
+    message: MessageContent,
     chatHistory?: ChatMessage[],
     streaming?: T,
   ): Promise<R>;
@@ -56,7 +56,11 @@ export class SimpleChatEngine implements ChatEngine {
   async chat<
     T extends boolean | undefined = undefined,
     R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(message: string, chatHistory?: ChatMessage[], streaming?: T): Promise<R> {
+  >(
+    message: MessageContent,
+    chatHistory?: ChatMessage[],
+    streaming?: T,
+  ): Promise<R> {
     //Streaming option
     if (streaming) {
       return this.streamChat(message, chatHistory) as R;
@@ -72,7 +76,7 @@ export class SimpleChatEngine implements ChatEngine {
   }
 
   protected async *streamChat(
-    message: string,
+    message: MessageContent,
     chatHistory?: ChatMessage[],
   ): AsyncGenerator<string, void, unknown> {
     chatHistory = chatHistory ?? this.chatHistory;
@@ -144,14 +148,14 @@ export class CondenseQuestionChatEngine implements ChatEngine {
     T extends boolean | undefined = undefined,
     R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
   >(
-    message: string,
+    message: MessageContent,
     chatHistory?: ChatMessage[] | undefined,
     streaming?: T,
   ): Promise<R> {
     chatHistory = chatHistory ?? this.chatHistory;
 
     const condensedQuestion = (
-      await this.condenseQuestion(chatHistory, message)
+      await this.condenseQuestion(chatHistory, extractText(message))
     ).message.content;
 
     const response = await this.queryEngine.query(condensedQuestion);
@@ -256,7 +260,7 @@ export class ContextChatEngine implements ChatEngine {
     T extends boolean | undefined = undefined,
     R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
   >(
-    message: string,
+    message: MessageContent,
     chatHistory?: ChatMessage[] | undefined,
     streaming?: T,
   ): Promise<R> {
@@ -272,7 +276,10 @@ export class ContextChatEngine implements ChatEngine {
       type: "wrapper",
       tags: ["final"],
     };
-    const context = await this.contextGenerator.generate(message, parentEvent);
+    const context = await this.contextGenerator.generate(
+      extractText(message),
+      parentEvent,
+    );
 
     chatHistory.push({ content: message, role: "user" });
 
@@ -291,7 +298,7 @@ export class ContextChatEngine implements ChatEngine {
   }
 
   protected async *streamChat(
-    message: string,
+    message: MessageContent,
     chatHistory?: ChatMessage[] | undefined,
   ): AsyncGenerator<string, void, unknown> {
     chatHistory = chatHistory ?? this.chatHistory;
@@ -301,7 +308,10 @@ export class ContextChatEngine implements ChatEngine {
       type: "wrapper",
       tags: ["final"],
     };
-    const context = await this.contextGenerator.generate(message, parentEvent);
+    const context = await this.contextGenerator.generate(
+      extractText(message),
+      parentEvent,
+    );
 
     chatHistory.push({ content: message, role: "user" });
 
@@ -330,8 +340,8 @@ export class ContextChatEngine implements ChatEngine {
 
 export interface MessageContentDetail {
   type: "text" | "image_url";
-  text: string;
-  image_url: { url: string };
+  text?: string;
+  image_url?: { url: string };
 }
 
 /**
@@ -339,6 +349,24 @@ export interface MessageContentDetail {
  */
 export type MessageContent = string | MessageContentDetail[];
 
+/**
+ * Extracts just the text from a multi-modal message or the message itself if it's just text.
+ *
+ * @param message The message to extract text from.
+ * @returns The extracted text
+ */
+function extractText(message: MessageContent): string {
+  if (Array.isArray(message)) {
+    // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
+    // so we can pass them to the context generator
+    return (message as MessageContentDetail[])
+      .filter((c) => c.type === "text")
+      .map((c) => c.text)
+      .join("\n\n");
+  }
+  return message;
+}
+
 /**
  * HistoryChatEngine is a ChatEngine that uses a `ChatHistory` object
  * to keeps track of chat's message history.
@@ -413,15 +441,8 @@ export class HistoryChatEngine {
     let requestMessages;
     let context;
     if (this.contextGenerator) {
-      if (Array.isArray(message)) {
-        // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
-        // so we can pass them to the context generator
-        message = (message as MessageContentDetail[])
-          .filter((c) => c.type === "text")
-          .map((c) => c.text)
-          .join("\n\n");
-      }
-      context = await this.contextGenerator.generate(message);
+      const textOnly = extractText(message);
+      context = await this.contextGenerator.generate(textOnly);
     }
     requestMessages = await chatHistory.requestMessages(
       context ? [context.message] : undefined,

packages/core/src/Node.ts

@@ -1,4 +1,4 @@
-import crypto from "crypto"; // TODO Node dependency
+import CryptoJS from "crypto-js";
 import { v4 as uuidv4 } from "uuid";
 
 export enum NodeRelationship {
@@ -175,13 +175,13 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
    * @returns
    */
   generateHash() {
-    const hashFunction = crypto.createHash("sha256");
+    const hashFunction = CryptoJS.algo.SHA256.create();
     hashFunction.update(`type=${this.getType()}`);
     hashFunction.update(
       `startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`,
     );
     hashFunction.update(this.getContent(MetadataMode.ALL));
-    return hashFunction.digest("base64");
+    return hashFunction.finalize().toString(CryptoJS.enc.Base64);
   }
 
   getType(): ObjectType {
@@ -272,12 +272,13 @@ export class Document<T extends Metadata = Metadata> extends TextNode<T> {
   }
 }
 
-export function jsonToNode(json: any) {
-  if (!json.type) {
+export function jsonToNode(json: any, type?: ObjectType) {
+  if (!json.type && !type) {
     throw new Error("Node type not found");
   }
+  const nodeType = type || json.type;
 
-  switch (json.type) {
+  switch (nodeType) {
     case ObjectType.TEXT:
       return new TextNode(json);
     case ObjectType.INDEX:
@@ -285,7 +286,7 @@ export function jsonToNode(json: any) {
     case ObjectType.DOCUMENT:
       return new Document(json);
     default:
-      throw new Error(`Invalid node type: ${json.type}`);
+      throw new Error(`Invalid node type: ${nodeType}`);
   }
 }
 

packages/core/src/Prompt.ts

@@ -36,6 +36,15 @@ Answer:`;
 
 export type TextQaPrompt = typeof defaultTextQaPrompt;
 
+export const anthropicTextQaPrompt = ({ context = "", query = "" }) => {
+  return `Context information:
+<context>
+${context}
+</context>
+Given the context information and not prior knowledge, answer the query.
+Query: ${query}`;
+};
+
 /*
 DEFAULT_SUMMARY_PROMPT_TMPL = (
     "Write a summary of the following. Try to use only the "

packages/core/src/QueryEngine.ts

@@ -1,4 +1,6 @@
 import { v4 as uuidv4 } from "uuid";
+import { Event } from "./callbacks/CallbackManager";
+import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   BaseQuestionGenerator,
@@ -10,8 +12,6 @@ import { CompactAndRefine, ResponseSynthesizer } from "./ResponseSynthesizer";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { QueryEngineTool, ToolMetadata } from "./Tool";
-import { Event } from "./callbacks/CallbackManager";
-import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
 
 /**
  * A query engine is a question answerer that can use one or more steps.

packages/core/src/ServiceContext.ts

@@ -1,8 +1,8 @@
-import { BaseEmbedding, OpenAIEmbedding } from "./Embedding";
+import { CallbackManager } from "./callbacks/CallbackManager";
+import { BaseEmbedding, OpenAIEmbedding } from "./embeddings";
+import { LLM, OpenAI } from "./llm/LLM";
 import { NodeParser, SimpleNodeParser } from "./NodeParser";
 import { PromptHelper } from "./PromptHelper";
-import { CallbackManager } from "./callbacks/CallbackManager";
-import { LLM, OpenAI } from "./llm/LLM";
 
 /**
  * The ServiceContext is a collection of components that are used in different parts of the application.

packages/core/src/embeddings/ClipEmbedding.ts

@@ -0,0 +1,78 @@
+import { MultiModalEmbedding } from "./MultiModalEmbedding";
+import { ImageType, readImage } from "./utils";
+
+export enum ClipEmbeddingModelType {
+  XENOVA_CLIP_VIT_BASE_PATCH32 = "Xenova/clip-vit-base-patch32",
+  XENOVA_CLIP_VIT_BASE_PATCH16 = "Xenova/clip-vit-base-patch16",
+}
+
+export class ClipEmbedding extends MultiModalEmbedding {
+  modelType: ClipEmbeddingModelType =
+    ClipEmbeddingModelType.XENOVA_CLIP_VIT_BASE_PATCH16;
+
+  private tokenizer: any;
+  private processor: any;
+  private visionModel: any;
+  private textModel: any;
+
+  async getTokenizer() {
+    if (!this.tokenizer) {
+      const { AutoTokenizer } = await import("@xenova/transformers");
+      this.tokenizer = await AutoTokenizer.from_pretrained(this.modelType);
+    }
+    return this.tokenizer;
+  }
+
+  async getProcessor() {
+    if (!this.processor) {
+      const { AutoProcessor } = await import("@xenova/transformers");
+      this.processor = await AutoProcessor.from_pretrained(this.modelType);
+    }
+    return this.processor;
+  }
+
+  async getVisionModel() {
+    if (!this.visionModel) {
+      const { CLIPVisionModelWithProjection } = await import(
+        "@xenova/transformers"
+      );
+      this.visionModel = await CLIPVisionModelWithProjection.from_pretrained(
+        this.modelType,
+      );
+    }
+
+    return this.visionModel;
+  }
+
+  async getTextModel() {
+    if (!this.textModel) {
+      const { CLIPTextModelWithProjection } = await import(
+        "@xenova/transformers"
+      );
+      this.textModel = await CLIPTextModelWithProjection.from_pretrained(
+        this.modelType,
+      );
+    }
+
+    return this.textModel;
+  }
+
+  async getImageEmbedding(image: ImageType): Promise<number[]> {
+    const loadedImage = await readImage(image);
+    const imageInputs = await (await this.getProcessor())(loadedImage);
+    const { image_embeds } = await (await this.getVisionModel())(imageInputs);
+    return image_embeds.data;
+  }
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    const textInputs = await (
+      await this.getTokenizer()
+    )([text], { padding: true, truncation: true });
+    const { text_embeds } = await (await this.getTextModel())(textInputs);
+    return text_embeds.data;
+  }
+
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getTextEmbedding(query);
+  }
+}

packages/core/src/embeddings/MultiModalEmbedding.ts

@@ -0,0 +1,17 @@
+import { BaseEmbedding } from "./types";
+import { ImageType } from "./utils";
+
+/*
+ * Base class for Multi Modal embeddings.
+ */
+
+export abstract class MultiModalEmbedding extends BaseEmbedding {
+  abstract getImageEmbedding(images: ImageType): Promise<number[]>;
+
+  async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
+    // Embed the input sequence of images asynchronously.
+    return Promise.all(
+      images.map((imgFilePath) => this.getImageEmbedding(imgFilePath)),
+    );
+  }
+}

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -0,0 +1,92 @@
+import { ClientOptions as OpenAIClientOptions } from "openai";
+import {
+  AzureOpenAIConfig,
+  getAzureBaseUrl,
+  getAzureConfigFromEnv,
+  getAzureModel,
+  shouldUseAzure,
+} from "../llm/azure";
+import { OpenAISession, getOpenAISession } from "../llm/openai";
+import { BaseEmbedding } from "./types";
+
+export enum OpenAIEmbeddingModelType {
+  TEXT_EMBED_ADA_002 = "text-embedding-ada-002",
+}
+
+export class OpenAIEmbedding extends BaseEmbedding {
+  model: OpenAIEmbeddingModelType;
+
+  // OpenAI session params
+  apiKey?: string = undefined;
+  maxRetries: number;
+  timeout?: number;
+  additionalSessionOptions?: Omit<
+    Partial<OpenAIClientOptions>,
+    "apiKey" | "maxRetries" | "timeout"
+  >;
+
+  session: OpenAISession;
+
+  constructor(init?: Partial<OpenAIEmbedding> & { azure?: AzureOpenAIConfig }) {
+    super();
+
+    this.model = OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002;
+
+    this.maxRetries = init?.maxRetries ?? 10;
+    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
+    this.additionalSessionOptions = init?.additionalSessionOptions;
+
+    if (init?.azure || shouldUseAzure()) {
+      const azureConfig = getAzureConfigFromEnv({
+        ...init?.azure,
+        model: getAzureModel(this.model),
+      });
+
+      if (!azureConfig.apiKey) {
+        throw new Error(
+          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable.",
+        );
+      }
+
+      this.apiKey = azureConfig.apiKey;
+      this.session =
+        init?.session ??
+        getOpenAISession({
+          azure: true,
+          apiKey: this.apiKey,
+          baseURL: getAzureBaseUrl(azureConfig),
+          maxRetries: this.maxRetries,
+          timeout: this.timeout,
+          defaultQuery: { "api-version": azureConfig.apiVersion },
+          ...this.additionalSessionOptions,
+        });
+    } else {
+      this.apiKey = init?.apiKey ?? undefined;
+      this.session =
+        init?.session ??
+        getOpenAISession({
+          apiKey: this.apiKey,
+          maxRetries: this.maxRetries,
+          timeout: this.timeout,
+          ...this.additionalSessionOptions,
+        });
+    }
+  }
+
+  private async getOpenAIEmbedding(input: string) {
+    const { data } = await this.session.openai.embeddings.create({
+      model: this.model,
+      input,
+    });
+
+    return data[0].embedding;
+  }
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    return this.getOpenAIEmbedding(text);
+  }
+
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getOpenAIEmbedding(query);
+  }
+}

packages/core/src/embeddings/index.ts

@@ -0,0 +1,5 @@
+export * from "./ClipEmbedding";
+export * from "./MultiModalEmbedding";
+export * from "./OpenAIEmbedding";
+export * from "./types";
+export * from "./utils";

packages/core/src/embeddings/types.ts

@@ -0,0 +1,24 @@
+import { similarity } from "./utils";
+
+/**
+ * Similarity type
+ * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
+ */
+export enum SimilarityType {
+  DEFAULT = "cosine",
+  DOT_PRODUCT = "dot_product",
+  EUCLIDEAN = "euclidean",
+}
+
+export abstract class BaseEmbedding {
+  similarity(
+    embedding1: number[],
+    embedding2: number[],
+    mode: SimilarityType = SimilarityType.DEFAULT,
+  ): number {
+    return similarity(embedding1, embedding2, mode);
+  }
+
+  abstract getTextEmbedding(text: string): Promise<number[]>;
+  abstract getQueryEmbedding(query: string): Promise<number[]>;
+}

packages/core/src/embeddings/utils.ts

@@ -1,33 +1,16 @@
-import { ClientOptions as OpenAIClientOptions } from "openai";
-
-import { DEFAULT_SIMILARITY_TOP_K } from "./constants";
-import {
-  AzureOpenAIConfig,
-  getAzureBaseUrl,
-  getAzureConfigFromEnv,
-  getAzureModel,
-  shouldUseAzure,
-} from "./llm/azure";
-import { OpenAISession, getOpenAISession } from "./llm/openai";
-import { VectorStoreQueryMode } from "./storage/vectorStore/types";
-
-/**
- * Similarity type
- * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
- */
-export enum SimilarityType {
-  DEFAULT = "cosine",
-  DOT_PRODUCT = "dot_product",
-  EUCLIDEAN = "euclidean",
-}
+import _ from "lodash";
+import { DEFAULT_SIMILARITY_TOP_K } from "../constants";
+import { VectorStoreQueryMode } from "../storage";
+import { SimilarityType } from "./types";
 
 /**
  * The similarity between two embeddings.
  * @param embedding1
  * @param embedding2
  * @param mode
- * @returns similartiy score with higher numbers meaning the two embeddings are more similar
+ * @returns similarity score with higher numbers meaning the two embeddings are more similar
  */
+
 export function similarity(
   embedding1: number[],
   embedding2: number[],
@@ -42,7 +25,6 @@ export function similarity(
   // will probably cause some avoidable loss of floating point precision
   // ml-distance is worth watching although they currently also use the naive
   // formulas
-
   function norm(x: number[]): number {
     let result = 0;
     for (let i = 0; i < x.length; i++) {
@@ -201,98 +183,14 @@ export function getTopKMMREmbeddings(
 
   return [resultSimilarities, resultIds];
 }
-
-export abstract class BaseEmbedding {
-  similarity(
-    embedding1: number[],
-    embedding2: number[],
-    mode: SimilarityType = SimilarityType.DEFAULT,
-  ): number {
-    return similarity(embedding1, embedding2, mode);
-  }
-
-  abstract getTextEmbedding(text: string): Promise<number[]>;
-  abstract getQueryEmbedding(query: string): Promise<number[]>;
-}
-
-enum OpenAIEmbeddingModelType {
-  TEXT_EMBED_ADA_002 = "text-embedding-ada-002",
-}
-
-export class OpenAIEmbedding extends BaseEmbedding {
-  model: OpenAIEmbeddingModelType;
-
-  // OpenAI session params
-  apiKey?: string = undefined;
-  maxRetries: number;
-  timeout?: number;
-  additionalSessionOptions?: Omit<
-    Partial<OpenAIClientOptions>,
-    "apiKey" | "maxRetries" | "timeout"
-  >;
-
-  session: OpenAISession;
-
-  constructor(init?: Partial<OpenAIEmbedding> & { azure?: AzureOpenAIConfig }) {
-    super();
-
-    this.model = OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002;
-
-    this.maxRetries = init?.maxRetries ?? 10;
-    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
-    this.additionalSessionOptions = init?.additionalSessionOptions;
-
-    if (init?.azure || shouldUseAzure()) {
-      const azureConfig = getAzureConfigFromEnv({
-        ...init?.azure,
-        model: getAzureModel(this.model),
-      });
-
-      if (!azureConfig.apiKey) {
-        throw new Error(
-          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable.",
-        );
-      }
-
-      this.apiKey = azureConfig.apiKey;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          azure: true,
-          apiKey: this.apiKey,
-          baseURL: getAzureBaseUrl(azureConfig),
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          defaultQuery: { "api-version": azureConfig.apiVersion },
-          ...this.additionalSessionOptions,
-        });
-    } else {
-      this.apiKey = init?.apiKey ?? undefined;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          apiKey: this.apiKey,
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          ...this.additionalSessionOptions,
-        });
-    }
-  }
-
-  private async getOpenAIEmbedding(input: string) {
-    const { data } = await this.session.openai.embeddings.create({
-      model: this.model,
-      input,
-    });
-
-    return data[0].embedding;
-  }
-
-  async getTextEmbedding(text: string): Promise<number[]> {
-    return this.getOpenAIEmbedding(text);
-  }
-
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getOpenAIEmbedding(query);
+export async function readImage(input: ImageType) {
+  const { RawImage } = await import("@xenova/transformers");
+  if (input instanceof Blob) {
+    return await RawImage.fromBlob(input);
+  } else if (_.isString(input) || input instanceof URL) {
+    return await RawImage.fromURL(input);
+  } else {
+    throw new Error(`Unsupported input type: ${typeof input}`);
   }
 }
+export type ImageType = string | Blob | URL;

packages/core/src/index.ts

@@ -1,6 +1,5 @@
 export * from "./ChatEngine";
 export * from "./ChatHistory";
-export * from "./Embedding";
 export * from "./GlobalsHelper";
 export * from "./Node";
 export * from "./NodeParser";
@@ -17,13 +16,16 @@ export * from "./TextSplitter";
 export * from "./Tool";
 export * from "./callbacks/CallbackManager";
 export * from "./constants";
+export * from "./embeddings";
 export * from "./indices";
 export * from "./llm/LLM";
+export * from "./readers/AssemblyAI";
 export * from "./readers/CSVReader";
 export * from "./readers/HTMLReader";
 export * from "./readers/MarkdownReader";
 export * from "./readers/NotionReader";
 export * from "./readers/PDFReader";
 export * from "./readers/SimpleDirectoryReader";
+export * from "./readers/SimpleMongoReader";
 export * from "./readers/base";
 export * from "./storage";

packages/core/src/indices/summary/SummaryIndex.ts

@@ -10,11 +10,11 @@ import {
   ServiceContext,
   serviceContextFromDefaults,
 } from "../../ServiceContext";
+import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   StorageContext,
   storageContextFromDefaults,
 } from "../../storage/StorageContext";
-import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   BaseIndex,
   BaseIndexInit,

packages/core/src/llm/LLM.ts

@@ -639,7 +639,7 @@ If a question does not make any sense, or is not factually coherent, explain why
 
 export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
   // both models have 100k context window, see https://docs.anthropic.com/claude/reference/selecting-a-model
-  "claude-2": { contextWindow: 100000 },
+  "claude-2": { contextWindow: 200000 },
   "claude-instant-1": { contextWindow: 100000 },
 };
 
@@ -705,10 +705,12 @@ export class Anthropic implements LLM {
         return (
           acc +
           `${
-            message.role === "assistant"
-              ? ANTHROPIC_AI_PROMPT
-              : ANTHROPIC_HUMAN_PROMPT
-          } ${message.content} `
+            message.role === "system"
+              ? ""
+              : message.role === "assistant"
+                ? ANTHROPIC_AI_PROMPT + " "
+                : ANTHROPIC_HUMAN_PROMPT + " "
+          }${message.content.trim()}`
         );
       }, "") + ANTHROPIC_AI_PROMPT
     );
@@ -729,6 +731,7 @@ export class Anthropic implements LLM {
       }
       return this.streamChat(messages, parentEvent) as R;
     }
+
     //Non-streaming
     const response = await this.session.anthropic.completions.create({
       model: this.model,

packages/core/src/readers/AssemblyAI.ts

@@ -0,0 +1,143 @@
+import {
+  AssemblyAI,
+  BaseServiceParams,
+  SubtitleFormat,
+  TranscribeParams,
+  TranscriptParagraph,
+  TranscriptSentence,
+} from "assemblyai";
+import { Document } from "../Node";
+import { BaseReader } from "./base";
+
+type AssemblyAIOptions = Partial<BaseServiceParams>;
+
+/**
+ * Base class for AssemblyAI Readers.
+ */
+abstract class AssemblyAIReader implements BaseReader {
+  protected client: AssemblyAI;
+
+  /**
+   * Creates a new AssemblyAI Reader.
+   * @param assemblyAIOptions The options to configure the AssemblyAI Reader.
+   * Configure the `assemblyAIOptions.apiKey` with your AssemblyAI API key, or configure it as the `ASSEMBLYAI_API_KEY` environment variable.
+   */
+  constructor(assemblyAIOptions?: AssemblyAIOptions) {
+    let options = assemblyAIOptions;
+    if (!options) {
+      options = {};
+    }
+    if (!options.apiKey) {
+      options.apiKey = process.env.ASSEMBLYAI_API_KEY;
+    }
+    if (!options.apiKey) {
+      throw new Error(
+        "No AssemblyAI API key provided. Pass an `apiKey` option, or configure the `ASSEMBLYAI_API_KEY` environment variable.",
+      );
+    }
+
+    this.client = new AssemblyAI(options as BaseServiceParams);
+  }
+
+  abstract loadData(...args: any[]): Promise<Document[]>;
+
+  protected async transcribeOrGetTranscript(params: TranscribeParams | string) {
+    if (typeof params === "string") {
+      return await this.client.transcripts.get(params);
+    } else {
+      return await this.client.transcripts.transcribe(params);
+    }
+  }
+
+  protected async getTranscriptId(params: TranscribeParams | string) {
+    if (typeof params === "string") {
+      return params;
+    } else {
+      return (await this.client.transcripts.transcribe(params)).id;
+    }
+  }
+}
+
+/**
+ * Transcribe audio and read the transcript as a document using AssemblyAI.
+ */
+class AudioTranscriptReader extends AssemblyAIReader {
+  /**
+   * Transcribe audio or get a transcript and load the transcript as a document using AssemblyAI.
+   * @param params Parameters to transcribe an audio file or get an existing transcript.
+   * @returns A promise that resolves to a single document containing the transcript text.
+   */
+  async loadData(params: TranscribeParams | string): Promise<Document[]> {
+    const transcript = await this.transcribeOrGetTranscript(params);
+    return [new Document({ text: transcript.text || undefined })];
+  }
+}
+
+/**
+ * Transcribe audio and return a document for each paragraph.
+ */
+class AudioTranscriptParagraphsReader extends AssemblyAIReader {
+  /**
+   * Transcribe audio or get a transcript, and returns a document for each paragraph.
+   * @param params The parameters to transcribe audio or get an existing transcript.
+   * @returns A promise that resolves to an array of documents, each containing a paragraph of the transcript.
+   */
+  async loadData(params: TranscribeParams | string): Promise<Document[]> {
+    let transcriptId = await this.getTranscriptId(params);
+    const paragraphsResponse =
+      await this.client.transcripts.paragraphs(transcriptId);
+    return paragraphsResponse.paragraphs.map(
+      (p: TranscriptParagraph) => new Document({ text: p.text }),
+    );
+  }
+}
+
+/**
+ * Transcribe audio and return a document for each sentence.
+ */
+class AudioTranscriptSentencesReader extends AssemblyAIReader {
+  /**
+   * Transcribe audio or get a transcript, and returns a document for each sentence.
+   * @param params The parameters to transcribe audio or get an existing transcript.
+   * @returns A promise that resolves to an array of documents, each containing a sentence of the transcript.
+   */
+  async loadData(params: TranscribeParams | string): Promise<Document[]> {
+    let transcriptId = await this.getTranscriptId(params);
+    const sentencesResponse =
+      await this.client.transcripts.sentences(transcriptId);
+    return sentencesResponse.sentences.map(
+      (p: TranscriptSentence) => new Document({ text: p.text }),
+    );
+  }
+}
+
+/**
+ * Transcribe audio a transcript and read subtitles for the transcript as `srt` or `vtt` format.
+ */
+class AudioSubtitlesReader extends AssemblyAIReader {
+  /**
+   * Transcribe audio or get a transcript and reads subtitles for the transcript as `srt` or `vtt` format.
+   * @param params The parameters to transcribe audio or get an existing transcript.
+   * @param subtitleFormat The format of the subtitles, either `srt` or `vtt`.
+   * @returns A promise that resolves a document containing the subtitles as the page content.
+   */
+  async loadData(
+    params: TranscribeParams | string,
+    subtitleFormat: SubtitleFormat = "srt",
+  ): Promise<Document[]> {
+    let transcriptId = await this.getTranscriptId(params);
+    const subtitles = await this.client.transcripts.subtitles(
+      transcriptId,
+      subtitleFormat,
+    );
+    return [new Document({ text: subtitles })];
+  }
+}
+
+export {
+  AudioSubtitlesReader,
+  AudioTranscriptParagraphsReader,
+  AudioTranscriptReader,
+  AudioTranscriptSentencesReader,
+};
+export type { AssemblyAIOptions, SubtitleFormat, TranscribeParams };

packages/core/src/readers/DocxReader.ts

@@ -1,7 +1,7 @@
 import mammoth from "mammoth";
 import { Document } from "../Node";
-import { GenericFileSystem } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
+import { GenericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
 export class DocxReader implements BaseReader {

packages/core/src/readers/SimpleMongoReader.ts

@@ -1,5 +1,5 @@
 import { MongoClient } from "mongodb";
-import { Document } from "../Node";
+import { Document, Metadata } from "../Node";
 import { BaseReader } from "./base";
 
 /**
@@ -13,39 +13,70 @@ export class SimpleMongoReader implements BaseReader {
   }
 
   /**
-   * Loads data from MongoDB collection
-   * @param {string} db_name - The name of the database to load.
-   * @param {string} collection_name - The name of the collection to load.
-   * @param {Number} [max_docs = 0] - Maximum number of documents to return. 0 means no limit.
-   * @param {Record<string, any>} [query_dict={}] - Specific query, as specified by MongoDB NodeJS documentation.
-   * @param {Record<string, any>} [query_options={}] - Specific query options, as specified by MongoDB NodeJS documentation.
-   * @param {Record<string, any>} [projection = {}] - Projection options, as specified by MongoDB NodeJS documentation.
-   * @returns {Promise<Document[]>}
+   * Flattens an array of strings or string arrays into a single-dimensional array of strings.
+   * @param texts - The array of strings or string arrays to flatten.
+   * @returns The flattened array of strings.
    */
-  async loadData(
-    db_name: string,
-    collection_name: string,
-    max_docs = 0,
-    //For later: Think about whether we want to pass generic objects in...
-    query_dict: Record<string, any> = {},
-    query_options: Record<string, any> = {},
-    projection: Record<string, any> = {},
-  ): Promise<Document[]> {
-    //Get items from collection using built-in functions
-    const cursor: Partial<Document>[] = await this.client
-      .db(db_name)
-      .collection(collection_name)
-      .find(query_dict, query_options)
-      .limit(max_docs)
-      .project(projection)
-      .toArray();
+  private flatten(texts: Array<string | string[]>): string[] {
+    return texts.reduce<string[]>(
+      (result, text) => result.concat(text instanceof Array ? text : [text]),
+      [],
+    );
+  }
+
+  /**
+   * Loads data from MongoDB collection
+   * @param {string} dbName - The name of the database to load.
+   * @param {string} collectionName - The name of the collection to load.
+   * @param {string[]} fieldNames - An array of field names to retrieve from each document. Defaults to ["text"].
+   * @param {string} separator - The separator to join multiple field values. Defaults to an empty string.
+   * @param {Record<string, any>} filterQuery - Specific query, as specified by MongoDB NodeJS documentation.
+   * @param {Number} maxDocs - The maximum number of documents to retrieve. Defaults to 0 (retrieve all documents).
+   * @param {string[]} metadataNames - An optional array of metadata field names. If specified extracts this information as metadata.
+   * @returns {Promise<Document[]>}
+   * @throws If a field specified in fieldNames or metadataNames is not found in a MongoDB document.
+   */
+  public async loadData(
+    dbName: string,
+    collectionName: string,
+    fieldNames: string[] = ["text"],
+    separator: string = "",
+    filterQuery: Record<string, any> = {},
+    maxDocs: number = 0,
+    metadataNames?: string[],
+  ): Promise<Document[]> {
+    const db = this.client.db(dbName);
+    // Get items from collection
+    const cursor = db
+      .collection(collectionName)
+      .find(filterQuery)
+      .limit(maxDocs);
 
-    //Aggregate results and return
     const documents: Document[] = [];
-    cursor.forEach((element: Partial<Document>) => {
-      //For later: Metadata filtering
-      documents.push(new Document({ text: JSON.stringify(element) }));
-    });
+
+    for await (const item of cursor) {
+      try {
+        const texts: Array<string | string[]> = fieldNames.map(
+          (name) => item[name],
+        );
+        const flattenedTexts = this.flatten(texts);
+        const text = flattenedTexts.join(separator);
+
+        let metadata: Metadata = {};
+        if (metadataNames) {
+          // extract metadata if fields are specified
+          metadata = Object.fromEntries(
+            metadataNames.map((name) => [name, item[name]]),
+          );
+        }
+
+        documents.push(new Document({ text, metadata }));
+      } catch (err) {
+        throw new Error(
+          `Field not found in Mongo document: ${(err as Error).message}`,
+        );
+      }
+    }
     return documents;
   }
 }

packages/core/src/storage/index.ts

@@ -7,5 +7,6 @@ export { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
 export * from "./indexStore/types";
 export { SimpleKVStore } from "./kvStore/SimpleKVStore";
 export * from "./kvStore/types";
+export { MongoDBAtlasVectorSearch } from "./vectorStore/MongoDBAtlasVectorStore";
 export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
 export * from "./vectorStore/types";

packages/core/src/storage/vectorStore/MongoDBAtlasVectorStore.ts

@@ -0,0 +1,164 @@
+import { BulkWriteOptions, Collection, MongoClient } from "mongodb";
+import { BaseNode, MetadataMode } from "../../Node";
+import {
+  MetadataFilters,
+  VectorStore,
+  VectorStoreQuery,
+  VectorStoreQueryResult,
+} from "./types";
+import { metadataDictToNode, nodeToMetadata } from "./utils";
+
+// Utility function to convert metadata filters to MongoDB filter
+function toMongoDBFilter(
+  standardFilters: MetadataFilters,
+): Record<string, any> {
+  const filters: Record<string, any> = {};
+  for (const filter of standardFilters.filters) {
+    filters[filter.key] = filter.value;
+  }
+  return filters;
+}
+
+// MongoDB Atlas Vector Store class implementing VectorStore
+export class MongoDBAtlasVectorSearch implements VectorStore {
+  storesText: boolean = true;
+  flatMetadata: boolean = true;
+
+  mongodbClient: MongoClient;
+  indexName: string;
+  embeddingKey: string;
+  idKey: string;
+  textKey: string;
+  metadataKey: string;
+  insertOptions?: BulkWriteOptions;
+  private collection: Collection;
+
+  constructor(
+    init: Partial<MongoDBAtlasVectorSearch> & {
+      dbName: string;
+      collectionName: string;
+    },
+  ) {
+    if (init.mongodbClient) {
+      this.mongodbClient = init.mongodbClient;
+    } else {
+      const mongoUri = process.env.MONGODB_URI;
+      if (!mongoUri) {
+        throw new Error(
+          "Must specify MONGODB_URI via env variable if not directly passing in client.",
+        );
+      }
+      this.mongodbClient = new MongoClient(mongoUri);
+    }
+
+    this.collection = this.mongodbClient
+      .db(init.dbName ?? "default_db")
+      .collection(init.collectionName ?? "default_collection");
+    this.indexName = init.indexName ?? "default";
+    this.embeddingKey = init.embeddingKey ?? "embedding";
+    this.idKey = init.idKey ?? "id";
+    this.textKey = init.textKey ?? "text";
+    this.metadataKey = init.metadataKey ?? "metadata";
+    this.insertOptions = init.insertOptions;
+  }
+
+  async add(nodes: BaseNode[]): Promise<string[]> {
+    if (!nodes || nodes.length === 0) {
+      return [];
+    }
+    const dataToInsert = nodes.map((node) => {
+      const metadata = nodeToMetadata(
+        node,
+        true,
+        this.textKey,
+        this.flatMetadata,
+      );
+
+      return {
+        [this.idKey]: node.id_,
+        [this.embeddingKey]: node.getEmbedding(),
+        [this.textKey]: node.getContent(MetadataMode.NONE) || "",
+        [this.metadataKey]: metadata,
+      };
+    });
+
+    console.debug("Inserting data into MongoDB: ", dataToInsert);
+    const insertResult = await this.collection.insertMany(
+      dataToInsert,
+      this.insertOptions,
+    );
+    console.debug("Result of insert: ", insertResult);
+    return nodes.map((node) => node.id_);
+  }
+
+  async delete(refDocId: string, deleteOptions?: any): Promise<void> {
+    await this.collection.deleteOne(
+      {
+        [`${this.metadataKey}.ref_doc_id`]: refDocId,
+      },
+      deleteOptions,
+    );
+  }
+
+  get client(): any {
+    return this.mongodbClient;
+  }
+
+  async query(
+    query: VectorStoreQuery,
+    options?: any,
+  ): Promise<VectorStoreQueryResult> {
+    const params: any = {
+      queryVector: query.queryEmbedding,
+      path: this.embeddingKey,
+      numCandidates: query.similarityTopK * 10,
+      limit: query.similarityTopK,
+      index: this.indexName,
+    };
+
+    if (query.filters) {
+      params.filter = toMongoDBFilter(query.filters);
+    }
+
+    const queryField = { $vectorSearch: params };
+    const pipeline = [
+      queryField,
+      {
+        $project: {
+          score: { $meta: "vectorSearchScore" },
+          [this.embeddingKey]: 0,
+        },
+      },
+    ];
+
+    console.debug("Running query pipeline: ", pipeline);
+    const cursor = await this.collection.aggregate(pipeline);
+
+    const nodes: BaseNode[] = [];
+    const ids: string[] = [];
+    const similarities: number[] = [];
+
+    for await (const res of await cursor) {
+      const text = res[this.textKey];
+      const score = res.score;
+      const id = res[this.idKey];
+      const metadata = res[this.metadataKey];
+
+      const node = metadataDictToNode(metadata);
+      node.setContent(text);
+
+      ids.push(id);
+      nodes.push(node);
+      similarities.push(score);
+    }
+
+    const result = {
+      nodes,
+      similarities,
+      ids,
+    };
+
+    console.debug("Result of query (ids):", ids);
+    return result;
+  }
+}

packages/core/src/storage/vectorStore/PGVectorStore.ts

@@ -0,0 +1,275 @@
+import pg from "pg";
+import pgvector from "pgvector/pg";
+
+import { VectorStore, VectorStoreQuery, VectorStoreQueryResult } from "./types";
+
+import { BaseNode, Document, Metadata, MetadataMode } from "../../Node";
+import { GenericFileSystem } from "../FileSystem";
+
+export const PGVECTOR_SCHEMA = "public";
+export const PGVECTOR_TABLE = "llamaindex_embedding";
+
+/**
+ * Provides support for writing and querying vector data in Postgres.
+ */
+export class PGVectorStore implements VectorStore {
+  storesText: boolean = true;
+
+  private collection: string = "";
+
+  /*
+    FROM pg LIBRARY:
+    type Config = {
+      user?: string, // default process.env.PGUSER || process.env.USER
+      password?: string or function, //default process.env.PGPASSWORD
+      host?: string, // default process.env.PGHOST
+      database?: string, // default process.env.PGDATABASE || user
+      port?: number, // default process.env.PGPORT
+      connectionString?: string, // e.g. postgres://user:password@host:5432/database
+      ssl?: any, // passed directly to node.TLSSocket, supports all tls.connect options
+      types?: any, // custom type parsers
+      statement_timeout?: number, // number of milliseconds before a statement in query will time out, default is no timeout
+      query_timeout?: number, // number of milliseconds before a query call will timeout, default is no timeout
+      application_name?: string, // The name of the application that created this Client instance
+      connectionTimeoutMillis?: number, // number of milliseconds to wait for connection, default is no timeout
+      idle_in_transaction_session_timeout?: number // number of milliseconds before terminating any session with an open idle transaction, default is no timeout
+    }  
+  */
+  db?: pg.Client;
+
+  constructor() {}
+
+  /**
+   * Setter for the collection property.
+   * Using a collection allows for simple segregation of vector data,
+   * e.g. by user, source, or access-level.
+   * Leave/set blank to ignore the collection value when querying.
+   * @param coll Name for the collection.
+   */
+  setCollection(coll: string) {
+    this.collection = coll;
+  }
+
+  /**
+   * Getter for the collection property.
+   * Using a collection allows for simple segregation of vector data,
+   * e.g. by user, source, or access-level.
+   * Leave/set blank to ignore the collection value when querying.
+   * @returns The currently-set collection value.  Default is empty string.
+   */
+  getCollection(): string {
+    return this.collection;
+  }
+
+  private async getDb(): Promise<pg.Client> {
+    if (!this.db) {
+      try {
+        // Create DB connection
+        // Read connection params from env - see comment block above
+        const db = new pg.Client();
+        await db.connect();
+
+        // Check vector extension
+        db.query("CREATE EXTENSION IF NOT EXISTS vector");
+        await pgvector.registerType(db);
+
+        // Check schema, table(s), index(es)
+        await this.checkSchema(db);
+
+        // All good?  Keep the connection reference
+        this.db = db;
+      } catch (err: any) {
+        console.error(err);
+        return Promise.reject(err);
+      }
+    }
+
+    return Promise.resolve(this.db);
+  }
+
+  private async checkSchema(db: pg.Client) {
+    await db.query(`CREATE SCHEMA IF NOT EXISTS ${PGVECTOR_SCHEMA}`);
+
+    const tbl = `CREATE TABLE IF NOT EXISTS ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE}(
+      id uuid DEFAULT gen_random_uuid() PRIMARY KEY,
+      external_id VARCHAR,
+      collection VARCHAR,
+      document TEXT,
+      metadata JSONB DEFAULT '{}',
+      embeddings VECTOR(1536)
+    )`;
+    await db.query(tbl);
+
+    const idxs = `CREATE INDEX IF NOT EXISTS idx_${PGVECTOR_TABLE}_external_id ON ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} (external_id);
+      CREATE INDEX IF NOT EXISTS idx_${PGVECTOR_TABLE}_collection ON ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} (collection);`;
+    await db.query(idxs);
+
+    // TODO add IVFFlat or HNSW indexing?
+    return db;
+  }
+
+  // isEmbeddingQuery?: boolean | undefined;
+
+  /**
+   * Connects to the database specified in environment vars.
+   * This method also checks and creates the vector extension,
+   * the destination table and indexes if not found.
+   * @returns A connection to the database, or the error encountered while connecting/setting up.
+   */
+  client() {
+    return this.getDb();
+  }
+
+  /**
+   * Delete all vector records for the specified collection.
+   * NOTE: Uses the collection property controlled by setCollection/getCollection.
+   * @returns The result of the delete query.
+   */
+  async clearCollection() {
+    const sql: string = `DELETE FROM ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} 
+      WHERE collection = $1`;
+
+    const db = (await this.getDb()) as pg.Client;
+    const ret = await db.query(sql, [this.collection]);
+
+    return ret;
+  }
+
+  /**
+   * Adds vector record(s) to the table.
+   * NOTE: Uses the collection property controlled by setCollection/getCollection.
+   * @param embeddingResults The Nodes to be inserted, optionally including metadata tuples.
+   * @returns A list of zero or more id values for the created records.
+   */
+  async add(embeddingResults: BaseNode<Metadata>[]): Promise<string[]> {
+    if (embeddingResults.length == 0) {
+      console.debug("Empty list sent to PGVectorStore::add");
+      return Promise.resolve([]);
+    }
+
+    const sql: string = `INSERT INTO ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} 
+      (id, external_id, collection, document, metadata, embeddings) 
+      VALUES ($1, $2, $3, $4, $5, $6)`;
+
+    const db = (await this.getDb()) as pg.Client;
+
+    let ret: string[] = [];
+    for (let index = 0; index < embeddingResults.length; index++) {
+      const row = embeddingResults[index];
+
+      let id: any = row.id_.length ? row.id_ : null;
+      let meta = row.metadata || {};
+      meta.create_date = new Date();
+
+      const params = [
+        id,
+        "",
+        this.collection,
+        row.getContent(MetadataMode.EMBED),
+        meta,
+        "[" + row.getEmbedding().join(",") + "]",
+      ];
+
+      try {
+        const result = await db.query(sql, params);
+
+        if (result.rows.length) {
+          id = result.rows[0].id as string;
+          ret.push(id);
+        }
+      } catch (err) {
+        const msg = `${err}`;
+        console.log(msg, err);
+      }
+    }
+
+    return Promise.resolve(ret);
+  }
+
+  /**
+   * Deletes a single record from the database by id.
+   * NOTE: Uses the collection property controlled by setCollection/getCollection.
+   * @param refDocId Unique identifier for the record to delete.
+   * @param deleteKwargs Required by VectorStore interface.  Currently ignored.
+   * @returns Promise that resolves if the delete query did not throw an error.
+   */
+  async delete(refDocId: string, deleteKwargs?: any): Promise<void> {
+    const collectionCriteria = this.collection.length
+      ? "AND collection = $2"
+      : "";
+    const sql: string = `DELETE FROM ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} 
+      WHERE id = $1 ${collectionCriteria}`;
+
+    const db = (await this.getDb()) as pg.Client;
+    const params = this.collection.length
+      ? [refDocId, this.collection]
+      : [refDocId];
+    await db.query(sql, params);
+    return Promise.resolve();
+  }
+
+  /**
+   * Query the vector store for the closest matching data to the query embeddings
+   * @param query The VectorStoreQuery to be used
+   * @param options Required by VectorStore interface.  Currently ignored.
+   * @returns Zero or more Document instances with data from the vector store.
+   */
+  async query(
+    query: VectorStoreQuery,
+    options?: any,
+  ): Promise<VectorStoreQueryResult> {
+    // TODO QUERY TYPES:
+    //    Distance:       SELECT embedding <-> $1 AS distance FROM items;
+    //    Inner Product:  SELECT (embedding <#> $1) * -1 AS inner_product FROM items;
+    //    Cosine Sim:     SELECT 1 - (embedding <=> $1) AS cosine_similarity FROM items;
+
+    const embedding = "[" + query.queryEmbedding?.join(",") + "]";
+    const max = query.similarityTopK ?? 2;
+    const where = this.collection.length ? "WHERE collection = $2" : "";
+    // TODO Add collection filter if set
+    const sql = `SELECT 
+        v.*, 
+        embeddings <-> $1 s 
+      FROM ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} v
+      ${where}
+      ORDER BY s 
+      LIMIT ${max}
+    `;
+
+    const db = (await this.getDb()) as pg.Client;
+    const params = this.collection.length
+      ? [embedding, this.collection]
+      : [embedding];
+    const results = await db.query(sql, params);
+
+    const nodes = results.rows.map((row) => {
+      return new Document({
+        id_: row.id,
+        text: row.document,
+        metadata: row.metadata,
+        embedding: row.embeddings,
+      });
+    });
+
+    const ret = {
+      nodes: nodes,
+      similarities: results.rows.map((row) => row.s),
+      ids: results.rows.map((row) => row.id),
+    };
+
+    return Promise.resolve(ret);
+  }
+
+  /**
+   * Required by VectorStore interface.  Currently ignored.
+   * @param persistPath
+   * @param fs
+   * @returns Resolved Promise.
+   */
+  persist(
+    persistPath: string,
+    fs?: GenericFileSystem | undefined,
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+}

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -1,11 +1,11 @@
 import _ from "lodash";
 import * as path from "path";
+import { BaseNode } from "../../Node";
 import {
   getTopKEmbeddings,
   getTopKEmbeddingsLearner,
   getTopKMMREmbeddings,
-} from "../../Embedding";
-import { BaseNode } from "../../Node";
+} from "../../embeddings";
 import { GenericFileSystem, exists } from "../FileSystem";
 import { DEFAULT_FS, DEFAULT_PERSIST_DIR } from "../constants";
 import {

packages/core/src/storage/vectorStore/types.ts

@@ -1,5 +1,4 @@
 import { BaseNode } from "../../Node";
-import { GenericFileSystem } from "../FileSystem";
 
 export interface VectorStoreQueryResult {
   nodes?: BaseNode[];
@@ -62,10 +61,9 @@ export interface VectorStore {
   isEmbeddingQuery?: boolean;
   client(): any;
   add(embeddingResults: BaseNode[]): Promise<string[]>;
-  delete(refDocId: string, deleteKwargs?: any): Promise<void>;
+  delete(refDocId: string, deleteOptions?: any): Promise<void>;
   query(
     query: VectorStoreQuery,
     options?: any,
   ): Promise<VectorStoreQueryResult>;
-  persist(persistPath: string, fs?: GenericFileSystem): Promise<void>;
 }

packages/core/src/storage/vectorStore/utils.ts

@@ -0,0 +1,59 @@
+import { BaseNode, jsonToNode, Metadata, ObjectType } from "../../Node";
+
+const DEFAULT_TEXT_KEY = "text";
+
+export function validateIsFlat(obj: { [key: string]: any }): void {
+  for (let key in obj) {
+    if (typeof obj[key] === "object" && obj[key] !== null) {
+      throw new Error(`Value for metadata ${key} must not be another object`);
+    }
+  }
+}
+
+export function nodeToMetadata(
+  node: BaseNode,
+  removeText: boolean = false,
+  textField: string = DEFAULT_TEXT_KEY,
+  flatMetadata: boolean = false,
+): Metadata {
+  const nodeObj = node.toJSON();
+  const metadata = node.metadata;
+
+  if (flatMetadata) {
+    validateIsFlat(node.metadata);
+  }
+
+  if (removeText) {
+    nodeObj[textField] = "";
+  }
+
+  nodeObj["embedding"] = null;
+
+  metadata["_node_content"] = JSON.stringify(nodeObj);
+  metadata["_node_type"] = node.constructor.name.replace("_", ""); // remove leading underscore to be compatible with Python
+
+  metadata["document_id"] = node.sourceNode?.nodeId || "None";
+  metadata["doc_id"] = node.sourceNode?.nodeId || "None";
+  metadata["ref_doc_id"] = node.sourceNode?.nodeId || "None";
+
+  return metadata;
+}
+
+export function metadataDictToNode(metadata: Metadata): BaseNode {
+  const nodeContent = metadata["_node_content"];
+  if (!nodeContent) {
+    throw new Error("Node content not found in metadata.");
+  }
+  const nodeObj = JSON.parse(nodeContent);
+
+  // Note: we're using the name of the class stored in `_node_type`
+  // and not the type attribute to reconstruct
+  // the node. This way we're compatible with LlamaIndex Python
+  const node_type = metadata["_node_type"];
+  switch (node_type) {
+    case "IndexNode":
+      return jsonToNode(nodeObj, ObjectType.INDEX);
+    default:
+      return jsonToNode(nodeObj, ObjectType.TEXT);
+  }
+}

packages/core/src/tests/CallbackManager.test.ts

@@ -1,18 +1,18 @@
-import { OpenAIEmbedding } from "../Embedding";
+import {
+  CallbackManager,
+  RetrievalCallbackResponse,
+  StreamCallbackResponse,
+} from "../callbacks/CallbackManager";
+import { OpenAIEmbedding } from "../embeddings";
+import { SummaryIndex } from "../indices/summary";
+import { VectorStoreIndex } from "../indices/vectorStore/VectorStoreIndex";
+import { OpenAI } from "../llm/LLM";
 import { Document } from "../Node";
 import {
   ResponseSynthesizer,
   SimpleResponseBuilder,
 } from "../ResponseSynthesizer";
 import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext";
-import {
-  CallbackManager,
-  RetrievalCallbackResponse,
-  StreamCallbackResponse,
-} from "../callbacks/CallbackManager";
-import { SummaryIndex } from "../indices/summary";
-import { VectorStoreIndex } from "../indices/vectorStore/VectorStoreIndex";
-import { OpenAI } from "../llm/LLM";
 import { mockEmbeddingModel, mockLlmGeneration } from "./utility/mockOpenAI";
 
 // Mock the OpenAI getOpenAISession function during testing

packages/core/src/tests/Embedding.test.ts

@@ -1,4 +1,4 @@
-import { SimilarityType, similarity } from "../Embedding";
+import { similarity, SimilarityType } from "../embeddings";
 
 describe("similarity", () => {
   test("throws error on mismatched lengths", () => {

packages/core/src/tests/Node.test.ts

@@ -0,0 +1,15 @@
+import { TextNode } from "../Node";
+
+describe("TextNode", () => {
+  let node: TextNode;
+
+  beforeEach(() => {
+    node = new TextNode({ text: "Hello World" });
+  });
+
+  describe("generateHash", () => {
+    it("should generate a hash", () => {
+      expect(node.hash).toBe("nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do=");
+    });
+  });
+});

packages/core/src/tests/utility/mockOpenAI.ts

@@ -1,6 +1,6 @@
-import { OpenAIEmbedding } from "../../Embedding";
-import { globalsHelper } from "../../GlobalsHelper";
 import { CallbackManager, Event } from "../../callbacks/CallbackManager";
+import { OpenAIEmbedding } from "../../embeddings";
+import { globalsHelper } from "../../GlobalsHelper";
 import { ChatMessage, OpenAI } from "../../llm/LLM";
 
 export function mockLlmGeneration({

packages/create-llama/CHANGELOG.md

@@ -1,5 +1,17 @@
 # create-llama
 
+## 0.0.10
+
+### Patch Changes
+
+- Bugfixes (thanks @marcusschiesser)
+
+## 0.0.9
+
+### Patch Changes
+
+- acfe232: Deployment fixes (thanks @seldo)
+
 ## 0.0.8
 
 ### Patch Changes

packages/create-llama/create-app.ts

@@ -12,6 +12,14 @@ import terminalLink from "terminal-link";
 import type { InstallTemplateArgs } from "./templates";
 import { installTemplate } from "./templates";
 
+export type InstallAppArgs = Omit<
+  InstallTemplateArgs,
+  "appName" | "root" | "isOnline" | "customApiPath"
+> & {
+  appPath: string;
+  frontend: boolean;
+};
+
 export async function createApp({
   template,
   framework,
@@ -22,13 +30,8 @@ export async function createApp({
   eslint,
   frontend,
   openAIKey,
-}: Omit<
-  InstallTemplateArgs,
-  "appName" | "root" | "isOnline" | "customApiPath"
-> & {
-  appPath: string;
-  frontend: boolean;
-}): Promise<void> {
+  model,
+}: InstallAppArgs): Promise<void> {
   const root = path.resolve(appPath);
 
   if (!(await isWriteable(path.dirname(root)))) {
@@ -65,6 +68,7 @@ export async function createApp({
     isOnline,
     eslint,
     openAIKey,
+    model,
   };
 
   if (frontend) {

packages/create-llama/index.ts

@@ -8,7 +8,7 @@ import path from "path";
 import { blue, bold, cyan, green, red, yellow } from "picocolors";
 import prompts from "prompts";
 import checkForUpdate from "update-check";
-import { createApp } from "./create-app";
+import { InstallAppArgs, createApp } from "./create-app";
 import { getPkgManager } from "./helpers/get-pkg-manager";
 import { isFolderEmpty } from "./helpers/is-folder-empty";
 import { validateNpmName } from "./helpers/validate-pkg";
@@ -155,21 +155,22 @@ async function run(): Promise<void> {
     process.exit(1);
   }
 
-  const preferences = (conf.get("preferences") || {}) as Record<
-    string,
-    boolean | string
-  >;
+  // TODO: use Args also for program
+  type Args = Omit<InstallAppArgs, "appPath" | "packageManager">;
 
-  const defaults: typeof preferences = {
-    template: "simple",
+  const preferences = (conf.get("preferences") || {}) as Args;
+
+  const defaults: Args = {
+    template: "streaming",
     framework: "nextjs",
     engine: "simple",
     ui: "html",
     eslint: true,
     frontend: false,
     openAIKey: "",
+    model: "gpt-3.5-turbo",
   };
-  const getPrefOrDefault = (field: string) =>
+  const getPrefOrDefault = (field: keyof Args) =>
     preferences[field] ?? defaults[field];
 
   const handlers = {
@@ -179,28 +180,6 @@ async function run(): Promise<void> {
     },
   };
 
-  if (!program.template) {
-    if (ciInfo.isCI) {
-      program.template = getPrefOrDefault("template");
-    } else {
-      const { template } = await prompts(
-        {
-          type: "select",
-          name: "template",
-          message: "Which template would you like to use?",
-          choices: [
-            { title: "Chat without streaming", value: "simple" },
-            { title: "Chat with streaming", value: "streaming" },
-          ],
-          initial: 1,
-        },
-        handlers,
-      );
-      program.template = template;
-      preferences.template = template;
-    }
-  }
-
   if (!program.framework) {
     if (ciInfo.isCI) {
       program.framework = getPrefOrDefault("framework");
@@ -224,6 +203,31 @@ async function run(): Promise<void> {
     }
   }
 
+  if (program.framework === "nextjs") {
+    program.template = "streaming";
+  }
+  if (!program.template) {
+    if (ciInfo.isCI) {
+      program.template = getPrefOrDefault("template");
+    } else {
+      const { template } = await prompts(
+        {
+          type: "select",
+          name: "template",
+          message: "Which template would you like to use?",
+          choices: [
+            { title: "Chat without streaming", value: "simple" },
+            { title: "Chat with streaming", value: "streaming" },
+          ],
+          initial: 1,
+        },
+        handlers,
+      );
+      program.template = template;
+      preferences.template = template;
+    }
+  }
+
   if (program.framework === "express" || program.framework === "fastapi") {
     // if a backend-only framework is selected, ask whether we should create a frontend
     if (!program.frontend) {
@@ -277,6 +281,32 @@ async function run(): Promise<void> {
     }
   }
 
+  if (program.framework === "nextjs") {
+    if (!program.model) {
+      if (ciInfo.isCI) {
+        program.model = getPrefOrDefault("model");
+      } else {
+        const { model } = await prompts(
+          {
+            type: "select",
+            name: "model",
+            message: "Which model would you like to use?",
+            choices: [
+              { title: "gpt-3.5-turbo", value: "gpt-3.5-turbo" },
+              { title: "gpt-4", value: "gpt-4" },
+              { title: "gpt-4-1106-preview", value: "gpt-4-1106-preview" },
+              { title: "gpt-4-vision-preview", value: "gpt-4-vision-preview" },
+            ],
+            initial: 0,
+          },
+          handlers,
+        );
+        program.model = model;
+        preferences.model = model;
+      }
+    }
+  }
+
   if (program.framework === "express" || program.framework === "nextjs") {
     if (!program.engine) {
       if (ciInfo.isCI) {
@@ -350,6 +380,7 @@ async function run(): Promise<void> {
     eslint: program.eslint,
     frontend: program.frontend,
     openAIKey: program.openAIKey,
+    model: program.model,
   });
   conf.set("preferences", preferences);
 }

packages/create-llama/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "keywords": [
     "rag",
     "llamaindex",

packages/create-llama/templates/components/engines/context/generate.mjs

@@ -5,6 +5,8 @@ import {
   VectorStoreIndex,
 } from "llamaindex";
 
+import * as dotenv from "dotenv";
+
 import {
   CHUNK_OVERLAP,
   CHUNK_SIZE,
@@ -12,6 +14,9 @@ import {
   STORAGE_DIR,
 } from "./constants.mjs";
 
+// Load environment variables from local .env file
+dotenv.config();
+
 async function getRuntime(func) {
   const start = Date.now();
   await func();

packages/create-llama/templates/components/ui/shadcn/chat/chat-input.tsx

@@ -1,29 +1,84 @@
+import { useState } from "react";
 import { Button } from "../button";
+import FileUploader from "../file-uploader";
 import { Input } from "../input";
+import UploadImagePreview from "../upload-image-preview";
 import { ChatHandler } from "./chat.interface";
 
 export default function ChatInput(
   props: Pick<
     ChatHandler,
-    "isLoading" | "handleSubmit" | "handleInputChange" | "input"
-  >,
+    | "isLoading"
+    | "input"
+    | "onFileUpload"
+    | "onFileError"
+    | "handleSubmit"
+    | "handleInputChange"
+  > & {
+    multiModal?: boolean;
+  },
 ) {
+  const [imageUrl, setImageUrl] = useState<string | null>(null);
+
+  const onSubmit = (e: React.FormEvent<HTMLFormElement>) => {
+    if (imageUrl) {
+      props.handleSubmit(e, {
+        data: { imageUrl: imageUrl },
+      });
+      setImageUrl(null);
+      return;
+    }
+    props.handleSubmit(e);
+  };
+
+  const onRemovePreviewImage = () => setImageUrl(null);
+
+  const handleUploadImageFile = async (file: File) => {
+    const base64 = await new Promise<string>((resolve, reject) => {
+      const reader = new FileReader();
+      reader.readAsDataURL(file);
+      reader.onload = () => resolve(reader.result as string);
+      reader.onerror = (error) => reject(error);
+    });
+    setImageUrl(base64);
+  };
+
+  const handleUploadFile = async (file: File) => {
+    try {
+      if (props.multiModal && file.type.startsWith("image/")) {
+        return await handleUploadImageFile(file);
+      }
+      props.onFileUpload?.(file);
+    } catch (error: any) {
+      props.onFileError?.(error.message);
+    }
+  };
+
   return (
     <form
-      onSubmit={props.handleSubmit}
-      className="flex w-full items-start justify-between gap-4 rounded-xl bg-white p-4 shadow-xl"
+      onSubmit={onSubmit}
+      className="rounded-xl bg-white p-4 shadow-xl space-y-4"
     >
-      <Input
-        autoFocus
-        name="message"
-        placeholder="Type a message"
-        className="flex-1"
-        value={props.input}
-        onChange={props.handleInputChange}
-      />
-      <Button type="submit" disabled={props.isLoading}>
-        Send message
-      </Button>
+      {imageUrl && (
+        <UploadImagePreview url={imageUrl} onRemove={onRemovePreviewImage} />
+      )}
+      <div className="flex w-full items-start justify-between gap-4 ">
+        <Input
+          autoFocus
+          name="message"
+          placeholder="Type a message"
+          className="flex-1"
+          value={props.input}
+          onChange={props.handleInputChange}
+        />
+        <FileUploader
+          onFileUpload={handleUploadFile}
+          onFileError={props.onFileError}
+        />
+        <Button type="submit" disabled={props.isLoading}>
+          Send message
+        </Button>
+      </div>
     </form>
   );
 }

packages/create-llama/templates/components/ui/shadcn/chat/chat-messages.tsx

@@ -1,4 +1,5 @@
 import { useEffect, useRef } from "react";
+import { Loader2 } from "lucide-react";
 
 import ChatActions from "./chat-actions";
 import ChatMessage from "./chat-message";
@@ -24,6 +25,11 @@ export default function ChatMessages(
     props.reload && !props.isLoading && isLastMessageFromAssistant;
   const showStop = props.stop && props.isLoading;
 
+  // `isPending` indicate
+  // that stream response is not yet received from the server,
+  // so we show a loading indicator to give a better UX.
+  const isPending = props.isLoading && !isLastMessageFromAssistant;
+
   useEffect(() => {
     scrollToBottom();
   }, [messageLength, lastMessage]);
@@ -37,6 +43,13 @@ export default function ChatMessages(
         {props.messages.map((m) => (
           <ChatMessage key={m.id} {...m} />
         ))}
+        {isPending && (
+          <div
+            className='flex justify-center items-center pt-10'
+          >
+            <Loader2 className="h-4 w-4 animate-spin"/>
+          </div>
+        )}
       </div>
       <div className="flex justify-end py-4">
         <ChatActions

packages/create-llama/templates/components/ui/shadcn/chat/chat.interface.ts

@@ -8,8 +8,15 @@ export interface ChatHandler {
   messages: Message[];
   input: string;
   isLoading: boolean;
-  handleSubmit: (e: React.FormEvent<HTMLFormElement>) => void;
+  handleSubmit: (
+    e: React.FormEvent<HTMLFormElement>,
+    ops?: {
+      data?: any;
+    },
+  ) => void;
   handleInputChange: (e: React.ChangeEvent<HTMLInputElement>) => void;
   reload?: () => void;
   stop?: () => void;
+  onFileUpload?: (file: File) => Promise<void>;
+  onFileError?: (errMsg: string) => void;
 }

packages/create-llama/templates/components/ui/shadcn/file-uploader.tsx

@@ -0,0 +1,105 @@
+"use client";
+
+import { Loader2, Paperclip } from "lucide-react";
+import { ChangeEvent, useState } from "react";
+import { buttonVariants } from "./button";
+import { cn } from "./lib/utils";
+
+export interface FileUploaderProps {
+  config?: {
+    inputId?: string;
+    fileSizeLimit?: number;
+    allowedExtensions?: string[];
+    checkExtension?: (extension: string) => string | null;
+    disabled: boolean;
+  };
+  onFileUpload: (file: File) => Promise<void>;
+  onFileError?: (errMsg: string) => void;
+}
+
+const DEFAULT_INPUT_ID = "fileInput";
+const DEFAULT_FILE_SIZE_LIMIT = 1024 * 1024 * 50; // 50 MB
+
+export default function FileUploader({
+  config,
+  onFileUpload,
+  onFileError,
+}: FileUploaderProps) {
+  const [uploading, setUploading] = useState(false);
+
+  const inputId = config?.inputId || DEFAULT_INPUT_ID;
+  const fileSizeLimit = config?.fileSizeLimit || DEFAULT_FILE_SIZE_LIMIT;
+  const allowedExtensions = config?.allowedExtensions;
+  const defaultCheckExtension = (extension: string) => {
+    if (allowedExtensions && !allowedExtensions.includes(extension)) {
+      return `Invalid file type. Please select a file with one of these formats: ${allowedExtensions!.join(
+        ",",
+      )}`;
+    }
+    return null;
+  };
+  const checkExtension = config?.checkExtension ?? defaultCheckExtension;
+
+  const isFileSizeExceeded = (file: File) => {
+    return file.size > fileSizeLimit;
+  };
+
+  const resetInput = () => {
+    const fileInput = document.getElementById(inputId) as HTMLInputElement;
+    fileInput.value = "";
+  };
+
+  const onFileChange = async (e: ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+    if (!file) return;
+
+    setUploading(true);
+    await handleUpload(file);
+    resetInput();
+    setUploading(false);
+  };
+
+  const handleUpload = async (file: File) => {
+    const onFileUploadError = onFileError || window.alert;
+    const fileExtension = file.name.split(".").pop() || "";
+    const extensionFileError = checkExtension(fileExtension);
+    if (extensionFileError) {
+      return onFileUploadError(extensionFileError);
+    }
+
+    if (isFileSizeExceeded(file)) {
+      return onFileUploadError(
+        `File size exceeded. Limit is ${fileSizeLimit / 1024 / 1024} MB`,
+      );
+    }
+
+    await onFileUpload(file);
+  };
+
+  return (
+    <div className="self-stretch">
+      <input
+        type="file"
+        id={inputId}
+        style={{ display: "none" }}
+        onChange={onFileChange}
+        accept={allowedExtensions?.join(",")}
+        disabled={config?.disabled || uploading}
+      />
+      <label
+        htmlFor={inputId}
+        className={cn(
+          buttonVariants({ variant: "secondary", size: "icon" }),
+          "cursor-pointer",
+          uploading && "opacity-50",
+        )}
+      >
+        {uploading ? (
+          <Loader2 className="h-4 w-4 animate-spin" />
+        ) : (
+          <Paperclip className="-rotate-45 w-4 h-4" />
+        )}
+      </label>
+    </div>
+  );
+}

packages/create-llama/templates/components/ui/shadcn/upload-image-preview.tsx

@@ -0,0 +1,32 @@
+import { XCircleIcon } from "lucide-react";
+import Image from "next/image";
+import { cn } from "./lib/utils";
+
+export default function UploadImagePreview({
+  url,
+  onRemove,
+}: {
+  url: string;
+  onRemove: () => void;
+}) {
+  return (
+    <div className="relative w-20 h-20 group">
+      <Image
+        src={url}
+        alt="Uploaded image"
+        fill
+        className="object-cover w-full h-full rounded-xl hover:brightness-75"
+      />
+      <div
+        className={cn(
+          "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full hidden group-hover:block",
+        )}
+      >
+        <XCircleIcon
+          className="w-6 h-6 bg-gray-500 text-white rounded-full"
+          onClick={onRemove}
+        />
+      </div>
+    </div>
+  );
+}

packages/create-llama/templates/index.ts

@@ -14,26 +14,14 @@ import {
   TemplateFramework,
 } from "./types";
 
-const envFileNameMap: Record<TemplateFramework, string> = {
-  nextjs: ".env.local",
-  express: ".env",
-  fastapi: ".env",
-};
-
-const createEnvLocalFile = async (
-  root: string,
-  framework: TemplateFramework,
-  openAIKey?: string,
-) => {
+const createEnvLocalFile = async (root: string, openAIKey?: string) => {
   if (openAIKey) {
-    const envFileName = envFileNameMap[framework];
-    if (!envFileName) return;
+    const envFileName = ".env";
     await fs.writeFile(
       path.join(root, envFileName),
       `OPENAI_API_KEY=${openAIKey}\n`,
     );
     console.log(`Created '${envFileName}' file containing OPENAI_API_KEY`);
-    process.env["OPENAI_API_KEY"] = openAIKey;
   }
 };
 
@@ -42,7 +30,16 @@ const copyTestData = async (
   framework: TemplateFramework,
   packageManager?: PackageManager,
   engine?: TemplateEngine,
+  openAIKey?: string,
 ) => {
+  if (framework === "nextjs") {
+    // XXX: This is a hack to make the build for nextjs work with pdf-parse
+    // pdf-parse needs './test/data/05-versions-space.pdf' to exist - can be removed when pdf-parse is removed
+    const srcFile = path.join(__dirname, "components", "data", "101.pdf");
+    const destPath = path.join(root, "test", "data");
+    await fs.mkdir(destPath, { recursive: true });
+    await fs.copyFile(srcFile, path.join(destPath, "05-versions-space.pdf"));
+  }
   if (engine === "context" || framework === "fastapi") {
     const srcPath = path.join(__dirname, "components", "data");
     const destPath = path.join(root, "data");
@@ -54,7 +51,7 @@ const copyTestData = async (
   }
 
   if (packageManager && engine === "context") {
-    if (process.env["OPENAI_API_KEY"]) {
+    if (openAIKey || process.env["OPENAI_API_KEY"]) {
       console.log(
         `\nRunning ${cyan(
           `${packageManager} run generate`,
@@ -104,6 +101,7 @@ const installTSTemplate = async ({
   eslint,
   customApiPath,
   forBackend,
+  model,
 }: InstallTemplateArgs) => {
   console.log(bold(`Using ${packageManager}.`));
 
@@ -176,6 +174,14 @@ const installTSTemplate = async ({
     });
   }
 
+  if (framework === "nextjs") {
+    await fs.writeFile(
+      path.join(root, "constants.ts"),
+      `export const MODEL = "${model || "gpt-3.5-turbo"}";\n`,
+    );
+    console.log("\nUsing OpenAI model: ", model || "gpt-3.5-turbo", "\n");
+  }
+
   /**
    * Update the package.json scripts.
    */
@@ -226,6 +232,7 @@ const installTSTemplate = async ({
       "tailwind-merge": "^2",
       "@radix-ui/react-slot": "^1",
       "class-variance-authority": "^0.7",
+      clsx: "^1.2.1",
       "lucide-react": "^0.291",
       remark: "^14.0.3",
       "remark-code-import": "^1.2.0",
@@ -313,7 +320,7 @@ export const installTemplate = async (
     // This is a backend, so we need to copy the test data and create the env file.
 
     // Copy the environment file to the target directory.
-    await createEnvLocalFile(props.root, props.framework, props.openAIKey);
+    await createEnvLocalFile(props.root, props.openAIKey);
 
     // Copy test pdf file
     await copyTestData(
@@ -321,6 +328,7 @@ export const installTemplate = async (
       props.framework,
       props.packageManager,
       props.engine,
+      props.openAIKey,
     );
   }
 };

packages/create-llama/templates/types.ts

@@ -18,4 +18,5 @@ export interface InstallTemplateArgs {
   customApiPath?: string;
   openAIKey?: string;
   forBackend?: string;
+  model: string;
 }

packages/create-llama/templates/types/simple/fastapi/README-template.md

@@ -9,6 +9,14 @@ poetry install
 poetry shell
 ```
 
+By default, we use the OpenAI LLM (though you can customize, see app/api/routers/chat.py). As a result you need to specify an `OPENAI_API_KEY` in an .env file in this directory.
+
+Example `backend/.env` file:
+
+```
+OPENAI_API_KEY=<openai_api_key>
+```
+
 Second, run the development server:
 
 ```

packages/create-llama/templates/types/simple/fastapi/gitignore

@@ -1,2 +1,3 @@
 __pycache__
 storage
+.env

packages/create-llama/templates/types/simple/nextjs/.env.example

@@ -1,3 +0,0 @@
-# Rename this file to `.env.local` to use environment variables locally with `next dev`
-# https://nextjs.org/docs/pages/building-your-application/configuring/environment-variables
-MY_HOST="example.com"

packages/create-llama/templates/types/simple/nextjs/README-template.md

@@ -1,30 +0,0 @@
-This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Next.js](https://nextjs.org/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
-
-## Getting Started
-
-First, install the dependencies:
-
-```
-npm install
-```
-
-Second, run the development server:
-
-```
-npm run dev
-```
-
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
-
-You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
-
-This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
-
-## Learn More
-
-To learn more about LlamaIndex, take a look at the following resources:
-
-- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
-- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
-
-You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

packages/create-llama/templates/types/simple/nextjs/app/api/chat/engine/index.ts

@@ -1,7 +0,0 @@
-import { LLM, SimpleChatEngine } from "llamaindex";
-
-export async function createChatEngine(llm: LLM) {
-  return new SimpleChatEngine({
-    llm,
-  });
-}

packages/create-llama/templates/types/simple/nextjs/app/api/chat/route.ts

@@ -1,47 +0,0 @@
-import { ChatMessage, OpenAI } from "llamaindex";
-import { NextRequest, NextResponse } from "next/server";
-import { createChatEngine } from "./engine";
-
-export const runtime = "nodejs";
-export const dynamic = "force-dynamic";
-
-export async function POST(request: NextRequest) {
-  try {
-    const body = await request.json();
-    const { messages }: { messages: ChatMessage[] } = body;
-    const lastMessage = messages.pop();
-    if (!messages || !lastMessage || lastMessage.role !== "user") {
-      return NextResponse.json(
-        {
-          error:
-            "messages are required in the request body and the last message must be from the user",
-        },
-        { status: 400 },
-      );
-    }
-
-    const llm = new OpenAI({
-      model: "gpt-3.5-turbo",
-    });
-
-    const chatEngine = await createChatEngine(llm);
-
-    const response = await chatEngine.chat(lastMessage.content, messages);
-    const result: ChatMessage = {
-      role: "assistant",
-      content: response.response,
-    };
-
-    return NextResponse.json({ result });
-  } catch (error) {
-    console.error("[LlamaIndex]", error);
-    return NextResponse.json(
-      {
-        error: (error as Error).message,
-      },
-      {
-        status: 500,
-      },
-    );
-  }
-}

packages/create-llama/templates/types/simple/nextjs/app/components/chat-section.tsx

@@ -1,80 +0,0 @@
-"use client";
-
-import { nanoid } from "nanoid";
-import { useState } from "react";
-import { ChatInput, ChatInputProps, ChatMessages, Message } from "./ui/chat";
-
-function useChat(): ChatInputProps & { messages: Message[] } {
-  const [messages, setMessages] = useState<Message[]>([]);
-  const [isLoading, setIsLoading] = useState(false);
-  const [input, setInput] = useState("");
-
-  const getAssistantMessage = async (messages: Message[]) => {
-    const response = await fetch(
-      process.env.NEXT_PUBLIC_CHAT_API ?? "/api/chat",
-      {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify({
-          messages,
-        }),
-      },
-    );
-    const data = await response.json();
-    const assistantMessage = data.result as Message;
-    return assistantMessage;
-  };
-
-  const handleSubmit = async (e: React.FormEvent<HTMLFormElement>) => {
-    e.preventDefault();
-    if (!input) return;
-
-    setIsLoading(true);
-
-    try {
-      const newMessages = [
-        ...messages,
-        { id: nanoid(), content: input, role: "user" },
-      ];
-      setMessages(newMessages);
-      setInput("");
-      const assistantMessage = await getAssistantMessage(newMessages);
-      setMessages([...newMessages, { ...assistantMessage, id: nanoid() }]);
-    } catch (error: any) {
-      console.log(error);
-      alert(error.message);
-    } finally {
-      setIsLoading(false);
-    }
-  };
-
-  const handleInputChange = (e: any): void => {
-    setInput(e.target.value);
-  };
-
-  return {
-    messages,
-    isLoading,
-    input,
-    handleSubmit,
-    handleInputChange,
-  };
-}
-
-export default function ChatSection() {
-  const { messages, isLoading, input, handleSubmit, handleInputChange } =
-    useChat();
-  return (
-    <div className="space-y-4 max-w-5xl w-full">
-      <ChatMessages messages={messages} />
-      <ChatInput
-        handleSubmit={handleSubmit}
-        isLoading={isLoading}
-        input={input}
-        handleInputChange={handleInputChange}
-      />
-    </div>
-  );
-}

packages/create-llama/templates/types/simple/nextjs/app/components/header.tsx

@@ -1,28 +0,0 @@
-import Image from "next/image";
-
-export default function Header() {
-  return (
-    <div className="z-10 max-w-5xl w-full items-center justify-between font-mono text-sm lg:flex">
-      <p className="fixed left-0 top-0 flex w-full justify-center border-b border-gray-300 bg-gradient-to-b from-zinc-200 pb-6 pt-8 backdrop-blur-2xl dark:border-neutral-800 dark:bg-zinc-800/30 dark:from-inherit lg:static lg:w-auto  lg:rounded-xl lg:border lg:bg-gray-200 lg:p-4 lg:dark:bg-zinc-800/30">
-        Get started by editing&nbsp;
-        <code className="font-mono font-bold">app/page.tsx</code>
-      </p>
-      <div className="fixed bottom-0 left-0 flex h-48 w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:h-auto lg:w-auto lg:bg-none">
-        <a
-          href="https://www.llamaindex.ai/"
-          className="flex items-center justify-center font-nunito text-lg font-bold gap-2"
-        >
-          <span>Built by LlamaIndex</span>
-          <Image
-            className="rounded-xl"
-            src="/llama.png"
-            alt="Llama Logo"
-            width={40}
-            height={40}
-            priority
-          />
-        </a>
-      </div>
-    </div>
-  );
-}

packages/create-llama/templates/types/simple/nextjs/app/components/ui/chat/chat-avatar.tsx

@@ -1,34 +0,0 @@
-"use client";
-
-import Image from "next/image";
-import { Message } from "./chat-messages";
-
-export default function ChatAvatar(message: Message) {
-  if (message.role === "user") {
-    return (
-      <div className="flex h-8 w-8 shrink-0 select-none items-center justify-center rounded-md border shadow bg-background">
-        <svg
-          xmlns="http://www.w3.org/2000/svg"
-          viewBox="0 0 256 256"
-          fill="currentColor"
-          className="h-4 w-4"
-        >
-          <path d="M230.92 212c-15.23-26.33-38.7-45.21-66.09-54.16a72 72 0 1 0-73.66 0c-27.39 8.94-50.86 27.82-66.09 54.16a8 8 0 1 0 13.85 8c18.84-32.56 52.14-52 89.07-52s70.23 19.44 89.07 52a8 8 0 1 0 13.85-8ZM72 96a56 56 0 1 1 56 56 56.06 56.06 0 0 1-56-56Z"></path>
-        </svg>
-      </div>
-    );
-  }
-
-  return (
-    <div className="flex h-8 w-8 shrink-0 select-none items-center justify-center rounded-md border  bg-black text-white">
-      <Image
-        className="rounded-md"
-        src="/llama.png"
-        alt="Llama Logo"
-        width={24}
-        height={24}
-        priority
-      />
-    </div>
-  );
-}

packages/create-llama/templates/types/simple/nextjs/app/components/ui/chat/chat-input.tsx

@@ -1,42 +0,0 @@
-"use client";
-
-export interface ChatInputProps {
-  /** The current value of the input */
-  input?: string;
-  /** An input/textarea-ready onChange handler to control the value of the input */
-  handleInputChange?: (
-    e:
-      | React.ChangeEvent<HTMLInputElement>
-      | React.ChangeEvent<HTMLTextAreaElement>,
-  ) => void;
-  /** Form submission handler to automatically reset input and append a user message  */
-  handleSubmit: (e: React.FormEvent<HTMLFormElement>) => void;
-  isLoading: boolean;
-}
-
-export default function ChatInput(props: ChatInputProps) {
-  return (
-    <>
-      <form
-        onSubmit={props.handleSubmit}
-        className="flex items-start justify-between w-full max-w-5xl p-4 bg-white rounded-xl shadow-xl gap-4"
-      >
-        <input
-          autoFocus
-          name="message"
-          placeholder="Type a message"
-          className="w-full p-4 rounded-xl shadow-inner flex-1"
-          value={props.input}
-          onChange={props.handleInputChange}
-        />
-        <button
-          disabled={props.isLoading}
-          type="submit"
-          className="p-4 text-white rounded-xl shadow-xl bg-gradient-to-r from-cyan-500 to-sky-500 disabled:opacity-50 disabled:cursor-not-allowed"
-        >
-          Send message
-        </button>
-      </form>
-    </>
-  );
-}

packages/create-llama/templates/types/simple/nextjs/app/components/ui/chat/chat-item.tsx

@@ -1,13 +0,0 @@
-"use client";
-
-import ChatAvatar from "./chat-avatar";
-import { Message } from "./chat-messages";
-
-export default function ChatItem(message: Message) {
-  return (
-    <div className="flex items-start gap-4 pt-5">
-      <ChatAvatar {...message} />
-      <p className="break-words">{message.content}</p>
-    </div>
-  );
-}

packages/create-llama/templates/types/simple/nextjs/app/components/ui/chat/chat-messages.tsx

@@ -1,38 +0,0 @@
-"use client";
-
-import { useEffect, useRef } from "react";
-import ChatItem from "./chat-item";
-
-export interface Message {
-  id: string;
-  content: string;
-  role: string;
-}
-
-export default function ChatMessages({ messages }: { messages: Message[] }) {
-  const scrollableChatContainerRef = useRef<HTMLDivElement>(null);
-
-  const scrollToBottom = () => {
-    if (scrollableChatContainerRef.current) {
-      scrollableChatContainerRef.current.scrollTop =
-        scrollableChatContainerRef.current.scrollHeight;
-    }
-  };
-
-  useEffect(() => {
-    scrollToBottom();
-  }, [messages.length]);
-
-  return (
-    <div className="w-full max-w-5xl p-4 bg-white rounded-xl shadow-xl">
-      <div
-        className="flex flex-col gap-5 divide-y h-[50vh] overflow-auto"
-        ref={scrollableChatContainerRef}
-      >
-        {messages.map((m: Message) => (
-          <ChatItem key={m.id} {...m} />
-        ))}
-      </div>
-    </div>
-  );
-}

packages/create-llama/templates/types/simple/nextjs/app/components/ui/chat/index.ts

@@ -1,6 +0,0 @@
-import ChatInput from "./chat-input";
-import ChatMessages from "./chat-messages";
-
-export type { ChatInputProps } from "./chat-input";
-export type { Message } from "./chat-messages";
-export { ChatInput, ChatMessages };

packages/create-llama/templates/types/simple/nextjs/app/globals.css

@@ -1,94 +0,0 @@
-@tailwind base;
-@tailwind components;
-@tailwind utilities;
-
-@layer base {
-  :root {
-    --background: 0 0% 100%;
-    --foreground: 222.2 47.4% 11.2%;
-
-    --muted: 210 40% 96.1%;
-    --muted-foreground: 215.4 16.3% 46.9%;
-
-    --popover: 0 0% 100%;
-    --popover-foreground: 222.2 47.4% 11.2%;
-
-    --border: 214.3 31.8% 91.4%;
-    --input: 214.3 31.8% 91.4%;
-
-    --card: 0 0% 100%;
-    --card-foreground: 222.2 47.4% 11.2%;
-
-    --primary: 222.2 47.4% 11.2%;
-    --primary-foreground: 210 40% 98%;
-
-    --secondary: 210 40% 96.1%;
-    --secondary-foreground: 222.2 47.4% 11.2%;
-
-    --accent: 210 40% 96.1%;
-    --accent-foreground: 222.2 47.4% 11.2%;
-
-    --destructive: 0 100% 50%;
-    --destructive-foreground: 210 40% 98%;
-
-    --ring: 215 20.2% 65.1%;
-
-    --radius: 0.5rem;
-  }
-
-  .dark {
-    --background: 224 71% 4%;
-    --foreground: 213 31% 91%;
-
-    --muted: 223 47% 11%;
-    --muted-foreground: 215.4 16.3% 56.9%;
-
-    --accent: 216 34% 17%;
-    --accent-foreground: 210 40% 98%;
-
-    --popover: 224 71% 4%;
-    --popover-foreground: 215 20.2% 65.1%;
-
-    --border: 216 34% 17%;
-    --input: 216 34% 17%;
-
-    --card: 224 71% 4%;
-    --card-foreground: 213 31% 91%;
-
-    --primary: 210 40% 98%;
-    --primary-foreground: 222.2 47.4% 1.2%;
-
-    --secondary: 222.2 47.4% 11.2%;
-    --secondary-foreground: 210 40% 98%;
-
-    --destructive: 0 63% 31%;
-    --destructive-foreground: 210 40% 98%;
-
-    --ring: 216 34% 17%;
-
-    --radius: 0.5rem;
-  }
-}
-
-@layer base {
-  * {
-    @apply border-border;
-  }
-  body {
-    @apply bg-background text-foreground;
-    font-feature-settings:
-      "rlig" 1,
-      "calt" 1;
-  }
-  .background-gradient {
-    background-color: #fff;
-    background-image: radial-gradient(
-        at 21% 11%,
-        rgba(186, 186, 233, 0.53) 0,
-        transparent 50%
-      ),
-      radial-gradient(at 85% 0, hsla(46, 57%, 78%, 0.52) 0, transparent 50%),
-      radial-gradient(at 91% 36%, rgba(194, 213, 255, 0.68) 0, transparent 50%),
-      radial-gradient(at 8% 40%, rgba(251, 218, 239, 0.46) 0, transparent 50%);
-  }
-}

packages/create-llama/templates/types/simple/nextjs/app/layout.tsx

@@ -1,22 +0,0 @@
-import type { Metadata } from "next";
-import { Inter } from "next/font/google";
-import "./globals.css";
-
-const inter = Inter({ subsets: ["latin"] });
-
-export const metadata: Metadata = {
-  title: "Create Llama App",
-  description: "Generated by create-llama",
-};
-
-export default function RootLayout({
-  children,
-}: {
-  children: React.ReactNode;
-}) {
-  return (
-    <html lang="en">
-      <body className={inter.className}>{children}</body>
-    </html>
-  );
-}

packages/create-llama/templates/types/simple/nextjs/app/page.tsx

@@ -1,11 +0,0 @@
-import ChatSection from "@/app/components/chat-section";
-import Header from "@/app/components/header";
-
-export default function Home() {
-  return (
-    <main className="flex min-h-screen flex-col items-center gap-10 p-24 background-gradient">
-      <Header />
-      <ChatSection />
-    </main>
-  );
-}

packages/create-llama/templates/types/simple/nextjs/eslintrc.json

@@ -1,3 +0,0 @@
-{
-  "extends": "next/core-web-vitals"
-}

packages/create-llama/templates/types/simple/nextjs/gitignore

@@ -1,35 +0,0 @@
-# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
-
-# dependencies
-/node_modules
-/.pnp
-.pnp.js
-
-# testing
-/coverage
-
-# next.js
-/.next/
-/out/
-
-# production
-/build
-
-# misc
-.DS_Store
-*.pem
-
-# debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# local env files
-.env*.local
-
-# vercel
-.vercel
-
-# typescript
-*.tsbuildinfo
-next-env.d.ts

packages/create-llama/templates/types/simple/nextjs/next-env.d.ts

@@ -1,5 +0,0 @@
-/// <reference types="next" />
-/// <reference types="next/image-types/global" />
-
-// NOTE: This file should not be edited
-// see https://nextjs.org/docs/basic-features/typescript for more information.

packages/create-llama/templates/types/simple/nextjs/next.config.app.js

@@ -1,11 +0,0 @@
-/** @type {import('next').NextConfig} */
-const nextConfig = {
-  experimental: {
-    serverComponentsExternalPackages: ["llamaindex"],
-    outputFileTracingIncludes: {
-      "/*": ["./cache/**/*"],
-    },
-  },
-};
-
-module.exports = nextConfig;

packages/create-llama/templates/types/simple/nextjs/next.config.static.js

@@ -1,13 +0,0 @@
-/** @type {import('next').NextConfig} */
-const nextConfig = {
-  output: "export",
-  images: { unoptimized: true },
-  experimental: {
-    serverComponentsExternalPackages: ["llamaindex"],
-    outputFileTracingIncludes: {
-      "/*": ["./cache/**/*"],
-    },
-  },
-};
-
-module.exports = nextConfig;

packages/create-llama/templates/types/simple/nextjs/package.json

@@ -1,28 +0,0 @@
-{
-  "name": "llama-index-nextjs",
-  "version": "1.0.0",
-  "scripts": {
-    "dev": "next dev",
-    "build": "next build",
-    "start": "next start",
-    "lint": "next lint"
-  },
-  "dependencies": {
-    "llamaindex": "0.0.31",
-    "nanoid": "^5",
-    "next": "^13",
-    "react": "^18",
-    "react-dom": "^18"
-  },
-  "devDependencies": {
-    "@types/node": "^20",
-    "@types/react": "^18",
-    "@types/react-dom": "^18",
-    "autoprefixer": "^10",
-    "eslint": "^8",
-    "eslint-config-next": "^13",
-    "postcss": "^8",
-    "tailwindcss": "^3",
-    "typescript": "^5"
-  }
-}

packages/create-llama/templates/types/simple/nextjs/postcss.config.js

@@ -1,6 +0,0 @@
-module.exports = {
-  plugins: {
-    tailwindcss: {},
-    autoprefixer: {},
-  },
-};