llamaindex 0.0.37

Several fixes for improving compatibility with Next.JS

.github/ISSUE_TEMPLATE/bugfix.yml

@@ -1,7 +1,6 @@
 name: Bugfix
-title: "Sweep: "
+title: ""
 description: Write something like "We notice ... behavior when ... happens instead of ...""
-labels: sweep
 body:
   - type: textarea
     id: description

.github/ISSUE_TEMPLATE/feature.yml

@@ -1,11 +1,10 @@
 name: Feature Request
-title: "Sweep: "
-description: Write something like "Write an api endpoint that does "..." in the "..." file"
-labels: sweep
+title: ""
+description: Write something like "Write an api endpoint that does "..." in the "..." file". If you would like to use sweep.dev prefix with "Sweep:"
 body:
   - type: textarea
     id: description
     attributes:
       label: Details
-      description: More details for Sweep
+      description: More details
       placeholder: The new endpoint should use the ... class from ... file because it contains ... logic

.github/ISSUE_TEMPLATE/refactor.yml

@@ -1,11 +1,10 @@
 name: Refactor
-title: "Sweep: "
-description: Write something like "Modify the ... api endpoint to use ... version and ... framework"
-labels: sweep
+title: ""
+description: Write something like "Modify the ... api endpoint to use ... version and ... framework" If you would like to use sweep.dev prefix with "Sweep:"
 body:
   - type: textarea
     id: description
     attributes:
       label: Details
-      description: More details for Sweep
+      description: More details
       placeholder: We are migrating this function to ... version because ...

.vscode/settings.json

@@ -4,5 +4,6 @@
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "[xml]": {
     "editor.defaultFormatter": "redhat.vscode-xml"
-  }
-}
+  },
+  "jest.rootPath": "./packages/core"
+}

apps/mongodb/2_load_and_index.ts

@@ -3,8 +3,8 @@ import * as dotenv from "dotenv";
 import {
   MongoDBAtlasVectorSearch,
   SimpleMongoReader,
-  VectorStoreIndex,
   storageContextFromDefaults,
+  VectorStoreIndex,
 } from "llamaindex";
 import { MongoClient } from "mongodb";
 

apps/mongodb/3_query.ts

@@ -2,8 +2,8 @@
 import * as dotenv from "dotenv";
 import {
   MongoDBAtlasVectorSearch,
-  VectorStoreIndex,
   serviceContextFromDefaults,
+  VectorStoreIndex,
 } from "llamaindex";
 import { MongoClient } from "mongodb";
 

apps/mongodb/CHANGELOG.md

@@ -0,0 +1,20 @@
+# mongodb-llamaindexts
+
+## 0.0.3
+
+### Patch Changes
+
+- Updated dependencies [3bab231]
+  - llamaindex@0.0.37
+
+## 0.0.2
+
+### Patch Changes
+
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+  - llamaindex@0.0.36

apps/mongodb/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.1",
+  "version": "0.0.3",
   "private": true,
   "name": "mongodb-llamaindexts",
   "dependencies": {

apps/simple/CHANGELOG.md

@@ -1,5 +1,24 @@
 # simple
 
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [3bab231]
+  - llamaindex@0.0.37
+
+## 0.0.34
+
+### Patch Changes
+
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+- Updated dependencies
+  - llamaindex@0.0.36
+
 ## 0.0.33
 
 ### Patch Changes

apps/simple/anthropic.ts

@@ -4,8 +4,6 @@ import { Anthropic } from "llamaindex";
   const anthropic = new Anthropic();
   const result = await anthropic.chat([
     { content: "You want to talk in rhymes.", role: "system" },
-    { content: "Hello, world!", role: "user" },
-    { content: "Hello!", role: "assistant" },
     {
       content:
         "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",

apps/simple/llmStream.ts

@@ -1,47 +0,0 @@
-import { ChatMessage, SimpleChatEngine } from "llamaindex";
-import { stdin as input, stdout as output } from "node:process";
-import readline from "node:readline/promises";
-import { Anthropic } from "../../packages/core/src/llm/LLM";
-
-async function main() {
-  const query: string = `
-Where is Istanbul?
-  `;
-
-  // const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 });
-  const llm = new Anthropic();
-  const message: ChatMessage = { content: query, role: "user" };
-
-  //TODO: Add callbacks later
-
-  //Stream Complete
-  //Note: Setting streaming flag to true or false will auto-set your return type to
-  //either an AsyncGenerator or a Response.
-  // Omitting the streaming flag automatically sets streaming to false
-
-  const chatEngine: SimpleChatEngine = new SimpleChatEngine({
-    chatHistory: undefined,
-    llm: llm,
-  });
-
-  const rl = readline.createInterface({ input, output });
-  while (true) {
-    const query = await rl.question("Query: ");
-
-    if (!query) {
-      break;
-    }
-
-    //Case 1: .chat(query, undefined, true) => Stream
-    //Case 2: .chat(query, undefined, false) => Response object
-    //Case 3: .chat(query, undefined) => Response object
-    const chatStream = await chatEngine.chat(query, undefined, true);
-    var accumulated_result = "";
-    for await (const part of chatStream) {
-      accumulated_result += part;
-      process.stdout.write(part);
-    }
-  }
-}
-
-main();

apps/simple/mongo.ts

@@ -1,6 +1,6 @@
 import { MongoClient } from "mongodb";
-import { Document } from "../../packages/core/src/Node";
 import { VectorStoreIndex } from "../../packages/core/src/indices";
+import { Document } from "../../packages/core/src/Node";
 import { SimpleMongoReader } from "../../packages/core/src/readers/SimpleMongoReader";
 
 import { stdin as input, stdout as output } from "node:process";

apps/simple/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.33",
+  "version": "0.0.35",
   "private": true,
   "name": "simple",
   "dependencies": {

apps/simple/pg-vector-store/README.md

@@ -0,0 +1,33 @@
+# Postgres Vector Store
+
+There are two scripts available here: load-docs.ts and query.ts
+
+## Prerequisites
+
+You'll need a postgres database instance against which to run these scripts. A simple docker command would look like this:
+
+> `docker run -d --rm --name vector-db -p 5432:5432 -e "POSTGRES_HOST_AUTH_METHOD=trust" ankane/pgvector`
+
+Set the PGHOST and PGUSER (and PGPASSWORD) environment variables to match your database setup.
+
+You'll also need a value for OPENAI_API_KEY in your environment.
+
+**NOTE:** Using `--rm` in the example docker command above means that the vector store will be deleted every time the container is stopped. For production purposes, use a volume to ensure persistence across restarts.
+
+## Setup and Loading Docs
+
+Read and follow the instructions in the README.md file located one directory up to make sure your JS/TS dependencies are set up. The commands listed below are also run from that parent directory.
+
+To import documents and save the embedding vectors to your database:
+
+> `npx ts-node pg-vector-store/load-docs.ts data`
+
+where data is the directory containing your input files. Using the _data_ directory in the example above will read all of the files in that directory using the llamaindexTS default readers for each file type.
+
+## RAG Querying
+
+To query using the resulting vector store:
+
+> `npx ts-node pg-vector-store/query.ts`
+
+The script will prompt for a question, then process and present the answer using the PGVectorStore data and your OpenAI API key. It will continue to prompt until you enter `q`, `quit` or `exit` as the next query.

apps/simple/pg-vector-store/load-docs.ts

@@ -0,0 +1,68 @@
+// load-docs.ts
+import fs from "fs/promises";
+import {
+  SimpleDirectoryReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { PGVectorStore } from "../../../packages/core/src/storage/vectorStore/PGVectorStore";
+
+async function getSourceFilenames(sourceDir: string) {
+  return await fs
+    .readdir(sourceDir)
+    .then((fileNames) => fileNames.map((file) => sourceDir + "/" + file));
+}
+
+function callback(
+  category: string,
+  name: string,
+  status: any,
+  message: string = "",
+): boolean {
+  console.log(category, name, status, message);
+  return true;
+}
+
+async function main(args: any) {
+  const sourceDir: string = args.length > 2 ? args[2] : "../data";
+
+  console.log(`Finding documents in ${sourceDir}`);
+  const fileList = await getSourceFilenames(sourceDir);
+  const count = fileList.length;
+  console.log(`Found ${count} files`);
+
+  console.log(`Importing contents from ${count} files in ${sourceDir}`);
+  var fileName = "";
+  try {
+    // Passing callback fn to the ctor here
+    // will enable looging to console.
+    // See callback fn, defined above.
+    const rdr = new SimpleDirectoryReader(callback);
+    const docs = await rdr.loadData({ directoryPath: sourceDir });
+
+    const pgvs = new PGVectorStore();
+    pgvs.setCollection(sourceDir);
+    pgvs.clearCollection();
+
+    const ctx = await storageContextFromDefaults({ vectorStore: pgvs });
+
+    console.debug("  - creating vector store");
+    const index = await VectorStoreIndex.fromDocuments(docs, {
+      storageContext: ctx,
+    });
+    console.debug("  - done.");
+  } catch (err) {
+    console.error(fileName, err);
+    console.log(
+      "If your PGVectorStore init failed, make sure to set env vars for PGUSER or USER, PGHOST, PGPORT and PGPASSWORD as needed.",
+    );
+    process.exit(1);
+  }
+
+  console.log(
+    "Done. Try running query.ts to ask questions against the imported embeddings.",
+  );
+  process.exit(0);
+}
+
+main(process.argv).catch((err) => console.error(err));

apps/simple/pg-vector-store/query.ts

@@ -0,0 +1,67 @@
+import { VectorStoreIndex } from "../../../packages/core/src/indices/vectorStore/VectorStoreIndex";
+import { serviceContextFromDefaults } from "../../../packages/core/src/ServiceContext";
+import { PGVectorStore } from "../../../packages/core/src/storage/vectorStore/PGVectorStore";
+
+async function main() {
+  const readline = require("readline").createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  try {
+    const pgvs = new PGVectorStore();
+    // Optional - set your collection name, default is no filter on this field.
+    // pgvs.setCollection();
+
+    const ctx = serviceContextFromDefaults();
+    const index = await VectorStoreIndex.fromVectorStore(pgvs, ctx);
+
+    // Query the index
+    const queryEngine = await index.asQueryEngine();
+
+    let question = "";
+    while (!isQuit(question)) {
+      question = await getUserInput(readline);
+
+      if (isQuit(question)) {
+        readline.close();
+        process.exit(0);
+      }
+
+      try {
+        const answer = await queryEngine.query(question);
+        console.log(answer.response);
+      } catch (error) {
+        console.error("Error:", error);
+      }
+    }
+  } catch (err) {
+    console.error(err);
+    console.log(
+      "If your PGVectorStore init failed, make sure to set env vars for PGUSER or USER, PGHOST, PGPORT and PGPASSWORD as needed.",
+    );
+    process.exit(1);
+  }
+}
+
+function isQuit(question: string) {
+  return ["q", "quit", "exit"].includes(question.trim().toLowerCase());
+}
+
+// Function to get user input as a promise
+function getUserInput(readline: any): Promise<string> {
+  return new Promise((resolve) => {
+    readline.question(
+      "What would you like to know?\n>",
+      (userInput: string) => {
+        resolve(userInput);
+      },
+    );
+  });
+}
+
+main()
+  .catch(console.error)
+  .finally(() => {
+    process.exit(1);
+  });

apps/simple/vectorIndexAnthropic.ts

@@ -2,7 +2,10 @@ import fs from "node:fs/promises";
 
 import {
   Anthropic,
+  anthropicTextQaPrompt,
+  CompactAndRefine,
   Document,
+  ResponseSynthesizer,
   serviceContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
@@ -18,12 +21,20 @@ async function main() {
 
   // Split text and create embeddings. Store them in a VectorStoreIndex
   const serviceContext = serviceContextFromDefaults({ llm: new Anthropic() });
+
+  const responseSynthesizer = new ResponseSynthesizer({
+    responseBuilder: new CompactAndRefine(
+      serviceContext,
+      anthropicTextQaPrompt,
+    ),
+  });
+
   const index = await VectorStoreIndex.fromDocuments([document], {
     serviceContext,
   });
 
   // Query the index
-  const queryEngine = index.asQueryEngine();
+  const queryEngine = index.asQueryEngine({ responseSynthesizer });
   const response = await queryEngine.query(
     "What did the author do in college?",
   );

examples/anthropic.ts

@@ -4,8 +4,6 @@ import { Anthropic } from "llamaindex";
   const anthropic = new Anthropic();
   const result = await anthropic.chat([
     { content: "You want to talk in rhymes.", role: "system" },
-    { content: "Hello, world!", role: "user" },
-    { content: "Hello!", role: "assistant" },
     {
       content:
         "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",

examples/clip.ts

@@ -0,0 +1,33 @@
+import { ClipEmbedding, similarity, SimilarityType } from "llamaindex";
+
+async function main() {
+  const clip = new ClipEmbedding();
+
+  // Get text embeddings
+  const text1 = "a car";
+  const textEmbedding1 = await clip.getTextEmbedding(text1);
+  const text2 = "a football match";
+  const textEmbedding2 = await clip.getTextEmbedding(text2);
+
+  // Get image embedding
+  const image =
+    "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg";
+  const imageEmbedding = await clip.getImageEmbedding(image);
+
+  // Calc similarity
+  const sim1 = similarity(
+    textEmbedding1,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+  const sim2 = similarity(
+    textEmbedding2,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+
+  console.log(`Similarity between "${text1}" and the image is ${sim1}`);
+  console.log(`Similarity between "${text2}" and the image is ${sim2}`);
+}
+
+main();

examples/llmStream.ts

@@ -1,47 +0,0 @@
-import { ChatMessage, SimpleChatEngine } from "llamaindex";
-import { stdin as input, stdout as output } from "node:process";
-import readline from "node:readline/promises";
-import { Anthropic } from "../../packages/core/src/llm/LLM";
-
-async function main() {
-  const query: string = `
-Where is Istanbul?
-  `;
-
-  // const llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 });
-  const llm = new Anthropic();
-  const message: ChatMessage = { content: query, role: "user" };
-
-  //TODO: Add callbacks later
-
-  //Stream Complete
-  //Note: Setting streaming flag to true or false will auto-set your return type to
-  //either an AsyncGenerator or a Response.
-  // Omitting the streaming flag automatically sets streaming to false
-
-  const chatEngine: SimpleChatEngine = new SimpleChatEngine({
-    chatHistory: undefined,
-    llm: llm,
-  });
-
-  const rl = readline.createInterface({ input, output });
-  while (true) {
-    const query = await rl.question("Query: ");
-
-    if (!query) {
-      break;
-    }
-
-    //Case 1: .chat(query, undefined, true) => Stream
-    //Case 2: .chat(query, undefined, false) => Response object
-    //Case 3: .chat(query, undefined) => Response object
-    const chatStream = await chatEngine.chat(query, undefined, true);
-    var accumulated_result = "";
-    for await (const part of chatStream) {
-      accumulated_result += part;
-      process.stdout.write(part);
-    }
-  }
-}
-
-main();

examples/vectorIndexAnthropic.ts

@@ -2,7 +2,10 @@ import fs from "node:fs/promises";
 
 import {
   Anthropic,
+  anthropicTextQaPrompt,
+  CompactAndRefine,
   Document,
+  ResponseSynthesizer,
   serviceContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
@@ -18,12 +21,20 @@ async function main() {
 
   // Split text and create embeddings. Store them in a VectorStoreIndex
   const serviceContext = serviceContextFromDefaults({ llm: new Anthropic() });
+
+  const responseSynthesizer = new ResponseSynthesizer({
+    responseBuilder: new CompactAndRefine(
+      serviceContext,
+      anthropicTextQaPrompt,
+    ),
+  });
+
   const index = await VectorStoreIndex.fromDocuments([document], {
     serviceContext,
   });
 
   // Query the index
-  const queryEngine = index.asQueryEngine();
+  const queryEngine = index.asQueryEngine({ responseSynthesizer });
   const response = await queryEngine.query(
     "What did the author do in college?",
   );

package.json

@@ -13,8 +13,8 @@
   "devDependencies": {
     "@changesets/cli": "^2.26.2",
     "@turbo/gen": "^1.10.16",
-    "@types/jest": "^29.5.8",
-    "eslint": "^8.53.0",
+    "@types/jest": "^29.5.10",
+    "eslint": "^8.54.0",
     "eslint-config-custom": "workspace:*",
     "husky": "^8.0.3",
     "jest": "^29.7.0",

packages/core/CHANGELOG.md

@@ -1,5 +1,22 @@
 # llamaindex
 
+## 0.0.37
+
+### Patch Changes
+
+- 3bab231: Fixed errors (#225 and #226) Thanks @marcusschiesser
+
+## 0.0.36
+
+### Patch Changes
+
+- Support for Claude 2.1
+- Add AssemblyAI integration (thanks @Swimburger)
+- Use cryptoJS (thanks @marcusschiesser)
+- Add PGVectorStore (thanks @mtutty)
+- Add CLIP embeddings (thanks @marcusschiesser)
+- Add MongoDB support (thanks @marcusschiesser)
+
 ## 0.0.35
 
 ### Patch Changes

packages/core/package.json

@@ -1,20 +1,23 @@
 {
   "name": "llamaindex",
-  "version": "0.0.35",
+  "version": "0.0.37",
   "license": "MIT",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.9.0",
+    "@anthropic-ai/sdk": "^0.9.1",
     "@notionhq/client": "^2.2.13",
+    "@xenova/transformers": "^2.8.0",
     "crypto-js": "^4.2.0",
-    "js-tiktoken": "^1.0.7",
+    "js-tiktoken": "^1.0.8",
     "lodash": "^4.17.21",
     "mammoth": "^1.6.0",
     "md-utils-ts": "^2.0.0",
-    "mongodb": "^6.2.0",
+    "mongodb": "^6.3.0",
     "notion-md-crawler": "^0.0.2",
-    "openai": "^4.16.1",
+    "openai": "^4.19.1",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
+    "pg": "^8.11.3",
+    "pgvector": "^0.1.5",
     "portkey-ai": "^0.1.16",
     "rake-modified": "^1.0.8",
     "replicate": "^0.21.1",
@@ -24,14 +27,15 @@
   },
   "devDependencies": {
     "@types/crypto-js": "^4.2.1",
-    "@types/lodash": "^4.14.200",
-    "@types/node": "^18.18.8",
-    "@types/papaparse": "^5.3.10",
-    "@types/pdf-parse": "^1.1.3",
-    "@types/uuid": "^9.0.6",
+    "@types/lodash": "^4.14.202",
+    "@types/node": "^18.18.12",
+    "@types/papaparse": "^5.3.13",
+    "@types/pdf-parse": "^1.1.4",
+    "@types/pg": "^8.10.7",
+    "@types/uuid": "^9.0.7",
     "node-stdlib-browser": "^1.2.0",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "^5.3.2"
   },
   "engines": {
     "node": ">=18.0.0"

packages/core/src/Prompt.ts

@@ -36,6 +36,15 @@ Answer:`;
 
 export type TextQaPrompt = typeof defaultTextQaPrompt;
 
+export const anthropicTextQaPrompt = ({ context = "", query = "" }) => {
+  return `Context information:
+<context>
+${context}
+</context>
+Given the context information and not prior knowledge, answer the query.
+Query: ${query}`;
+};
+
 /*
 DEFAULT_SUMMARY_PROMPT_TMPL = (
     "Write a summary of the following. Try to use only the "

packages/core/src/QueryEngine.ts

@@ -1,4 +1,6 @@
 import { v4 as uuidv4 } from "uuid";
+import { Event } from "./callbacks/CallbackManager";
+import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   BaseQuestionGenerator,
@@ -10,8 +12,6 @@ import { CompactAndRefine, ResponseSynthesizer } from "./ResponseSynthesizer";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { QueryEngineTool, ToolMetadata } from "./Tool";
-import { Event } from "./callbacks/CallbackManager";
-import { BaseNodePostprocessor } from "./indices/BaseNodePostprocessor";
 
 /**
  * A query engine is a question answerer that can use one or more steps.

packages/core/src/ServiceContext.ts

@@ -1,8 +1,8 @@
-import { BaseEmbedding, OpenAIEmbedding } from "./Embedding";
+import { CallbackManager } from "./callbacks/CallbackManager";
+import { BaseEmbedding, OpenAIEmbedding } from "./embeddings";
+import { LLM, OpenAI } from "./llm/LLM";
 import { NodeParser, SimpleNodeParser } from "./NodeParser";
 import { PromptHelper } from "./PromptHelper";
-import { CallbackManager } from "./callbacks/CallbackManager";
-import { LLM, OpenAI } from "./llm/LLM";
 
 /**
  * The ServiceContext is a collection of components that are used in different parts of the application.

packages/core/src/embeddings/ClipEmbedding.ts

@@ -0,0 +1,78 @@
+import { MultiModalEmbedding } from "./MultiModalEmbedding";
+import { ImageType, readImage } from "./utils";
+
+export enum ClipEmbeddingModelType {
+  XENOVA_CLIP_VIT_BASE_PATCH32 = "Xenova/clip-vit-base-patch32",
+  XENOVA_CLIP_VIT_BASE_PATCH16 = "Xenova/clip-vit-base-patch16",
+}
+
+export class ClipEmbedding extends MultiModalEmbedding {
+  modelType: ClipEmbeddingModelType =
+    ClipEmbeddingModelType.XENOVA_CLIP_VIT_BASE_PATCH16;
+
+  private tokenizer: any;
+  private processor: any;
+  private visionModel: any;
+  private textModel: any;
+
+  async getTokenizer() {
+    if (!this.tokenizer) {
+      const { AutoTokenizer } = await import("@xenova/transformers");
+      this.tokenizer = await AutoTokenizer.from_pretrained(this.modelType);
+    }
+    return this.tokenizer;
+  }
+
+  async getProcessor() {
+    if (!this.processor) {
+      const { AutoProcessor } = await import("@xenova/transformers");
+      this.processor = await AutoProcessor.from_pretrained(this.modelType);
+    }
+    return this.processor;
+  }
+
+  async getVisionModel() {
+    if (!this.visionModel) {
+      const { CLIPVisionModelWithProjection } = await import(
+        "@xenova/transformers"
+      );
+      this.visionModel = await CLIPVisionModelWithProjection.from_pretrained(
+        this.modelType,
+      );
+    }
+
+    return this.visionModel;
+  }
+
+  async getTextModel() {
+    if (!this.textModel) {
+      const { CLIPTextModelWithProjection } = await import(
+        "@xenova/transformers"
+      );
+      this.textModel = await CLIPTextModelWithProjection.from_pretrained(
+        this.modelType,
+      );
+    }
+
+    return this.textModel;
+  }
+
+  async getImageEmbedding(image: ImageType): Promise<number[]> {
+    const loadedImage = await readImage(image);
+    const imageInputs = await (await this.getProcessor())(loadedImage);
+    const { image_embeds } = await (await this.getVisionModel())(imageInputs);
+    return image_embeds.data;
+  }
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    const textInputs = await (
+      await this.getTokenizer()
+    )([text], { padding: true, truncation: true });
+    const { text_embeds } = await (await this.getTextModel())(textInputs);
+    return text_embeds.data;
+  }
+
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getTextEmbedding(query);
+  }
+}

packages/core/src/embeddings/MultiModalEmbedding.ts

@@ -0,0 +1,17 @@
+import { BaseEmbedding } from "./types";
+import { ImageType } from "./utils";
+
+/*
+ * Base class for Multi Modal embeddings.
+ */
+
+export abstract class MultiModalEmbedding extends BaseEmbedding {
+  abstract getImageEmbedding(images: ImageType): Promise<number[]>;
+
+  async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
+    // Embed the input sequence of images asynchronously.
+    return Promise.all(
+      images.map((imgFilePath) => this.getImageEmbedding(imgFilePath)),
+    );
+  }
+}

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -0,0 +1,92 @@
+import { ClientOptions as OpenAIClientOptions } from "openai";
+import {
+  AzureOpenAIConfig,
+  getAzureBaseUrl,
+  getAzureConfigFromEnv,
+  getAzureModel,
+  shouldUseAzure,
+} from "../llm/azure";
+import { OpenAISession, getOpenAISession } from "../llm/openai";
+import { BaseEmbedding } from "./types";
+
+export enum OpenAIEmbeddingModelType {
+  TEXT_EMBED_ADA_002 = "text-embedding-ada-002",
+}
+
+export class OpenAIEmbedding extends BaseEmbedding {
+  model: OpenAIEmbeddingModelType;
+
+  // OpenAI session params
+  apiKey?: string = undefined;
+  maxRetries: number;
+  timeout?: number;
+  additionalSessionOptions?: Omit<
+    Partial<OpenAIClientOptions>,
+    "apiKey" | "maxRetries" | "timeout"
+  >;
+
+  session: OpenAISession;
+
+  constructor(init?: Partial<OpenAIEmbedding> & { azure?: AzureOpenAIConfig }) {
+    super();
+
+    this.model = OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002;
+
+    this.maxRetries = init?.maxRetries ?? 10;
+    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
+    this.additionalSessionOptions = init?.additionalSessionOptions;
+
+    if (init?.azure || shouldUseAzure()) {
+      const azureConfig = getAzureConfigFromEnv({
+        ...init?.azure,
+        model: getAzureModel(this.model),
+      });
+
+      if (!azureConfig.apiKey) {
+        throw new Error(
+          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable.",
+        );
+      }
+
+      this.apiKey = azureConfig.apiKey;
+      this.session =
+        init?.session ??
+        getOpenAISession({
+          azure: true,
+          apiKey: this.apiKey,
+          baseURL: getAzureBaseUrl(azureConfig),
+          maxRetries: this.maxRetries,
+          timeout: this.timeout,
+          defaultQuery: { "api-version": azureConfig.apiVersion },
+          ...this.additionalSessionOptions,
+        });
+    } else {
+      this.apiKey = init?.apiKey ?? undefined;
+      this.session =
+        init?.session ??
+        getOpenAISession({
+          apiKey: this.apiKey,
+          maxRetries: this.maxRetries,
+          timeout: this.timeout,
+          ...this.additionalSessionOptions,
+        });
+    }
+  }
+
+  private async getOpenAIEmbedding(input: string) {
+    const { data } = await this.session.openai.embeddings.create({
+      model: this.model,
+      input,
+    });
+
+    return data[0].embedding;
+  }
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    return this.getOpenAIEmbedding(text);
+  }
+
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getOpenAIEmbedding(query);
+  }
+}

packages/core/src/embeddings/index.ts

@@ -0,0 +1,5 @@
+export * from "./ClipEmbedding";
+export * from "./MultiModalEmbedding";
+export * from "./OpenAIEmbedding";
+export * from "./types";
+export * from "./utils";

packages/core/src/embeddings/types.ts

@@ -0,0 +1,24 @@
+import { similarity } from "./utils";
+
+/**
+ * Similarity type
+ * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
+ */
+export enum SimilarityType {
+  DEFAULT = "cosine",
+  DOT_PRODUCT = "dot_product",
+  EUCLIDEAN = "euclidean",
+}
+
+export abstract class BaseEmbedding {
+  similarity(
+    embedding1: number[],
+    embedding2: number[],
+    mode: SimilarityType = SimilarityType.DEFAULT,
+  ): number {
+    return similarity(embedding1, embedding2, mode);
+  }
+
+  abstract getTextEmbedding(text: string): Promise<number[]>;
+  abstract getQueryEmbedding(query: string): Promise<number[]>;
+}

packages/core/src/embeddings/utils.ts

@@ -1,33 +1,16 @@
-import { ClientOptions as OpenAIClientOptions } from "openai";
-
-import { DEFAULT_SIMILARITY_TOP_K } from "./constants";
-import {
-  AzureOpenAIConfig,
-  getAzureBaseUrl,
-  getAzureConfigFromEnv,
-  getAzureModel,
-  shouldUseAzure,
-} from "./llm/azure";
-import { OpenAISession, getOpenAISession } from "./llm/openai";
-import { VectorStoreQueryMode } from "./storage/vectorStore/types";
-
-/**
- * Similarity type
- * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
- */
-export enum SimilarityType {
-  DEFAULT = "cosine",
-  DOT_PRODUCT = "dot_product",
-  EUCLIDEAN = "euclidean",
-}
+import _ from "lodash";
+import { DEFAULT_SIMILARITY_TOP_K } from "../constants";
+import { VectorStoreQueryMode } from "../storage";
+import { SimilarityType } from "./types";
 
 /**
  * The similarity between two embeddings.
  * @param embedding1
  * @param embedding2
  * @param mode
- * @returns similartiy score with higher numbers meaning the two embeddings are more similar
+ * @returns similarity score with higher numbers meaning the two embeddings are more similar
  */
+
 export function similarity(
   embedding1: number[],
   embedding2: number[],
@@ -42,7 +25,6 @@ export function similarity(
   // will probably cause some avoidable loss of floating point precision
   // ml-distance is worth watching although they currently also use the naive
   // formulas
-
   function norm(x: number[]): number {
     let result = 0;
     for (let i = 0; i < x.length; i++) {
@@ -201,98 +183,14 @@ export function getTopKMMREmbeddings(
 
   return [resultSimilarities, resultIds];
 }
-
-export abstract class BaseEmbedding {
-  similarity(
-    embedding1: number[],
-    embedding2: number[],
-    mode: SimilarityType = SimilarityType.DEFAULT,
-  ): number {
-    return similarity(embedding1, embedding2, mode);
-  }
-
-  abstract getTextEmbedding(text: string): Promise<number[]>;
-  abstract getQueryEmbedding(query: string): Promise<number[]>;
-}
-
-enum OpenAIEmbeddingModelType {
-  TEXT_EMBED_ADA_002 = "text-embedding-ada-002",
-}
-
-export class OpenAIEmbedding extends BaseEmbedding {
-  model: OpenAIEmbeddingModelType;
-
-  // OpenAI session params
-  apiKey?: string = undefined;
-  maxRetries: number;
-  timeout?: number;
-  additionalSessionOptions?: Omit<
-    Partial<OpenAIClientOptions>,
-    "apiKey" | "maxRetries" | "timeout"
-  >;
-
-  session: OpenAISession;
-
-  constructor(init?: Partial<OpenAIEmbedding> & { azure?: AzureOpenAIConfig }) {
-    super();
-
-    this.model = OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002;
-
-    this.maxRetries = init?.maxRetries ?? 10;
-    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
-    this.additionalSessionOptions = init?.additionalSessionOptions;
-
-    if (init?.azure || shouldUseAzure()) {
-      const azureConfig = getAzureConfigFromEnv({
-        ...init?.azure,
-        model: getAzureModel(this.model),
-      });
-
-      if (!azureConfig.apiKey) {
-        throw new Error(
-          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable.",
-        );
-      }
-
-      this.apiKey = azureConfig.apiKey;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          azure: true,
-          apiKey: this.apiKey,
-          baseURL: getAzureBaseUrl(azureConfig),
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          defaultQuery: { "api-version": azureConfig.apiVersion },
-          ...this.additionalSessionOptions,
-        });
-    } else {
-      this.apiKey = init?.apiKey ?? undefined;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          apiKey: this.apiKey,
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          ...this.additionalSessionOptions,
-        });
-    }
-  }
-
-  private async getOpenAIEmbedding(input: string) {
-    const { data } = await this.session.openai.embeddings.create({
-      model: this.model,
-      input,
-    });
-
-    return data[0].embedding;
-  }
-
-  async getTextEmbedding(text: string): Promise<number[]> {
-    return this.getOpenAIEmbedding(text);
-  }
-
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getOpenAIEmbedding(query);
+export async function readImage(input: ImageType) {
+  const { RawImage } = await import("@xenova/transformers");
+  if (input instanceof Blob) {
+    return await RawImage.fromBlob(input);
+  } else if (_.isString(input) || input instanceof URL) {
+    return await RawImage.fromURL(input);
+  } else {
+    throw new Error(`Unsupported input type: ${typeof input}`);
   }
 }
+export type ImageType = string | Blob | URL;

packages/core/src/index.ts

@@ -1,6 +1,5 @@
 export * from "./ChatEngine";
 export * from "./ChatHistory";
-export * from "./Embedding";
 export * from "./GlobalsHelper";
 export * from "./Node";
 export * from "./NodeParser";
@@ -17,6 +16,7 @@ export * from "./TextSplitter";
 export * from "./Tool";
 export * from "./callbacks/CallbackManager";
 export * from "./constants";
+export * from "./embeddings";
 export * from "./indices";
 export * from "./llm/LLM";
 export * from "./readers/CSVReader";

packages/core/src/indices/summary/SummaryIndex.ts

@@ -10,11 +10,11 @@ import {
   ServiceContext,
   serviceContextFromDefaults,
 } from "../../ServiceContext";
+import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   StorageContext,
   storageContextFromDefaults,
 } from "../../storage/StorageContext";
-import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   BaseIndex,
   BaseIndexInit,

packages/core/src/llm/LLM.ts

@@ -639,7 +639,7 @@ If a question does not make any sense, or is not factually coherent, explain why
 
 export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
   // both models have 100k context window, see https://docs.anthropic.com/claude/reference/selecting-a-model
-  "claude-2": { contextWindow: 100000 },
+  "claude-2": { contextWindow: 200000 },
   "claude-instant-1": { contextWindow: 100000 },
 };
 
@@ -705,10 +705,12 @@ export class Anthropic implements LLM {
         return (
           acc +
           `${
-            message.role === "assistant"
-              ? ANTHROPIC_AI_PROMPT
-              : ANTHROPIC_HUMAN_PROMPT
-          } ${message.content} `
+            message.role === "system"
+              ? ""
+              : message.role === "assistant"
+                ? ANTHROPIC_AI_PROMPT + " "
+                : ANTHROPIC_HUMAN_PROMPT + " "
+          }${message.content.trim()}`
         );
       }, "") + ANTHROPIC_AI_PROMPT
     );
@@ -729,6 +731,7 @@ export class Anthropic implements LLM {
       }
       return this.streamChat(messages, parentEvent) as R;
     }
+
     //Non-streaming
     const response = await this.session.anthropic.completions.create({
       model: this.model,

packages/core/src/readers/DocxReader.ts

@@ -1,7 +1,7 @@
 import mammoth from "mammoth";
 import { Document } from "../Node";
-import { GenericFileSystem } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
+import { GenericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
 export class DocxReader implements BaseReader {

packages/core/src/storage/vectorStore/PGVectorStore.ts

@@ -0,0 +1,266 @@
+import pg from "pg";
+import pgvector from "pgvector/pg";
+
+import { VectorStore, VectorStoreQuery, VectorStoreQueryResult } from "./types";
+
+import { BaseNode, Document, Metadata, MetadataMode } from "../../Node";
+import { GenericFileSystem } from "../FileSystem";
+
+export const PGVECTOR_SCHEMA = "public";
+export const PGVECTOR_TABLE = "llamaindex_embedding";
+
+/**
+ * Provides support for writing and querying vector data in Postgres.
+ */
+export class PGVectorStore implements VectorStore {
+  storesText: boolean = true;
+
+  private collection: string = "";
+
+  /*
+    FROM pg LIBRARY:
+    type Config = {
+      user?: string, // default process.env.PGUSER || process.env.USER
+      password?: string or function, //default process.env.PGPASSWORD
+      host?: string, // default process.env.PGHOST
+      database?: string, // default process.env.PGDATABASE || user
+      port?: number, // default process.env.PGPORT
+      connectionString?: string, // e.g. postgres://user:password@host:5432/database
+      ssl?: any, // passed directly to node.TLSSocket, supports all tls.connect options
+      types?: any, // custom type parsers
+      statement_timeout?: number, // number of milliseconds before a statement in query will time out, default is no timeout
+      query_timeout?: number, // number of milliseconds before a query call will timeout, default is no timeout
+      application_name?: string, // The name of the application that created this Client instance
+      connectionTimeoutMillis?: number, // number of milliseconds to wait for connection, default is no timeout
+      idle_in_transaction_session_timeout?: number // number of milliseconds before terminating any session with an open idle transaction, default is no timeout
+    }  
+  */
+  db?: pg.Client;
+
+  constructor() {}
+
+  /**
+   * Setter for the collection property.
+   * Using a collection allows for simple segregation of vector data,
+   * e.g. by user, source, or access-level.
+   * Leave/set blank to ignore the collection value when querying.
+   * @param coll Name for the collection.
+   */
+  setCollection(coll: string) {
+    this.collection = coll;
+  }
+
+  /**
+   * Getter for the collection property.
+   * Using a collection allows for simple segregation of vector data,
+   * e.g. by user, source, or access-level.
+   * Leave/set blank to ignore the collection value when querying.
+   * @returns The currently-set collection value.  Default is empty string.
+   */
+  getCollection(): string {
+    return this.collection;
+  }
+
+  private async getDb(): Promise<pg.Client> {
+    if (!this.db) {
+      try {
+        // Create DB connection
+        // Read connection params from env - see comment block above
+        const db = new pg.Client();
+        await db.connect();
+
+        // Check vector extension
+        db.query("CREATE EXTENSION IF NOT EXISTS vector");
+        await pgvector.registerType(db);
+
+        // Check schema, table(s), index(es)
+        await this.checkSchema(db);
+
+        // All good?  Keep the connection reference
+        this.db = db;
+      } catch (err: any) {
+        console.error(err);
+        return Promise.reject(err);
+      }
+    }
+
+    return Promise.resolve(this.db);
+  }
+
+  private async checkSchema(db: pg.Client) {
+    await db.query(`CREATE SCHEMA IF NOT EXISTS ${PGVECTOR_SCHEMA}`);
+
+    const tbl = `CREATE TABLE IF NOT EXISTS ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE}(
+      id uuid DEFAULT gen_random_uuid() PRIMARY KEY,
+      external_id VARCHAR,
+      collection VARCHAR,
+      document TEXT,
+      metadata JSONB DEFAULT '{}',
+      embeddings VECTOR(1536)
+    )`;
+    await db.query(tbl);
+
+    const idxs = `CREATE INDEX IF NOT EXISTS idx_${PGVECTOR_TABLE}_external_id ON ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} (external_id);
+      CREATE INDEX IF NOT EXISTS idx_${PGVECTOR_TABLE}_collection ON ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} (collection);`;
+    await db.query(idxs);
+
+    // TODO add IVFFlat or HNSW indexing?
+    return db;
+  }
+
+  // isEmbeddingQuery?: boolean | undefined;
+
+  /**
+   * Connects to the database specified in environment vars.
+   * This method also checks and creates the vector extension,
+   * the destination table and indexes if not found.
+   * @returns A connection to the database, or the error encountered while connecting/setting up.
+   */
+  client() {
+    return this.getDb();
+  }
+
+  /**
+   * Delete all vector records for the specified collection.
+   * NOTE: Uses the collection property controlled by setCollection/getCollection.
+   * @returns The result of the delete query.
+   */
+  async clearCollection() {
+    const sql: string = `DELETE FROM ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} 
+      WHERE collection = $1`;
+
+    const db = (await this.getDb()) as pg.Client;
+    const ret = await db.query(sql, [this.collection]);
+
+    return ret;
+  }
+
+  /**
+   * Adds vector record(s) to the table.
+   * NOTE: Uses the collection property controlled by setCollection/getCollection.
+   * @param embeddingResults The Nodes to be inserted, optionally including metadata tuples.
+   * @returns A list of zero or more id values for the created records.
+   */
+  async add(embeddingResults: BaseNode<Metadata>[]): Promise<string[]> {
+    const sql: string = `INSERT INTO ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} 
+      (id, external_id, collection, document, metadata, embeddings) 
+      VALUES ($1, $2, $3, $4, $5, $6)`;
+
+    const db = (await this.getDb()) as pg.Client;
+
+    let ret: string[] = [];
+    for (let index = 0; index < embeddingResults.length; index++) {
+      const row = embeddingResults[index];
+
+      let id: any = row.id_.length ? row.id_ : null;
+      let meta = row.metadata || {};
+      meta.create_date = new Date();
+
+      const params = [
+        id,
+        "",
+        this.collection,
+        row.getContent(MetadataMode.EMBED),
+        meta,
+        "[" + row.getEmbedding().join(",") + "]",
+      ];
+
+      try {
+        const result = await db.query(sql, params);
+
+        if (result.rows.length) {
+          id = result.rows[0].id as string;
+          ret.push(id);
+        }
+      } catch (err) {
+        const msg = `${err}`;
+        console.log(msg, err);
+      }
+    }
+
+    return Promise.resolve(ret);
+  }
+
+  /**
+   * Deletes a single record from the database by id.
+   * NOTE: Uses the collection property controlled by setCollection/getCollection.
+   * @param refDocId Unique identifier for the record to delete.
+   * @param deleteKwargs Required by VectorStore interface.  Currently ignored.
+   * @returns Promise that resolves if the delete query did not throw an error.
+   */
+  async delete(refDocId: string, deleteKwargs?: any): Promise<void> {
+    const collectionCriteria = this.collection.length
+      ? "AND collection = $2"
+      : "";
+    const sql: string = `DELETE FROM ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE} 
+      WHERE id = $1 ${collectionCriteria}`;
+
+    const db = (await this.getDb()) as pg.Client;
+    const params = this.collection.length
+      ? [refDocId, this.collection]
+      : [refDocId];
+    await db.query(sql, params);
+    return Promise.resolve();
+  }
+
+  /**
+   * Query the vector store for the closest matching data to the query embeddings
+   * @param query The VectorStoreQuery to be used
+   * @param options Required by VectorStore interface.  Currently ignored.
+   * @returns Zero or more Document instances with data from the vector store.
+   */
+  async query(
+    query: VectorStoreQuery,
+    options?: any,
+  ): Promise<VectorStoreQueryResult> {
+    // TODO QUERY TYPES:
+    //    Distance:       SELECT embedding <-> $1 AS distance FROM items;
+    //    Inner Product:  SELECT (embedding <#> $1) * -1 AS inner_product FROM items;
+    //    Cosine Sim:     SELECT 1 - (embedding <=> $1) AS cosine_similarity FROM items;
+
+    const embedding = "[" + query.queryEmbedding?.join(",") + "]";
+    const max = query.similarityTopK ?? 2;
+    const where = this.collection.length ? "WHERE collection = $2" : "";
+    // TODO Add collection filter if set
+    const sql = `SELECT * FROM ${PGVECTOR_SCHEMA}.${PGVECTOR_TABLE}
+      ${where}
+      ORDER BY embeddings <-> $1 LIMIT ${max}
+    `;
+
+    const db = (await this.getDb()) as pg.Client;
+    const params = this.collection.length
+      ? [embedding, this.collection]
+      : [embedding];
+    const results = await db.query(sql, params);
+
+    const nodes = results.rows.map((row) => {
+      return new Document({
+        id_: row.id,
+        text: row.document,
+        metadata: row.metadata,
+        embedding: row.embeddings,
+      });
+    });
+
+    const ret = {
+      nodes: nodes,
+      similarities: results.rows.map((row) => row.embeddings),
+      ids: results.rows.map((row) => row.id),
+    };
+
+    return Promise.resolve(ret);
+  }
+
+  /**
+   * Required by VectorStore interface.  Currently ignored.
+   * @param persistPath
+   * @param fs
+   * @returns Resolved Promise.
+   */
+  persist(
+    persistPath: string,
+    fs?: GenericFileSystem | undefined,
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+}

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -1,11 +1,11 @@
 import _ from "lodash";
 import * as path from "path";
+import { BaseNode } from "../../Node";
 import {
   getTopKEmbeddings,
   getTopKEmbeddingsLearner,
   getTopKMMREmbeddings,
-} from "../../Embedding";
-import { BaseNode } from "../../Node";
+} from "../../embeddings";
 import { GenericFileSystem, exists } from "../FileSystem";
 import { DEFAULT_FS, DEFAULT_PERSIST_DIR } from "../constants";
 import {

packages/core/src/storage/vectorStore/utils.ts

@@ -1,4 +1,4 @@
-import { BaseNode, Metadata, ObjectType, jsonToNode } from "../../Node";
+import { BaseNode, jsonToNode, Metadata, ObjectType } from "../../Node";
 
 const DEFAULT_TEXT_KEY = "text";
 

packages/core/src/tests/CallbackManager.test.ts

@@ -1,18 +1,18 @@
-import { OpenAIEmbedding } from "../Embedding";
+import {
+  CallbackManager,
+  RetrievalCallbackResponse,
+  StreamCallbackResponse,
+} from "../callbacks/CallbackManager";
+import { OpenAIEmbedding } from "../embeddings";
+import { SummaryIndex } from "../indices/summary";
+import { VectorStoreIndex } from "../indices/vectorStore/VectorStoreIndex";
+import { OpenAI } from "../llm/LLM";
 import { Document } from "../Node";
 import {
   ResponseSynthesizer,
   SimpleResponseBuilder,
 } from "../ResponseSynthesizer";
 import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext";
-import {
-  CallbackManager,
-  RetrievalCallbackResponse,
-  StreamCallbackResponse,
-} from "../callbacks/CallbackManager";
-import { SummaryIndex } from "../indices/summary";
-import { VectorStoreIndex } from "../indices/vectorStore/VectorStoreIndex";
-import { OpenAI } from "../llm/LLM";
 import { mockEmbeddingModel, mockLlmGeneration } from "./utility/mockOpenAI";
 
 // Mock the OpenAI getOpenAISession function during testing

packages/core/src/tests/Embedding.test.ts

@@ -1,4 +1,4 @@
-import { SimilarityType, similarity } from "../Embedding";
+import { similarity, SimilarityType } from "../embeddings";
 
 describe("similarity", () => {
   test("throws error on mismatched lengths", () => {

packages/core/src/tests/utility/mockOpenAI.ts

@@ -1,6 +1,6 @@
-import { OpenAIEmbedding } from "../../Embedding";
-import { globalsHelper } from "../../GlobalsHelper";
 import { CallbackManager, Event } from "../../callbacks/CallbackManager";
+import { OpenAIEmbedding } from "../../embeddings";
+import { globalsHelper } from "../../GlobalsHelper";
 import { ChatMessage, OpenAI } from "../../llm/LLM";
 
 export function mockLlmGeneration({

packages/create-llama/templates/components/engines/context/generate.mjs

@@ -5,6 +5,8 @@ import {
   VectorStoreIndex,
 } from "llamaindex";
 
+import * as dotenv from "dotenv";
+
 import {
   CHUNK_OVERLAP,
   CHUNK_SIZE,
@@ -12,6 +14,9 @@ import {
   STORAGE_DIR,
 } from "./constants.mjs";
 
+// Load environment variables from local .env file
+dotenv.config();
+
 async function getRuntime(func) {
   const start = Date.now();
   await func();

packages/create-llama/templates/index.ts

@@ -14,26 +14,14 @@ import {
   TemplateFramework,
 } from "./types";
 
-const envFileNameMap: Record<TemplateFramework, string> = {
-  nextjs: ".env.local",
-  express: ".env",
-  fastapi: ".env",
-};
-
-const createEnvLocalFile = async (
-  root: string,
-  framework: TemplateFramework,
-  openAIKey?: string,
-) => {
+const createEnvLocalFile = async (root: string, openAIKey?: string) => {
   if (openAIKey) {
-    const envFileName = envFileNameMap[framework];
-    if (!envFileName) return;
+    const envFileName = ".env";
     await fs.writeFile(
       path.join(root, envFileName),
       `OPENAI_API_KEY=${openAIKey}\n`,
     );
     console.log(`Created '${envFileName}' file containing OPENAI_API_KEY`);
-    process.env["OPENAI_API_KEY"] = openAIKey;
   }
 };
 
@@ -42,7 +30,16 @@ const copyTestData = async (
   framework: TemplateFramework,
   packageManager?: PackageManager,
   engine?: TemplateEngine,
+  openAIKey?: string,
 ) => {
+  if (framework === "nextjs") {
+    // XXX: This is a hack to make the build for nextjs work with pdf-parse
+    // pdf-parse needs './test/data/05-versions-space.pdf' to exist - can be removed when pdf-parse is removed
+    const srcFile = path.join(__dirname, "components", "data", "101.pdf");
+    const destPath = path.join(root, "test", "data");
+    await fs.mkdir(destPath, { recursive: true });
+    await fs.copyFile(srcFile, path.join(destPath, "05-versions-space.pdf"));
+  }
   if (engine === "context" || framework === "fastapi") {
     const srcPath = path.join(__dirname, "components", "data");
     const destPath = path.join(root, "data");
@@ -54,7 +51,7 @@ const copyTestData = async (
   }
 
   if (packageManager && engine === "context") {
-    if (process.env["OPENAI_API_KEY"]) {
+    if (openAIKey || process.env["OPENAI_API_KEY"]) {
       console.log(
         `\nRunning ${cyan(
           `${packageManager} run generate`,
@@ -226,6 +223,7 @@ const installTSTemplate = async ({
       "tailwind-merge": "^2",
       "@radix-ui/react-slot": "^1",
       "class-variance-authority": "^0.7",
+      clsx: "^1.2.1",
       "lucide-react": "^0.291",
       remark: "^14.0.3",
       "remark-code-import": "^1.2.0",
@@ -313,7 +311,7 @@ export const installTemplate = async (
     // This is a backend, so we need to copy the test data and create the env file.
 
     // Copy the environment file to the target directory.
-    await createEnvLocalFile(props.root, props.framework, props.openAIKey);
+    await createEnvLocalFile(props.root, props.openAIKey);
 
     // Copy test pdf file
     await copyTestData(
@@ -321,6 +319,7 @@ export const installTemplate = async (
       props.framework,
       props.packageManager,
       props.engine,
+      props.openAIKey,
     );
   }
 };

packages/create-llama/templates/types/simple/fastapi/README-template.md

@@ -9,6 +9,14 @@ poetry install
 poetry shell
 ```
 
+By default, we use the OpenAI LLM (though you can customize, see app/api/routers/chat.py). As a result you need to specify an `OPENAI_API_KEY` in an .env file in this directory.
+
+Example `backend/.env` file:
+
+```
+OPENAI_API_KEY=<openai_api_key>
+```
+
 Second, run the development server:
 
 ```

packages/create-llama/templates/types/simple/nextjs/next.config.app.js

@@ -1,5 +1,15 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
+  webpack: (config) => {
+    // See https://webpack.js.org/configuration/resolve/#resolvealias
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      sharp$: false,
+      "onnxruntime-node$": false,
+      mongodb$: false,
+    };
+    return config;
+  },
   experimental: {
     serverComponentsExternalPackages: ["llamaindex"],
     outputFileTracingIncludes: {

packages/create-llama/templates/types/simple/nextjs/next.config.static.js

@@ -2,6 +2,16 @@
 const nextConfig = {
   output: "export",
   images: { unoptimized: true },
+  webpack: (config) => {
+    // See https://webpack.js.org/configuration/resolve/#resolvealias
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      sharp$: false,
+      "onnxruntime-node$": false,
+      mongodb$: false,
+    };
+    return config;
+  },
   experimental: {
     serverComponentsExternalPackages: ["llamaindex"],
     outputFileTracingIncludes: {

packages/create-llama/templates/types/simple/nextjs/package.json

@@ -9,6 +9,7 @@
   },
   "dependencies": {
     "llamaindex": "0.0.31",
+    "dotenv": "^16.3.1",
     "nanoid": "^5",
     "next": "^13",
     "react": "^18",
@@ -18,11 +19,11 @@
     "@types/node": "^20",
     "@types/react": "^18",
     "@types/react-dom": "^18",
-    "autoprefixer": "^10",
+    "autoprefixer": "^10.1",
     "eslint": "^8",
     "eslint-config-next": "^13",
     "postcss": "^8",
-    "tailwindcss": "^3",
+    "tailwindcss": "^3.3",
     "typescript": "^5"
   }
 }

packages/create-llama/templates/types/simple/nextjs/tsconfig.json

@@ -1,7 +1,11 @@
 {
   "compilerOptions": {
     "target": "es5",
-    "lib": ["dom", "dom.iterable", "esnext"],
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
     "allowJs": true,
     "skipLibCheck": true,
     "strict": true,
@@ -19,9 +23,19 @@
       }
     ],
     "paths": {
-      "@/*": ["./*"]
-    }
+      "@/*": [
+        "./*"
+      ]
+    },
+    "forceConsistentCasingInFileNames": true,
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
-  "exclude": ["node_modules"]
-}
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}

packages/create-llama/templates/types/streaming/express/README-template.md

@@ -18,7 +18,7 @@ Then call the express API endpoint `/api/chat` to see the result:
 
 ```
 curl --location 'localhost:8000/api/chat' \
---header 'Content-Type: application/json' \
+--header 'Content-Type: text/plain' \
 --data '{ "messages": [{ "role": "user", "content": "Hello" }] }'
 ```
 

packages/create-llama/templates/types/streaming/express/package.json

@@ -9,7 +9,7 @@
     "dev": "concurrently \"tsup index.ts --format esm --dts --watch\" \"nodemon -q dist/index.js\""
   },
   "dependencies": {
-    "ai": "^2",
+    "ai": "^2.2.5",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "express": "^4",
@@ -25,4 +25,4 @@
     "tsup": "^7",
     "typescript": "^5"
   }
-}
+}

packages/create-llama/templates/types/streaming/fastapi/README-template.md

@@ -9,6 +9,14 @@ poetry install
 poetry shell
 ```
 
+By default, we use the OpenAI LLM (though you can customize, see app/api/routers/chat.py). As a result you need to specify an `OPENAI_API_KEY` in an .env file in this directory.
+
+Example `backend/.env` file:
+
+```
+OPENAI_API_KEY=<openai_api_key>
+```
+
 Second, run the development server:
 
 ```

packages/create-llama/templates/types/streaming/nextjs/next.config.app.js

@@ -1,5 +1,15 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
+  webpack: (config) => {
+    // See https://webpack.js.org/configuration/resolve/#resolvealias
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      sharp$: false,
+      "onnxruntime-node$": false,
+      mongodb$: false,
+    };
+    return config;
+  },
   experimental: {
     serverComponentsExternalPackages: ["llamaindex"],
     outputFileTracingIncludes: {

packages/create-llama/templates/types/streaming/nextjs/next.config.static.js

@@ -2,6 +2,16 @@
 const nextConfig = {
   output: "export",
   images: { unoptimized: true },
+  webpack: (config) => {
+    // See https://webpack.js.org/configuration/resolve/#resolvealias
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      sharp$: false,
+      "onnxruntime-node$": false,
+      mongodb$: false,
+    };
+    return config;
+  },
   experimental: {
     serverComponentsExternalPackages: ["llamaindex"],
     outputFileTracingIncludes: {

packages/create-llama/templates/types/streaming/nextjs/package.json

@@ -8,8 +8,9 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "ai": "^2",
+    "ai": "^2.2.5",
     "llamaindex": "0.0.31",
+    "dotenv": "^16.3.1",
     "next": "^13",
     "react": "^18",
     "react-dom": "^18"
@@ -18,11 +19,11 @@
     "@types/node": "^20",
     "@types/react": "^18",
     "@types/react-dom": "^18",
-    "autoprefixer": "^10",
+    "autoprefixer": "^10.1",
     "eslint": "^8",
     "eslint-config-next": "^13",
     "postcss": "^8",
-    "tailwindcss": "^3",
+    "tailwindcss": "^3.3",
     "typescript": "^5"
   }
 }

packages/create-llama/templates/types/streaming/nextjs/tsconfig.json

@@ -1,7 +1,11 @@
 {
   "compilerOptions": {
     "target": "es5",
-    "lib": ["dom", "dom.iterable", "esnext"],
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
     "allowJs": true,
     "skipLibCheck": true,
     "strict": true,
@@ -19,9 +23,19 @@
       }
     ],
     "paths": {
-      "@/*": ["./*"]
-    }
+      "@/*": [
+        "./*"
+      ]
+    },
+    "forceConsistentCasingInFileNames": true,
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
-  "exclude": ["node_modules"]
-}
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}