RELEASING: Releasing 1 package(s)

Releases:
  llamaindex@0.1.2

[skip ci]

.changeset/nervous-numbers-swim.md

@@ -1,5 +0,0 @@
----
-"create-llama": patch
----
-
-Add an option that allows the user to run the generated app

.changeset/shy-cars-know.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add node_modules to gitignore in Express backends

.npmrc

@@ -1 +1,2 @@
 auto-install-peers = true
+enable-pre-post-scripts = true

.prettierignore

@@ -1,4 +1,5 @@
 apps/docs/i18n
+apps/docs/docs/api
 pnpm-lock.yaml
 lib/
 dist/

apps/docs/CHANGELOG.md

@@ -0,0 +1,7 @@
+# docs
+
+## 0.0.1
+
+### Patch Changes
+
+- 3154f52: chore: add qdrant readme

apps/docs/docs/modules/documents_and_nodes.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 0
+sidebar_position: 1
 ---
 
 # Documents and Nodes

apps/docs/docs/modules/llm.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 0
+sidebar_position: 1
 ---
 
 # LLM

apps/docs/docs/modules/vectorStores/_category_.yml

@@ -0,0 +1,2 @@
+label: "Vector Stores"
+position: 0

apps/docs/docs/modules/vectorStores/qdrant.md

@@ -0,0 +1,88 @@
+# Qdrant Vector Store
+
+To run this example, you need to have a Qdrant instance running. You can run it with Docker:
+
+```bash
+docker pull qdrant/qdrant
+docker run -p 6333:6333 qdrant/qdrant
+```
+
+## Importing the modules
+
+```ts
+import fs from "node:fs/promises";
+import { Document, VectorStoreIndex, QdrantVectorStore } from "llamaindex";
+```
+
+## Load the documents
+
+```ts
+const path = "node_modules/llamaindex/examples/abramov.txt";
+const essay = await fs.readFile(path, "utf-8");
+```
+
+## Setup Qdrant
+
+```ts
+const vectorStore = new QdrantVectorStore({
+  url: "http://localhost:6333",
+  port: 6333,
+});
+```
+
+## Setup the index
+
+```ts
+const document = new Document({ text: essay, id_: path });
+
+const index = await VectorStoreIndex.fromDocuments([document], {
+  vectorStore,
+});
+```
+
+## Query the index
+
+```ts
+const queryEngine = index.asQueryEngine();
+
+const response = await queryEngine.query({
+  query: "What did the author do in college?",
+});
+
+// Output response
+console.log(response.toString());
+```
+
+## Full code
+
+```ts
+import fs from "node:fs/promises";
+import { Document, VectorStoreIndex, QdrantVectorStore } from "llamaindex";
+
+async function main() {
+  const path = "node_modules/llamaindex/examples/abramov.txt";
+  const essay = await fs.readFile(path, "utf-8");
+
+  const vectorStore = new QdrantVectorStore({
+    url: "http://localhost:6333",
+    port: 6333,
+  });
+
+  const document = new Document({ text: essay, id_: path });
+
+  const index = await VectorStoreIndex.fromDocuments([document], {
+    vectorStore,
+  });
+
+  const queryEngine = index.asQueryEngine();
+
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);
+```

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.0",
+  "version": "0.0.1",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/jupyter/.gitignore

@@ -0,0 +1 @@
+.ipynb_checkpoints/

examples/jupyter/README.md

@@ -0,0 +1,31 @@
+# Jupyter examples
+
+## Preparation
+
+1. Install Deno, e.g. on macOS:
+
+```
+brew install deno
+```
+
+2. Install Jupyter
+
+```
+pip3 install jupyterlab
+```
+
+3. Install Deno kernel
+
+```
+deno jupyter --unstable --install
+```
+
+4. Run Jupyter
+
+```
+jupyter lab
+```
+
+## Run examples
+
+Then you can open in Jupyter any of the examples in this directory.

examples/jupyter/nodeparser.ipynb

@@ -0,0 +1,82 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "8be89595-8885-4d5e-b1da-1df04eda5c7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import {\n",
+    "  Document,\n",
+    "  SimpleNodeParser\n",
+    "} from \"npm:llamaindex\";"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "65de03f9-455a-4c59-9089-093cb6998af7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[\n",
+      "  TextNode {\n",
+      "    id_: \u001b[32m\"1b2ab25e-562a-4821-bdde-860bd23121c1\"\u001b[39m,\n",
+      "    metadata: {},\n",
+      "    excludedEmbedMetadataKeys: [],\n",
+      "    excludedLlmMetadataKeys: [],\n",
+      "    relationships: {\n",
+      "      SOURCE: {\n",
+      "        nodeId: \u001b[32m\"0cb8de0e-845f-4e73-a7bd-f04426aacfed\"\u001b[39m,\n",
+      "        metadata: {},\n",
+      "        hash: \u001b[32m\"jatVVXETDFjV2fV1/fbTrdpY6ZGnSYekq9m1X/Ff1qs=\"\u001b[39m\n",
+      "      }\n",
+      "    },\n",
+      "    hash: \u001b[32m\"zVyeDsfMwWH1CqK2269o5uzGWl/DpIWO4ZcVCuyENi4=\"\u001b[39m,\n",
+      "    text: \u001b[32m\"I am 10 years old. John is 20 years old.\"\u001b[39m,\n",
+      "    metadataSeparator: \u001b[32m\"\\n\"\u001b[39m\n",
+      "  }\n",
+      "]\n"
+     ]
+    }
+   ],
+   "source": [
+    "const nodeParser = new SimpleNodeParser();\n",
+    "const nodes = nodeParser.getNodesFromDocuments([\n",
+    "    new Document({ text: \"I am 10 years old. John is 20 years old.\" }),\n",
+    "]);\n",
+    "\n",
+    "console.log(nodes);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fc0ec12f-2062-47af-916d-7c77ca39433a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Deno",
+   "language": "typescript",
+   "name": "deno"
+  },
+  "language_info": {
+   "file_extension": ".ts",
+   "mimetype": "text/x.typescript",
+   "name": "typescript",
+   "nb_converter": "script",
+   "pygments_lexer": "typescript",
+   "version": "5.3.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/jupyter/vectorIndex.ipynb

@@ -0,0 +1,83 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "8be89595-8885-4d5e-b1da-1df04eda5c7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import {\n",
+    "  Document,\n",
+    "  VectorStoreIndex\n",
+    "} from \"npm:llamaindex\";"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "65de03f9-455a-4c59-9089-093cb6998af7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "In college, the author studied subjects such as linear algebra and physics, but did not find them particularly interesting. They also slacked off and skipped lectures, leading to gaps in their knowledge. They had a negative experience with their English classes and became resentful and suspicious of higher education. They eventually dropped out of college and did not return until five years later to pick up their papers.\n"
+     ]
+    }
+   ],
+   "source": [
+    "// Create Document object with essay\n",
+    "const resp = await fetch('https://raw.githubusercontent.com/run-llama/LlamaIndexTS/main/packages/core/examples/abramov.txt');\n",
+    "const text = await resp.text();\n",
+    "const document = new Document({ text });\n",
+    "\n",
+    "// Split text and create embeddings. Store them in a VectorStoreIndex\n",
+    "const index = await VectorStoreIndex.fromDocuments([document]);\n",
+    "\n",
+    "// Query the index\n",
+    "const queryEngine = index.asQueryEngine();\n",
+    "const response = await queryEngine.query({\n",
+    "    query: \"What did the author do in college?\",\n",
+    "});\n",
+    "\n",
+    "// Output response\n",
+    "console.log(response.toString());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fc0ec12f-2062-47af-916d-7c77ca39433a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "68bdd292-5cf7-46e0-8646-51be1f070ad6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Deno",
+   "language": "typescript",
+   "name": "deno"
+  },
+  "language_info": {
+   "file_extension": ".ts",
+   "mimetype": "text/x.typescript",
+   "name": "typescript",
+   "nb_converter": "script",
+   "pygments_lexer": "typescript",
+   "version": "5.3.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/together-ai.ts

@@ -23,6 +23,7 @@ const together = new TogetherLLM({
   for await (const message of generator) {
     process.stdout.write(message.delta);
   }
+  console.log();
   const embedding = new TogetherEmbedding();
   const vector = await embedding.getTextEmbedding("Hello world!");
   console.log("vector:", vector);

examples/vectorIndexEmbed3.ts

@@ -0,0 +1,41 @@
+import fs from "node:fs/promises";
+
+import {
+  Document,
+  OpenAIEmbedding,
+  VectorStoreIndex,
+  serviceContextFromDefaults,
+} from "llamaindex";
+
+async function main() {
+  // Load essay from abramov.txt in Node
+  const path = "node_modules/llamaindex/examples/abramov.txt";
+
+  const essay = await fs.readFile(path, "utf-8");
+
+  // Create Document object with essay
+  const document = new Document({ text: essay, id_: path });
+
+  // Create service context and specify text-embedding-3-large
+  const embedModel = new OpenAIEmbedding({
+    model: "text-embedding-3-large",
+    dimensions: 1024,
+  });
+  const serviceContext = serviceContextFromDefaults({ embedModel });
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  const index = await VectorStoreIndex.fromDocuments([document], {
+    serviceContext,
+  });
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);

package.json

@@ -18,7 +18,7 @@
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",
-    "@turbo/gen": "^1.11.2",
+    "@turbo/gen": "^1.11.3",
     "@types/jest": "^29.5.11",
     "eslint": "^8.56.0",
     "eslint-config-custom": "workspace:*",
@@ -27,8 +27,8 @@
     "lint-staged": "^15.2.0",
     "prettier": "^3.2.4",
     "prettier-plugin-organize-imports": "^3.2.4",
-    "ts-jest": "^29.1.1",
-    "turbo": "^1.11.2",
+    "ts-jest": "^29.1.2",
+    "turbo": "^1.11.3",
     "typescript": "^5.3.3"
   },
   "packageManager": "pnpm@8.10.5+sha256.a4bd9bb7b48214bbfcd95f264bd75bb70d100e5d4b58808f5cd6ab40c6ac21c5",

packages/core/CHANGELOG.md

@@ -1,5 +1,35 @@
 # llamaindex
 
+## 0.1.2
+
+- e4b807a: fix: invalid package.json
+
+## 0.1.1
+
+No changes for this release.
+
+## 0.1.0
+
+### Minor Changes
+
+- 3154f52: chore: add qdrant readme
+
+### Patch Changes
+
+- bb66cb7: add new OpenAI embeddings (with dimension reduction support)
+
+## 0.0.51
+
+### Patch Changes
+
+- fda8024: revert: export conditions not working with moduleResolution `node`
+
+## 0.0.50
+
+### Patch Changes
+
+- 8a729cd: fix bugs in Together.AI integration (thanks @Nutlope for reporting)
+
 ## 0.0.49
 
 ### Patch Changes

packages/core/README.md

@@ -1,130 +0,0 @@
-# LlamaIndex.TS
-
-[![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
-[![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
-[![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
-[![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
-
-LlamaIndex is a data framework for your LLM application.
-
-Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.
-
-Documentation: https://ts.llamaindex.ai/
-
-## What is LlamaIndex.TS?
-
-LlamaIndex.TS aims to be a lightweight, easy to use set of libraries to help you integrate large language models into your applications with your own data.
-
-## Getting started with an example:
-
-LlamaIndex.TS requires Node v18 or higher. You can download it from https://nodejs.org or use https://nvm.sh (our preferred option).
-
-In a new folder:
-
-```bash
-export OPENAI_API_KEY="sk-......" # Replace with your key from https://platform.openai.com/account/api-keys
-pnpm init
-pnpm install typescript
-pnpm exec tsc --init # if needed
-pnpm install llamaindex
-pnpm install @types/node
-```
-
-Create the file example.ts
-
-```ts
-// example.ts
-import fs from "fs/promises";
-import { Document, VectorStoreIndex } from "llamaindex";
-
-async function main() {
-  // Load essay from abramov.txt in Node
-  const essay = await fs.readFile(
-    "node_modules/llamaindex/examples/abramov.txt",
-    "utf-8",
-  );
-
-  // Create Document object with essay
-  const document = new Document({ text: essay });
-
-  // Split text and create embeddings. Store them in a VectorStoreIndex
-  const index = await VectorStoreIndex.fromDocuments([document]);
-
-  // Query the index
-  const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
-
-  // Output response
-  console.log(response.toString());
-}
-
-main();
-```
-
-Then you can run it using
-
-```bash
-pnpx ts-node example.ts
-```
-
-## Playground
-
-Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground
-
-## Core concepts for getting started:
-
-- [Document](/packages/core/src/Node.ts): A document represents a text file, PDF file or other contiguous piece of data.
-
-- [Node](/packages/core/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
-
-- [Embedding](/packages/core/src/Embedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton.
-
-- [Indices](/packages/core/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
-
-- [QueryEngine](/packages/core/src/QueryEngine.ts): Query engines are what generate the query you put in and give you back the result. Query engines generally combine a pre-built prompt with selected Nodes from your Index to give the LLM the context it needs to answer your query.
-
-- [ChatEngine](/packages/core/src/ChatEngine.ts): A ChatEngine helps you build a chatbot that will interact with your Indices.
-
-- [SimplePrompt](/packages/core/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
-
-## Note: NextJS:
-
-If you're using NextJS App Router, you'll need to use the NodeJS runtime (default) and add the following config to your next.config.js to have it use imports/exports in the same way Node does.
-
-```js
-export const runtime = "nodejs"; // default
-```
-
-```js
-// next.config.js
-/** @type {import('next').NextConfig} */
-const nextConfig = {
-  webpack: (config) => {
-    config.resolve.alias = {
-      ...config.resolve.alias,
-      sharp$: false,
-      "onnxruntime-node$": false,
-    };
-    return config;
-  },
-};
-
-module.exports = nextConfig;
-```
-
-## Supported LLMs:
-
-- OpenAI GPT-3.5-turbo and GPT-4
-- Anthropic Claude Instant and Claude 2
-- Llama2 Chat LLMs (70B, 13B, and 7B parameters)
-- MistralAI Chat LLMs
-
-## Contributing:
-
-We are in the very early days of LlamaIndex.TS. If you’re interested in hacking on it with us check out our [contributing guide](/CONTRIBUTING.md)
-
-## Bugs? Questions?
-
-Please join our Discord! https://discord.com/invite/eN6D2HQ4aX

packages/core/package.json

@@ -1,6 +1,7 @@
 {
   "name": "llamaindex",
-  "version": "0.0.49",
+  "private": true,
+  "version": "0.1.2",
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.9.1",
@@ -11,7 +12,7 @@
     "@qdrant/js-client-rest": "^1.7.0",
     "@xenova/transformers": "^2.10.0",
     "assemblyai": "^4.0.0",
-    "chromadb": "^1.7.3",
+    "chromadb": "~1.7.3",
     "file-type": "^18.7.0",
     "js-tiktoken": "^1.0.8",
     "lodash": "^4.17.21",
@@ -19,7 +20,7 @@
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.3.0",
     "notion-md-crawler": "^0.0.2",
-    "openai": "^4.20.1",
+    "openai": "^4.26.0",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
     "pdfjs-dist": "4.0.269",
@@ -33,12 +34,14 @@
   },
   "devDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
+    "@types/edit-json-file": "^1.7.3",
     "@types/jest": "^29.5.11",
     "@types/lodash": "^4.14.202",
     "@types/node": "^18.19.6",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.10.9",
     "bunchee": "^4.4.1",
+    "edit-json-file": "^1.8.0",
     "madge": "^6.1.0",
     "typescript": "^5.3.3"
   },
@@ -50,154 +53,19 @@
   "exports": {
     ".": {
       "types": "./dist/index.d.mts",
-      "edge-light": "./dist/index.edge-light.mjs",
       "import": "./dist/index.mjs",
+      "edge-light": "./dist/index.mjs",
       "require": "./dist/index.js"
     },
     "./env": {
       "types": "./dist/env.d.mts",
-      "edge-light": "./dist/env.edge-light.mjs",
       "import": "./dist/env.mjs",
+      "edge-light": "./dist/env.mjs",
       "require": "./dist/env.js"
-    },
-    "./storage/FileSystem": {
-      "types": "./dist/storage/FileSystem.d.mts",
-      "edge-light": "./dist/storage/FileSystem.edge-light.mjs",
-      "import": "./dist/storage/FileSystem.mjs",
-      "require": "./dist/storage/FileSystem.js"
-    },
-    "./ChatHistory": {
-      "types": "./dist/ChatHistory.d.mts",
-      "import": "./dist/ChatHistory.mjs",
-      "require": "./dist/ChatHistory.js"
-    },
-    "./constants": {
-      "types": "./dist/constants.d.mts",
-      "import": "./dist/constants.mjs",
-      "require": "./dist/constants.js"
-    },
-    "./GlobalsHelper": {
-      "types": "./dist/GlobalsHelper.d.mts",
-      "import": "./dist/GlobalsHelper.mjs",
-      "require": "./dist/GlobalsHelper.js"
-    },
-    "./Node": {
-      "types": "./dist/Node.d.mts",
-      "import": "./dist/Node.mjs",
-      "require": "./dist/Node.js"
-    },
-    "./OutputParser": {
-      "types": "./dist/OutputParser.d.mts",
-      "import": "./dist/OutputParser.mjs",
-      "require": "./dist/OutputParser.js"
-    },
-    "./Prompt": {
-      "types": "./dist/Prompt.d.mts",
-      "import": "./dist/Prompt.mjs",
-      "require": "./dist/Prompt.js"
-    },
-    "./PromptHelper": {
-      "types": "./dist/PromptHelper.d.mts",
-      "import": "./dist/PromptHelper.mjs",
-      "require": "./dist/PromptHelper.js"
-    },
-    "./QueryEngine": {
-      "types": "./dist/QueryEngine.d.mts",
-      "import": "./dist/QueryEngine.mjs",
-      "require": "./dist/QueryEngine.js"
-    },
-    "./QuestionGenerator": {
-      "types": "./dist/QuestionGenerator.d.mts",
-      "import": "./dist/QuestionGenerator.mjs",
-      "require": "./dist/QuestionGenerator.js"
-    },
-    "./Response": {
-      "types": "./dist/Response.d.mts",
-      "import": "./dist/Response.mjs",
-      "require": "./dist/Response.js"
-    },
-    "./Retriever": {
-      "types": "./dist/Retriever.d.mts",
-      "import": "./dist/Retriever.mjs",
-      "require": "./dist/Retriever.js"
-    },
-    "./ServiceContext": {
-      "types": "./dist/ServiceContext.d.mts",
-      "import": "./dist/ServiceContext.mjs",
-      "require": "./dist/ServiceContext.js"
-    },
-    "./TextSplitter": {
-      "types": "./dist/TextSplitter.d.mts",
-      "import": "./dist/TextSplitter.mjs",
-      "require": "./dist/TextSplitter.js"
-    },
-    "./Tool": {
-      "types": "./dist/Tool.d.mts",
-      "import": "./dist/Tool.mjs",
-      "require": "./dist/Tool.js"
-    },
-    "./readers/AssemblyAI": {
-      "types": "./dist/readers/AssemblyAI.d.mts",
-      "import": "./dist/readers/AssemblyAI.mjs",
-      "require": "./dist/readers/AssemblyAI.js"
-    },
-    "./readers/base": {
-      "types": "./dist/readers/base.d.mts",
-      "import": "./dist/readers/base.mjs",
-      "require": "./dist/readers/base.js"
-    },
-    "./readers/CSVReader": {
-      "types": "./dist/readers/CSVReader.d.mts",
-      "import": "./dist/readers/CSVReader.mjs",
-      "require": "./dist/readers/CSVReader.js"
-    },
-    "./readers/DocxReader": {
-      "types": "./dist/readers/DocxReader.d.mts",
-      "import": "./dist/readers/DocxReader.mjs",
-      "require": "./dist/readers/DocxReader.js"
-    },
-    "./readers/HTMLReader": {
-      "types": "./dist/readers/HTMLReader.d.mts",
-      "import": "./dist/readers/HTMLReader.mjs",
-      "require": "./dist/readers/HTMLReader.js"
-    },
-    "./readers/ImageReader": {
-      "types": "./dist/readers/ImageReader.d.mts",
-      "import": "./dist/readers/ImageReader.mjs",
-      "require": "./dist/readers/ImageReader.js"
-    },
-    "./readers/MarkdownReader": {
-      "types": "./dist/readers/MarkdownReader.d.mts",
-      "import": "./dist/readers/MarkdownReader.mjs",
-      "require": "./dist/readers/MarkdownReader.js"
-    },
-    "./readers/NotionReader": {
-      "types": "./dist/readers/NotionReader.d.mts",
-      "import": "./dist/readers/NotionReader.mjs",
-      "require": "./dist/readers/NotionReader.js"
-    },
-    "./readers/PDFReader": {
-      "types": "./dist/readers/PDFReader.d.mts",
-      "import": "./dist/readers/PDFReader.mjs",
-      "require": "./dist/readers/PDFReader.js"
-    },
-    "./readers/SimpleDirectoryReader": {
-      "types": "./dist/readers/SimpleDirectoryReader.d.mts",
-      "import": "./dist/readers/SimpleDirectoryReader.mjs",
-      "require": "./dist/readers/SimpleDirectoryReader.js"
-    },
-    "./readers/SimpleMongoReader": {
-      "types": "./dist/readers/SimpleMongoReader.d.mts",
-      "import": "./dist/readers/SimpleMongoReader.mjs",
-      "require": "./dist/readers/SimpleMongoReader.js"
     }
   },
   "files": [
-    "dist",
-    "examples",
-    "src",
-    "types",
-    "CHANGELOG.md"
+    "**"
   ],
   "repository": {
     "type": "git",
@@ -208,6 +76,10 @@
     "lint": "eslint .",
     "test": "jest",
     "build": "bunchee",
+    "postbuild": "pnpm run copy && pnpm run modify-package-json",
+    "copy": "cp -r package.json CHANGELOG.md ../../README.md ../../LICENSE examples src dist/",
+    "modify-package-json": "node ./scripts/modify-package-json.mjs",
+    "prepublish": "pnpm run modify-package-json && echo \"please cd ./dist and run pnpm publish\" && exit 1",
     "dev": "bunchee -w",
     "circular-check": "madge --circular ./src/*.ts"
   }

packages/core/scripts/modify-package-json.mjs

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+/**
+ * This script is used to modify the package.json file in the dist folder
+ * so that it can be published to npm.
+ */
+import editJsonFile from "edit-json-file";
+import fs from "node:fs/promises";
+
+{
+  await fs.copyFile("./package.json", "./dist/package.json");
+  const file = editJsonFile("./dist/package.json");
+
+  file.unset("scripts");
+  file.unset("private");
+  await new Promise((resolve) => file.save(resolve));
+}
+{
+  const packageJson = await fs.readFile("./dist/package.json", "utf8");
+  const modifiedPackageJson = packageJson.replaceAll("./dist/", "./");
+  await fs.writeFile(
+    "./dist/package.json",
+    JSON.stringify(JSON.parse(modifiedPackageJson), null, 2),
+    "utf8",
+  );
+}

packages/core/src/constants.ts

@@ -6,6 +6,4 @@ export const DEFAULT_CHUNK_OVERLAP = 20;
 export const DEFAULT_CHUNK_OVERLAP_RATIO = 0.1;
 export const DEFAULT_SIMILARITY_TOP_K = 2;
 
-// NOTE: for text-embedding-ada-002
-export const DEFAULT_EMBEDDING_DIM = 1536;
 export const DEFAULT_PADDING = 5;

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -9,28 +9,55 @@ import {
 import { OpenAISession, getOpenAISession } from "../llm/openai";
 import { BaseEmbedding } from "./types";
 
-export enum OpenAIEmbeddingModelType {
-  TEXT_EMBED_ADA_002 = "text-embedding-ada-002",
-}
+export const ALL_OPENAI_EMBEDDING_MODELS = {
+  "text-embedding-ada-002": {
+    dimensions: 1536,
+    maxTokens: 8191,
+  },
+  "text-embedding-3-small": {
+    dimensions: 1536,
+    dimensionOptions: [512, 1536],
+    maxTokens: 8191,
+  },
+  "text-embedding-3-large": {
+    dimensions: 3072,
+    dimensionOptions: [256, 1024, 3072],
+    maxTokens: 8191,
+  },
+};
 
 export class OpenAIEmbedding extends BaseEmbedding {
-  model: OpenAIEmbeddingModelType | string;
+  /** embeddding model. defaults to "text-embedding-ada-002" */
+  model: string;
+  /** number of dimensions of the resulting vector, for models that support choosing fewer dimensions. undefined will default to model default */
+  dimensions: number | undefined;
 
   // OpenAI session params
+
+  /** api key */
   apiKey?: string = undefined;
+  /** maximum number of retries, default 10 */
   maxRetries: number;
+  /** timeout in ms, default 60 seconds  */
   timeout?: number;
+  /** other session options for OpenAI */
   additionalSessionOptions?: Omit<
     Partial<OpenAIClientOptions>,
     "apiKey" | "maxRetries" | "timeout"
   >;
 
+  /** session object */
   session: OpenAISession;
 
+  /**
+   * OpenAI Embedding
+   * @param init - initial parameters
+   */
   constructor(init?: Partial<OpenAIEmbedding> & { azure?: AzureOpenAIConfig }) {
     super();
 
-    this.model = OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002;
+    this.model = init?.model ?? "text-embedding-ada-002";
+    this.dimensions = init?.dimensions; // if no dimensions provided, will be undefined/not sent to OpenAI
 
     this.maxRetries = init?.maxRetries ?? 10;
     this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
@@ -76,6 +103,7 @@ export class OpenAIEmbedding extends BaseEmbedding {
   private async getOpenAIEmbedding(input: string) {
     const { data } = await this.session.openai.embeddings.create({
       model: this.model,
+      dimensions: this.dimensions, // only sent to OpenAI if set by user
       input,
     });
 

packages/core/src/embeddings/together.ts

@@ -1,16 +1,26 @@
 import { OpenAIEmbedding } from "./OpenAIEmbedding";
 
 export class TogetherEmbedding extends OpenAIEmbedding {
-  override model: string;
   constructor(init?: Partial<OpenAIEmbedding>) {
+    const {
+      apiKey = process.env.TOGETHER_API_KEY,
+      additionalSessionOptions = {},
+      model = "togethercomputer/m2-bert-80M-32k-retrieval",
+      ...rest
+    } = init ?? {};
+
+    if (!apiKey) {
+      throw new Error("Set Together Key in TOGETHER_API_KEY env variable"); // Tell user to set correct env variable, and not OPENAI_API_KEY
+    }
+
+    additionalSessionOptions.baseURL =
+      additionalSessionOptions.baseURL ?? "https://api.together.xyz/v1";
+
     super({
-      apiKey: process.env.TOGETHER_API_KEY,
-      ...init,
-      additionalSessionOptions: {
-        ...init?.additionalSessionOptions,
-        baseURL: "https://api.together.xyz/v1",
-      },
+      apiKey,
+      additionalSessionOptions,
+      model,
+      ...rest,
     });
-    this.model = init?.model ?? "togethercomputer/m2-bert-80M-32k-retrieval";
   }
 }

packages/core/src/llm/LLM.ts

@@ -41,14 +41,21 @@ import {
 export const GPT4_MODELS = {
   "gpt-4": { contextWindow: 8192 },
   "gpt-4-32k": { contextWindow: 32768 },
+  "gpt-4-32k-0613": { contextWindow: 32768 },
+  "gpt-4-turbo-preview": { contextWindow: 128000 },
   "gpt-4-1106-preview": { contextWindow: 128000 },
-  "gpt-4-vision-preview": { contextWindow: 8192 },
+  "gpt-4-0125-preview": { contextWindow: 128000 },
+  "gpt-4-vision-preview": { contextWindow: 128000 },
 };
 
+// NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
 export const GPT35_MODELS = {
   "gpt-3.5-turbo": { contextWindow: 4096 },
+  "gpt-3.5-turbo-0613": { contextWindow: 4096 },
   "gpt-3.5-turbo-16k": { contextWindow: 16384 },
+  "gpt-3.5-turbo-16k-0613": { contextWindow: 16384 },
   "gpt-3.5-turbo-1106": { contextWindow: 16384 },
+  "gpt-3.5-turbo-0125": { contextWindow: 16384 },
 };
 
 /**

packages/core/src/llm/azure.ts

@@ -17,6 +17,14 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
   },
   "gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
   "gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
+  "gpt-4-vision-preview": {
+    contextWindow: 128000,
+    openAIModel: "gpt-4-vision-preview",
+  },
+  "gpt-4-1106-preview": {
+    contextWindow: 128000,
+    openAIModel: "gpt-4-1106-preview",
+  },
 };
 
 const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
@@ -25,13 +33,29 @@ const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
     openAIModel: "text-embedding-ada-002",
     maxTokens: 8191,
   },
+  "text-embedding-3-small": {
+    dimensions: 1536,
+    dimensionOptions: [512, 1536],
+    openAIModel: "text-embedding-3-small",
+    maxTokens: 8191,
+  },
+  "text-embedding-3-large": {
+    dimensions: 3072,
+    dimensionOptions: [256, 1024, 3072],
+    openAIModel: "text-embedding-3-large",
+    maxTokens: 8191,
+  },
 };
 
 const ALL_AZURE_API_VERSIONS = [
   "2022-12-01",
   "2023-05-15",
-  "2023-06-01-preview",
-  "2023-07-01-preview",
+  "2023-03-15-preview", // retiring 2024-04-02
+  "2023-06-01-preview", // retiring 2024-04-02
+  "2023-07-01-preview", // retiring 2024-04-02
+  "2023-08-01-preview", // retiring 2024-04-02
+  "2023-09-01-preview",
+  "2023-12-01-preview",
 ];
 
 const DEFAULT_API_VERSION = "2023-05-15";

packages/core/src/llm/together.ts

@@ -2,13 +2,25 @@ import { OpenAI } from "./LLM";
 
 export class TogetherLLM extends OpenAI {
   constructor(init?: Partial<OpenAI>) {
+    const {
+      apiKey = process.env.TOGETHER_API_KEY,
+      additionalSessionOptions = {},
+      model = "togethercomputer/llama-2-7b-chat",
+      ...rest
+    } = init ?? {};
+
+    if (!apiKey) {
+      throw new Error("Set Together Key in TOGETHER_API_KEY env variable"); // Tell user to set correct env variable, and not OPENAI_API_KEY
+    }
+
+    additionalSessionOptions.baseURL =
+      additionalSessionOptions.baseURL ?? "https://api.together.xyz/v1";
+
     super({
-      ...init,
-      apiKey: process.env.TOGETHER_API_KEY,
-      additionalSessionOptions: {
-        ...init?.additionalSessionOptions,
-        baseURL: "https://api.together.xyz/v1",
-      },
+      apiKey,
+      additionalSessionOptions,
+      model,
+      ...rest,
     });
   }
 }

packages/core/src/storage/vectorStore/PGVectorStore.ts

@@ -20,6 +20,7 @@ export class PGVectorStore implements VectorStore {
   private schemaName: string = PGVECTOR_SCHEMA;
   private tableName: string = PGVECTOR_TABLE;
   private connectionString: string | undefined = undefined;
+  private dimensions: number = 1536;
 
   private db?: pg.Client;
 
@@ -38,15 +39,18 @@ export class PGVectorStore implements VectorStore {
    * @param {string} config.schemaName - The name of the schema (optional). Defaults to PGVECTOR_SCHEMA.
    * @param {string} config.tableName - The name of the table (optional). Defaults to PGVECTOR_TABLE.
    * @param {string} config.connectionString - The connection string (optional).
+   * @param {number} config.dimensions - The dimensions of the embedding model.
    */
   constructor(config?: {
     schemaName?: string;
     tableName?: string;
     connectionString?: string;
+    dimensions?: number;
   }) {
     this.schemaName = config?.schemaName ?? PGVECTOR_SCHEMA;
     this.tableName = config?.tableName ?? PGVECTOR_TABLE;
     this.connectionString = config?.connectionString;
+    this.dimensions = config?.dimensions ?? 1536;
   }
 
   /**
@@ -108,7 +112,7 @@ export class PGVectorStore implements VectorStore {
       collection VARCHAR,
       document TEXT,
       metadata JSONB DEFAULT '{}',
-      embeddings VECTOR(1536)
+      embeddings VECTOR(${this.dimensions})
     )`;
     await db.query(tbl);
 

packages/core/src/storage/vectorStore/QdrantVectorStore.ts

@@ -54,11 +54,9 @@ export class QdrantVectorStore implements VectorStore {
     apiKey,
     batchSize,
   }: QdrantParams) {
-    if (!client && (!url || !apiKey)) {
-      if (!url || !apiKey || !collectionName) {
-        throw new Error(
-          "QdrantVectorStore requires url, apiKey and collectionName",
-        );
+    if (!client && !url) {
+      if (!url || !collectionName) {
+        throw new Error("QdrantVectorStore requires url and collectionName");
       }
     }
 

packages/create-llama/CHANGELOG.md

@@ -1,5 +1,18 @@
 # create-llama
 
+## 0.0.18
+
+### Patch Changes
+
+- 88d3b41: fix packaging
+
+## 0.0.17
+
+### Patch Changes
+
+- fa17f7e: Add an option that allows the user to run the generated app
+- 9e5d8e1: Add an option to select a local PDF file as data source
+
 ## 0.0.16
 
 ### Patch Changes

packages/create-llama/create-app.ts

@@ -35,6 +35,7 @@ export async function createApp({
   vectorDb,
   externalPort,
   postInstallAction,
+  contextFile,
 }: InstallAppArgs): Promise<void> {
   const root = path.resolve(appPath);
 
@@ -77,6 +78,7 @@ export async function createApp({
     vectorDb,
     externalPort,
     postInstallAction,
+    contextFile,
   };
 
   if (frontend) {

packages/create-llama/helpers/index.ts

@@ -70,22 +70,32 @@ const copyTestData = async (
   engine?: TemplateEngine,
   openAiKey?: string,
   vectorDb?: TemplateVectorDB,
+  contextFile?: string,
   // eslint-disable-next-line max-params
 ) => {
   if (engine === "context") {
-    const srcPath = path.join(
-      __dirname,
-      "..",
-      "templates",
-      "components",
-      "data",
-    );
     const destPath = path.join(root, "data");
-    console.log(`\nCopying test data to ${cyan(destPath)}\n`);
-    await copy("**", destPath, {
-      parents: true,
-      cwd: srcPath,
-    });
+    if (contextFile) {
+      console.log(`\nCopying provided file to ${cyan(destPath)}\n`);
+      await fs.mkdir(destPath, { recursive: true });
+      await fs.copyFile(
+        contextFile,
+        path.join(destPath, path.basename(contextFile)),
+      );
+    } else {
+      const srcPath = path.join(
+        __dirname,
+        "..",
+        "templates",
+        "components",
+        "data",
+      );
+      console.log(`\nCopying test data to ${cyan(destPath)}\n`);
+      await copy("**", destPath, {
+        parents: true,
+        cwd: srcPath,
+      });
+    }
   }
 
   if (packageManager && engine === "context") {
@@ -168,6 +178,7 @@ export const installTemplate = async (
       props.engine,
       props.openAiKey,
       props.vectorDb,
+      props.contextFile,
     );
   } else {
     // this is a frontend for a full-stack app, create .env file with model information

packages/create-llama/helpers/types.ts

@@ -15,6 +15,7 @@ export interface InstallTemplateArgs {
   template: TemplateType;
   framework: TemplateFramework;
   engine: TemplateEngine;
+  contextFile?: string;
   ui: TemplateUI;
   eslint: boolean;
   customApiPath?: string;

packages/create-llama/index.ts

@@ -240,6 +240,7 @@ async function run(): Promise<void> {
     vectorDb: program.vectorDb,
     externalPort: program.externalPort,
     postInstallAction: program.postInstallAction,
+    contextFile: program.contextFile,
   });
   conf.set("preferences", preferences);
 

packages/create-llama/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.0.16",
+  "version": "0.0.18",
   "keywords": [
     "rag",
     "llamaindex",
@@ -17,7 +17,7 @@
     "create-llama": "./dist/index.js"
   },
   "files": [
-    "./dist/index.js",
+    "dist",
     "./templates"
   ],
   "scripts": {

packages/create-llama/questions.ts

@@ -1,7 +1,8 @@
+import { execSync } from "child_process";
 import ciInfo from "ci-info";
 import fs from "fs";
 import path from "path";
-import { blue, green } from "picocolors";
+import { blue, green, red } from "picocolors";
 import prompts from "prompts";
 import { InstallAppArgs } from "./create-app";
 import { TemplateFramework } from "./helpers";
@@ -9,6 +10,22 @@ import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant";
 import { getRepoRootFolders } from "./helpers/repo";
 
 export type QuestionArgs = Omit<InstallAppArgs, "appPath" | "packageManager">;
+const MACOS_FILE_SELECTION_SCRIPT = `
+osascript -l JavaScript -e '
+  a = Application.currentApplication();
+  a.includeStandardAdditions = true;
+  a.chooseFile({ withPrompt: "Please select a file to process:" }).toString()
+'`;
+
+const WINDOWS_FILE_SELECTION_SCRIPT = `
+Add-Type -AssemblyName System.Windows.Forms
+$openFileDialog = New-Object System.Windows.Forms.OpenFileDialog
+$openFileDialog.InitialDirectory = [Environment]::GetFolderPath('Desktop')
+$result = $openFileDialog.ShowDialog()
+if ($result -eq 'OK') {
+  $openFileDialog.FileName
+}
+`;
 
 const defaults: QuestionArgs = {
   template: "streaming",
@@ -55,6 +72,45 @@ const getVectorDbChoices = (framework: TemplateFramework) => {
   return displayedChoices;
 };
 
+const selectPDFFile = async () => {
+  // Popup to select a PDF file
+  try {
+    let selectedFilePath: string = "";
+    switch (process.platform) {
+      case "win32": // Windows
+        selectedFilePath = execSync(WINDOWS_FILE_SELECTION_SCRIPT, {
+          shell: "powershell.exe",
+        })
+          .toString()
+          .trim();
+        break;
+      case "darwin": // MacOS
+        selectedFilePath = execSync(MACOS_FILE_SELECTION_SCRIPT)
+          .toString()
+          .trim();
+        break;
+      default: // Unsupported OS
+        console.log(red("Unsupported OS error!"));
+        process.exit(1);
+    }
+    // Check is pdf file
+    if (!selectedFilePath.endsWith(".pdf")) {
+      console.log(
+        red("Unsupported file error! Please select a valid PDF file!"),
+      );
+      process.exit(1);
+    }
+    return selectedFilePath;
+  } catch (error) {
+    console.log(
+      red(
+        "Got error when trying to select file! Please try again or select other options.",
+      ),
+    );
+    process.exit(1);
+  }
+};
+
 export const onPromptState = (state: any) => {
   if (state.aborted) {
     // If we don't re-enable the terminal cursor before exiting
@@ -243,24 +299,40 @@ export const askQuestions = async (
     if (ciInfo.isCI) {
       program.engine = getPrefOrDefault("engine");
     } else {
-      const { engine } = await prompts(
+      let choices = [
+        {
+          title: "No data, just a simple chat",
+          value: "simple",
+        },
+        { title: "Use an example PDF", value: "exampleFile" },
+      ];
+      if (process.platform === "win32" || process.platform === "darwin") {
+        choices.push({ title: "Use a local PDF file", value: "localFile" });
+      }
+
+      const { dataSource } = await prompts(
         {
           type: "select",
-          name: "engine",
+          name: "dataSource",
           message: "Which data source would you like to use?",
-          choices: [
-            {
-              title: "No data, just a simple chat",
-              value: "simple",
-            },
-            { title: "Use an example PDF", value: "context" },
-          ],
+          choices: choices,
           initial: 1,
         },
         handlers,
       );
-      program.engine = engine;
-      preferences.engine = engine;
+      switch (dataSource) {
+        case "simple":
+          program.engine = "simple";
+          break;
+        case "exampleFile":
+          program.engine = "context";
+          break;
+        case "localFile":
+          program.engine = "context";
+          // If the user selected the "pdf" option, ask them to select a file
+          program.contextFile = await selectPDFFile();
+          break;
+      }
     }
     if (program.engine !== "simple" && !program.vectorDb) {
       if (ciInfo.isCI) {

packages/create-llama/templates/types/simple/express/gitignore

@@ -1,2 +1,3 @@
 # local env files
 .env
+node_modules/

packages/create-llama/templates/types/simple/express/package.json

@@ -14,9 +14,6 @@
     "express": "^4.18.2",
     "llamaindex": "0.0.37"
   },
-  "overrides": {
-    "chromadb": "1.7.3"
-  },
   "devDependencies": {
     "@types/cors": "^2.8.17",
     "@types/express": "^4.17.21",

packages/create-llama/templates/types/streaming/express/gitignore

@@ -1,2 +1,3 @@
 # local env files
 .env
+node_modules/

packages/create-llama/templates/types/streaming/express/package.json

@@ -15,9 +15,6 @@
     "express": "^4.18.2",
     "llamaindex": "0.0.37"
   },
-  "overrides": {
-    "chromadb": "1.7.3"
-  },
   "devDependencies": {
     "@types/cors": "^2.8.16",
     "@types/express": "^4.17.21",

packages/create-llama/templates/types/streaming/nextjs/package.json

@@ -27,9 +27,6 @@
     "supports-color": "^9.4.0",
     "tailwind-merge": "^2.1.0"
   },
-  "overrides": {
-    "chromadb": "1.7.3"
-  },
   "devDependencies": {
     "@types/node": "^20.10.3",
     "@types/react": "^18.2.42",