RELEASING: Releasing 2 package(s)

Releases:
  llamaindex@0.0.48
  create-llama@0.0.16

[skip ci]

.eslintrc.js

@@ -7,4 +7,8 @@ module.exports = {
       rootDir: ["apps/*/"],
     },
   },
+  rules: {
+    "max-params": ["error", 4],
+  },
+  ignorePatterns: ["dist/"],
 };

.github/workflows/e2e.yml

@@ -8,6 +8,9 @@ on:
       - ".github/workflows/e2e.yml"
     branches: [main]
 
+env:
+  POETRY_VERSION: "1.6.1"
+
 jobs:
   e2e:
     name: create-llama
@@ -16,10 +19,19 @@ jobs:
       fail-fast: true
       matrix:
         node-version: [18, 20]
+        python-version: ["3.11"]
         os: [macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
       - uses: pnpm/action-setup@v2
       - name: Setup Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4

.github/workflows/test.yml

@@ -18,3 +18,21 @@ jobs:
         run: pnpm install
       - name: Run tests
         run: pnpm run test
+  typecheck:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+      - name: Install dependencies
+        run: pnpm install
+      - name: Build
+        run: pnpm run build
+        working-directory: ./packages/core
+      - name: Run Type Check
+        run: pnpm run type-check

.gitignore

@@ -37,9 +37,7 @@ yarn-error.log*
 .vercel
 
 dist/
-
-# vs code
-.vscode/launch.json
+lib/
 
 .cache
 test-results/

.prettierignore

@@ -1,3 +1,4 @@
 apps/docs/i18n
 pnpm-lock.yaml
-
+lib/
+dist/

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Example",
+      "skipFiles": ["<node_internals>/**"],
+      "runtimeExecutable": "pnpm",
+      "cwd": "${workspaceFolder}/examples",
+      "runtimeArgs": ["ts-node", "${fileBasename}"]
+    }
+  ]
+}

README.md

@@ -1,5 +1,10 @@
 # LlamaIndex.TS
 
+[![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
+
 LlamaIndex is a data framework for your LLM application.
 
 Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.
@@ -101,7 +106,6 @@ const nextConfig = {
       ...config.resolve.alias,
       sharp$: false,
       "onnxruntime-node$": false,
-      mongodb$: false,
     };
     return config;
   },

apps/docs/.gitignore

@@ -7,6 +7,7 @@
 # Generated files
 .docusaurus
 .cache-loader
+lib
 
 # Misc
 .DS_Store

apps/docs/docs/modules/high_level/chat_engine.md

@@ -11,7 +11,16 @@ const retriever = index.asRetriever();
 const chatEngine = new ContextChatEngine({ retriever });
 
 // start chatting
-const response = await chatEngine.chat(query);
+const response = await chatEngine.chat({ message: query });
+```
+
+The `chat` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await chatEngine.chat({ message: query, stream: true });
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
 ```
 
 ## Api References

apps/docs/docs/modules/high_level/query_engine.md

@@ -8,7 +8,16 @@ A query engine wraps a `Retriever` and a `ResponseSynthesizer` into a pipeline,
 
 ```typescript
 const queryEngine = index.asQueryEngine();
-const response = await queryEngine.query("query string");
+const response = await queryEngine.query({ query: "query string" });
+```
+
+The `query` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await queryEngine.query({ query: "query string", stream: true });
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
 ```
 
 ## Sub Question Query Engine

apps/docs/docs/modules/low_level/response_synthesizer.md

@@ -35,13 +35,26 @@ const nodesWithScore: NodeWithScore[] = [
   },
 ];
 
-const response = await responseSynthesizer.synthesize(
-  "What age am I?",
+const response = await responseSynthesizer.synthesize({
+  query: "What age am I?",
   nodesWithScore,
-);
+});
 console.log(response.response);
 ```
 
+The `synthesize` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await responseSynthesizer.synthesize({
+  query: "What age am I?",
+  nodesWithScore,
+  stream: true,
+});
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
+```
+
 ## API Reference
 
 - [ResponseSynthesizer](../../api/classes/ResponseSynthesizer.md)

apps/docs/docs/modules/observability/_category_.yml

@@ -0,0 +1 @@
+label: Observability

apps/docs/docs/modules/observability/index.md

@@ -0,0 +1,35 @@
+# Observability
+
+LlamaIndex provides **one-click observability** 🔭 to allow you to build principled LLM applications in a production setting.
+
+A key requirement for principled development of LLM applications over your data (RAG systems, agents) is being able to observe, debug, and evaluate
+your system - both as a whole and for each component.
+
+This feature allows you to seamlessly integrate the LlamaIndex library with powerful observability/evaluation tools offered by our partners.
+Configure a variable once, and you'll be able to do things like the following:
+
+- View LLM/prompt inputs/outputs
+- Ensure that the outputs of any component (LLMs, embeddings) are performing as expected
+- View call traces for both indexing and querying
+
+Each provider has similarities and differences. Take a look below for the full set of guides for each one!
+
+## OpenLLMetry
+
+[OpenLLMetry](https://github.com/traceloop/openllmetry-js) is an open-source project based on OpenTelemetry for tracing and monitoring
+LLM applications. It connects to [all major observability platforms](https://www.traceloop.com/docs/openllmetry/integrations/introduction) and installs in minutes.
+
+### Usage Pattern
+
+```bash
+npm install @traceloop/node-server-sdk
+```
+
+```js
+import * as traceloop from "@traceloop/node-server-sdk";
+
+traceloop.initialize({
+  apiKey: process.env.TRACELOOP_API_KEY,
+  disableBatch: true,
+});
+```

apps/docs/package.json

@@ -28,8 +28,10 @@
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "2.4.3",
+    "@docusaurus/theme-classic": "^2.4.3",
     "@docusaurus/types": "^2.4.3",
     "@tsconfig/docusaurus": "^2.0.1",
+    "@types/node": "^18.19.6",
     "docusaurus-plugin-typedoc": "^0.19.2",
     "typedoc": "^0.24.8",
     "typedoc-plugin-markdown": "^3.16.0",

apps/docs/tsconfig.json

@@ -1,7 +1,11 @@
 {
   // This file is not used in compilation. It is here just for a nice editor experience.
-  "extends": "@tsconfig/docusaurus/tsconfig.json",
+  "extends": "./node_modules/@tsconfig/docusaurus/tsconfig.json",
   "compilerOptions": {
-    "baseUrl": "."
+    "baseUrl": ".",
+    "composite": true,
+    "incremental": true,
+    "outDir": "./lib",
+    "tsBuildInfoFile": "./lib/.tsbuildinfo"
   }
 }

examples/anthropic.ts

@@ -2,13 +2,15 @@ import { Anthropic } from "llamaindex";
 
 (async () => {
   const anthropic = new Anthropic();
-  const result = await anthropic.chat([
-    { content: "You want to talk in rhymes.", role: "system" },
-    {
-      content:
-        "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
-      role: "user",
-    },
-  ]);
+  const result = await anthropic.chat({
+    messages: [
+      { content: "You want to talk in rhymes.", role: "system" },
+      {
+        content:
+          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
+        role: "user",
+      },
+    ],
+  });
   console.log(result);
 })();

examples/astradb/load.ts

@@ -10,7 +10,7 @@ const collectionName = "movie_reviews";
 async function main() {
   try {
     const reader = new PapaCSVReader(false);
-    const docs = await reader.loadData("astradb/data/movie_reviews.csv");
+    const docs = await reader.loadData("../data/movie_reviews.csv");
 
     const astraVS = new AstraDBVectorStore();
     await astraVS.create(collectionName, {

examples/astradb/query.ts

@@ -18,7 +18,9 @@ async function main() {
 
     const queryEngine = await index.asQueryEngine({ retriever });
 
-    const results = await queryEngine.query("What is the best reviewed movie?");
+    const results = await queryEngine.query({
+      query: "What is the best reviewed movie?",
+    });
 
     console.log(results.response);
   } catch (e) {

examples/chatEngine.ts

@@ -25,8 +25,11 @@ async function main() {
 
   while (true) {
     const query = await rl.question("Query: ");
-    const response = await chatEngine.chat(query);
-    console.log(response.toString());
+    const stream = await chatEngine.chat({ message: query, stream: true });
+    console.log();
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.response);
+    }
   }
 }
 

examples/chatHistory.ts

@@ -0,0 +1,34 @@
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+
+import { OpenAI, SimpleChatEngine, SummaryChatHistory } from "llamaindex";
+
+async function main() {
+  // Set maxTokens to 75% of the context window size of 4096
+  // This will trigger the summarizer once the chat history reaches 25% of the context window size (1024 tokens)
+  const llm = new OpenAI({ model: "gpt-3.5-turbo", maxTokens: 4096 * 0.75 });
+  const chatHistory = new SummaryChatHistory({ llm });
+  const chatEngine = new SimpleChatEngine({ llm });
+  const rl = readline.createInterface({ input, output });
+
+  while (true) {
+    const query = await rl.question("Query: ");
+    const stream = await chatEngine.chat({
+      message: query,
+      chatHistory,
+      stream: true,
+    });
+    if (chatHistory.getLastSummary()) {
+      // Print the summary of the conversation so far that is produced by the SummaryChatHistory
+      console.log(`Summary: ${chatHistory.getLastSummary()?.content}`);
+    }
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.response);
+    }
+    console.log();
+  }
+}
+
+main().catch(console.error);

examples/chromadb/README.md

@@ -0,0 +1,13 @@
+# Chroma Vector Store Example
+
+How to run `examples/chromadb/test.ts`:
+
+Export your OpenAI API Key using `export OPEN_API_KEY=insert your api key here`
+
+If you haven't installed chromadb, run `pip install chromadb`. Start the server using `chroma run`.
+
+Now, open a new terminal window and inside `examples`, run `pnpx ts-node chromadb/test.ts`.
+
+Here's the output for the input query `Tell me about Godfrey Cheshire's rating of La Sapienza.`:
+
+`Godfrey Cheshire gave La Sapienza a rating of 4 out of 4, describing it as fresh and the most astonishing and important movie to emerge from France in quite some time.`

examples/chromadb/test.ts

@@ -0,0 +1,40 @@
+import {
+  ChromaVectorStore,
+  PapaCSVReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+
+const collectionName = "movie_reviews";
+
+async function main() {
+  const sourceFile: string = "./data/movie_reviews.csv";
+
+  try {
+    console.log(`Loading data from ${sourceFile}`);
+    const reader = new PapaCSVReader(false, ", ", "\n", {
+      header: true,
+    });
+    const docs = await reader.loadData(sourceFile);
+
+    console.log("Creating ChromaDB vector store");
+    const chromaVS = new ChromaVectorStore({ collectionName });
+    const ctx = await storageContextFromDefaults({ vectorStore: chromaVS });
+
+    console.log("Embedding documents and adding to index");
+    const index = await VectorStoreIndex.fromDocuments(docs, {
+      storageContext: ctx,
+    });
+
+    console.log("Querying index");
+    const queryEngine = index.asQueryEngine();
+    const response = await queryEngine.query({
+      query: "Tell me about Godfrey Cheshire's rating of La Sapienza.",
+    });
+    console.log(response.toString());
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+main();

examples/data/movie_reviews.csv

@@ -1,101 +1,101 @@
-title,reviewid,creationdate,criticname,originalscore,reviewstate,reviewtext
-Beavers,1145982,2003-05-23,Ivan M. Lincoln,3.5/4,fresh,"Timed to be just long enough for most youngsters' brief attention spans -- and it's packed with plenty of interesting activity, both on land and under the water."
-Blood Mask,1636744,2007-06-02,The Foywonder,1/5,rotten,"It doesn't matter if a movie costs 300 million or only 300 dollars; good is good and bad is bad, and Bloodmask: The Possession of Nicole Lameroux is just plain bad."
-City Hunter: Shinjuku Private Eyes,2590987,2019-05-28,Reuben Baron,,fresh,"The choreography is so precise and lifelike at points one might wonder whether the movie was rotoscoped, but no live-action reference footage was used. The quality is due to the skill of the animators and Kodama's love for professional wrestling."
-City Hunter: Shinjuku Private Eyes,2558908,2019-02-14,Matt Schley,2.5/5,rotten,The film's out-of-touch attempts at humor may find them hunting for the reason the franchise was so popular in the first place.
-Dangerous Men,2504681,2018-08-29,Pat Padua,,fresh,Its clumsy determination is endearing and sometimes wildly entertaining
-Dangerous Men,2299284,2015-12-13,Eric Melin,4/5,fresh,"With every new minute, there's another head-scratching choice that's bound to elicit some amazing out-loud responses, so this feels like a true party flick."
-Dangerous Men,2295858,2015-11-22,Matt Donato,7/10,fresh,"Emotionless reaction shots, zero characterization, guns that have absolutely no special effects when blasted - Dangerous Men is rare winning dish from a one star restaurant."
-Dangerous Men,2295338,2015-11-19,Peter Keough,0.5/4,rotten,"Conceivably, it could serve as a primer for students on how not to make a movie, and perhaps as a deconstruction of filmic conventions for the more theoretical minded."
-Dangerous Men,2294641,2015-11-16,Jason Wilson,3/10,rotten,"If you're not a fan of garbage cinema, even for the fun of it, Dangerous Men is best to be avoided."
-Dangerous Men,2294129,2015-11-12,Soren Andersen,0/4,rotten,"""Dangerous Men,"" the picture's production notes inform, took 26 years to reach the big screen. After having seen it, I wonder: What was the rush?"
-Dangerous Men,2293902,2015-11-12,Maitland McDonagh,,rotten,Will entertain some viewers and infuriate others with its clunky mix of feminist fury and awkward action sequences.
-Dangerous Men,2293900,2015-11-12,Marjorie Baumgarten,1.5/5,rotten,"This is a bad movie, but one that awakens your senses every so often with flashes of originality and abundant self-belief."
-Dangerous Men,2293815,2015-11-12,Katie Rife,B+,fresh,"Ridiculous, artless, and wildly entertaining, Dangerous Men is more than the sum of its fascinatingly misguided parts, although it will take a special sort of moviegoer to truly appreciate (or endure, depending on your perspective) its charms."
-Dangerous Men,2293605,2015-11-11,Amy Nicholson,C,fresh,To sit through it feels like honoring the dreamers of the world who at least get shit done. Is it terrible? Of course. Is there belly-dancing? Duh.
-Small Town Wisconsin,102711819,2022-07-22,Peter Gray,,fresh,Small Town Wisconsin could hit some home truths for viewers, and though being faced with the truth isn’t always pleasant, it feels necessary in growing towards a happier fruition.
-Small Town Wisconsin,102711545,2022-07-22,Tim Grierson,,fresh,"This low-key drama has lovely interludes and some nicely understated performances, although director Niels Mueller doesn’t glean too many new insights from Jason Naczek’s familiar story..."
-Small Town Wisconsin,102700937,2022-06-16,Sumner Forbes,8.5/10,fresh,"Small Town Wisconsin is a success in almost every regard, and if you can see over the legions of cheeseheads in the rows ahead of you, it shouldn’t be missed."
-Small Town Wisconsin,102699897,2022-06-14,Tara McNamara,3/5,fresh,Just like Wayne, Small Town Wisconsin has flaws, but the poignancy of the story will stick with you for a long time.
-Small Town Wisconsin,102698744,2022-06-10,Rob Thomas,3/4,fresh,It’s a movie with its heart in the right place, and does both small town and big city Wisconsin proud.
-Small Town Wisconsin,102698639,2022-06-10,Todd Jorgenson,,rotten,Despite some intriguing character dynamics and performances that generate sympathy for this fractured family, the film stumbles when it veers into melodrama without the narrative dexterity to tackle its weightier ambitions.
-Small Town Wisconsin,102698482,2022-06-10,Jackie K. Cooper,7/10,fresh,This is the kind of movie that draws you so deeply into its story you are reluctant to let it end.
-Small Town Wisconsin,102698164,2022-06-09,Glenn Kenny,,fresh,"Mueller’s direction is patient and sensitive, the cast is accomplished and committed, and the picture’s comedic aspects sometimes earn a chuckle."
-Small Town Wisconsin,102697854,2022-06-08,Brian Orndorf,B+,fresh,Naczek isn't interested in making a soap opera with this examination of fallibility, going somewhere much more authentic when exploring character aches and pains.
-Small Town Wisconsin,102695788,2022-06-02,Eddie Harrison,4/5,fresh,…a warm-hearted story of everyday life that’s easy to recommend for those who like films about people rather than portals and vortexes…
-Small Town Wisconsin,102695250,2022-05-31,Laura Clifford,C,rotten,Debuting screenwriter Jason Naczek has concocted a manchild redemption story using metaphors as heavy as a hammer and a fairy godmother who makes everything alright with a seeming flip of the switch.
-Small Town Wisconsin,2733251,2020-10-12,Jared Mobarak,B,fresh,Small Town Wisconsin is always proving itself to be more than its familiar premise thanks to Naczek's ability to infuse a lot more drama into the mix than one custody battle.
-Tejano,2564925,2019-03-07,Joe Friar,3/4,fresh,The story of a South Texas ranch hand who gets mixed up with a Mexican cartel moves with pulse-pounding velocity and features top performances from a talented cast of actors with Texas roots.
-Tejano,2557738,2019-02-12,Cary Darling,4/5,fresh,"An entertaining blast of Texas noir that nods toward the work of the Coen brothers, Quentin Tarantino and fellow Austinite Greg Kwedar's 2016 low-budget thriller ""Transpecos"" as well as ""Breaking Bad."""
-Tejano,2547231,2019-01-10,Danielle White,3/5,fresh,The story itself slithers with twists and turns and unexpected betrayals. It's almost ridiculous how many characters die in this film.
-Tejano,2530119,2018-11-08,Chris Salce,9/10,fresh,"Tejano is one of those films that can be described as a hidden gem as it sneaks under the radar and will have you talking, telling your friends about it, and wanting to watch it again."
-Death of a Salesman,2770637,2021-02-23,Michael Dougan,,fresh,"Miller has taken a small, intimate tale and expanded it into a treatise on larger themes, primarily the abuse of the American Dream."
-Death of a Salesman,1950734,2011-01-02,Randy White,5/5,fresh,A classic American tragedy.
-Death of a Salesman,1422415,2005-08-04,Jules Brenner,4/5,fresh,
-Death of a Salesman,1409415,2005-07-05,Emanuel Levy,3/5,fresh,
-Death of a Salesman,839546,2003-02-06,Frederic and Mary Ann Brussat,,fresh,"Death of a Salesman, directed by Volker Schlondorff, draws out the multiple meanings of this Pulitzer Prize-winning play by Arthur Miller about change, family and fatherhood, work and love."
-Death of a Salesman,788410,2002-09-29,Dan Lybarger,4/5,fresh,"Schlndorff's artificial settings and some amazing performances help keep this from looking like a typical ""filmed play."""
-Death of a Salesman,751951,2002-08-08,Cory Cheney,4/5,fresh,
-Death of a Salesman,743794,2002-07-26,Bob Grimm,5/5,fresh,
-Death of a Salesman,743291,2002-07-26,Scott Weinberg,5/5,fresh,They MAKE you watch it in English class for a good reason!
-Sahara,1137710,2003-05-13,Dragan Antulov,5/10,fresh,
-The Debt,2628192,2019-09-20,Diego Batlle,,fresh,A Bresson-esque movie that is always enigmatic. [Full Review in Spanish]
-The Debt,2627988,2019-09-20,Gaspar Zimerman,,fresh,The story [Director Gustavo Fontán] tells is an excuse to give way to the exploration of feelings and sensations that avoid verbality. [Full review in Spanish]
-Peppermint Candy,2725008,2020-09-16,A.S. Hamrah,,fresh,"South Korean political history of the previous twenty years, Peppermint Candy is not tempered by its hysterical edge, which adds unpredictable violence to its vignettes of romantic, domestic, and business failure."
-Peppermint Candy,2541271,2018-12-16,Panos Kotzathanasis,,fresh,"Lee Chang-dong presents a melodrama that stands apart from the plethora of similar productions due to its intense political element, because it doesn't lose its seriousness at any point and because it doesn't become hyperbolic in his effort to draw tears"
-Peppermint Candy,1883708,2010-05-11,Anton Bitel,,fresh,"This is Korea's millennial elegy, filtering its search for times past through a confection no less bittersweet than Proust's madeleine."
-Peppermint Candy,1706014,2008-01-29,Beth Accomando,9/10,fresh,The film offers a heartbreaking drama told in reverse chronology and spanning twenty years in both the life of the main character and the political history of Korea.
-Peppermint Candy,1231988,2003-12-22,Greg Muskewitz,2/5,rotten,
-Peppermint Candy,1187104,2003-08-14,Joshua Tanzer,4/4,fresh,"It's a story about the original sin of a nation as well as one character. There has rarely been a better film made, ever"
-Prison Girls,2475348,2018-05-03,Roger Ebert,,rotten,Prison Girls didn't have a lot of prison sets because it was a big-budget exploitation movie. Maybe.
-Gimme the Power,2575688,2019-04-09,Afroxander,,fresh,"Rubio's film shows ambition where none is required, making Gimme the Power a lot like Molotov's music: politically engaged without having to take itself too seriously."
-Paa,2673089,2020-02-27,Nikhat Kazmi,3.5/5,fresh,"The film, which peters off into vague sub-plots about slum redevelopment and unwarranted media-bashing in the first half, suddenly picks up and scales new heights in the second half."
-Paa,2578129,2019-04-17,Shubhra Gupta,2/5,rotten,"Disappointingly, Paa is not as out-of-the-box as it could have been."
-Paa,2429810,2017-10-24,Anil Sinanan,3/5,rotten,Will Auro survive to know his Pa and reunite his parents? Forget about the disease: this is a vanity vehicle designed to showcase the Big B's versatility.
-Paa,1860476,2009-12-14,Frank Lovece,,rotten,This would-be tearjerker without the musical numbers of typical Bollywood fare is for die-hard Amitabh Bachchan fans only.
-Paa,1860473,2009-12-14,David Chute,,fresh,"The film owes much of its interest to the alertness and sincerity of the younger Bachchan and the luminous Vidya Balan as the anguished parents, and to the soft wash of the tasteful playback songs supplied by Ilaiyaraaja."
-Paa,1858964,2009-12-05,Avi Offer,5.85/10,rotten,"Well-acted, funny and occasionally witty with terrific make-up design. However, it's often convoluted, awkwardly paced and too uneven as a whole."
-Paa,1858853,2009-12-04,Frank Lovece,,fresh,"A would-be tearjerker without the singing-dancing musical numbers of typical Bollywood fare seen in the U.S., the lackluster Paa is for die-hard Amitabh Bachchan fans only%u2014of which there is no small number."
-Paa,1858816,2009-12-04,Rachel Saltz,3/5,fresh,Odd and sometimes oddly affecting.
-Alraune (A Daughter of Destiny) (Mandrake) (Unholy Love),2835964,2021-10-30,Erich Hellmund-Waldow,,fresh,"The acting is not only artistic, it is also as realistic as can be possible in such a film."
-Alraune (A Daughter of Destiny) (Mandrake) (Unholy Love),2357086,2016-10-17,C. Hooper Trask,,fresh,"Aimed straight for the gooseflesh, it strikes directly into the centre of the target."
-Toorbos,2760593,2021-01-29,Neil Young,,fresh,Built around a luminous and intriguing central performance by dancer-actor Elani Dekker.
-Toorbos,2752827,2020-12-21,Guy Lodge,,fresh,"A satisfying marriage of folky period romance and environmental parable from the misty, mossy depths of South Africa's Knysna forest region..."
-Connors' War,1555113,2006-11-09,David Nusair,1.5/4,rotten,"...although Criss does show some potential as a performer, his efforts to step into the shoes of a blind character are laughable."
-Connors' War,1539106,2006-09-19,Scott Weinberg,2/5,rotten,"Standard cable fodder all the way, with only a few solid action scenes and maybe one colorful performance in the whole thing."
-Born to Kill,2710947,2020-08-05,Mike Massie,10/10,fresh,"One of the most acerbic of all films noir, boasting essentially no redeemable characters (or a wealth of deliciously evil villains) while also being utterly enthralling."
-Born to Kill,2340106,2016-07-15,David Nusair,3/4,fresh,...a fairly typical film-noir premise that's employed to watchable yet entirely unmemorable effect by Robert Wise...
-Born to Kill,1507021,2006-05-16,Nick Schager,B,fresh,Competent if slightly too tame for a supposedly sleazy story.
-Born to Kill,1501617,2006-05-01,Fernando F. Croce,,fresh,"The usually meek Robert Wise trades his chameleonic tastefulness for full-on, jazzy misanthropy in this nasty melodrama."
-Born to Kill,1433953,2005-09-09,Jeffrey M. Anderson,3/4,fresh,"Hard to watch, but effective and alluring nonetheless."
-Born to Kill,1123980,2003-04-02,Dennis Schwartz,C,rotten,A revolting B film noir...
-The Soong Sisters,1402087,2005-06-15,Emanuel Levy,3/5,fresh,
-La Sapienza,102772380,2023-01-24,Vadim Rizov,,fresh,"Sapienza is a pretty lovely film. Symmetricities are everywhere, starting with that opening architectural showreel, which deliberately avoids perfect symmetricity..."
-La Sapienza,2767839,2021-02-14,Dustin Chang,,fresh,Their sincere expression of these thoughts rings true and melts away its artificiality in its presentation soon enough. This is the beauty of La Sapienza and Green films in general.
-La Sapienza,2598336,2019-06-18,C.J. Prince,,fresh,"It's a nice entry point into a peculiar cinematic universe, and those willing to open themselves to it will find a lot to enjoy."
-La Sapienza,2503963,2018-08-28,Charles Mudede,,fresh,"If architecture aspires to the condition of music, the acting in La Sapienza aspires to the condition of architecture. You will love the ending of this very original and elegant and arty work."
-La Sapienza,2314368,2016-03-12,Forrest Cardamenis,B,fresh,This startling architectural juxtaposition feels like a wake-up call.
-La Sapienza,2275677,2015-08-03,Nicole Armour,,fresh,"While Green's film is dense with historical fact and theory, it's not averse to plumbing life's mysteries. Suffused with warmth, it expresses a potent admiration for human striving and accomplishment."
-La Sapienza,2273804,2015-07-23,Norman Wilner,2/5,rotten,"The uncomplicated narrative resists stylization; Green's presentation turns everyone into mannequins, rendering their emotions theoretical. That may well be his point, but it didn't work for me."
-La Sapienza,2269287,2015-06-26,Sam Lubell,,fresh,"On the surface, writer-director Eugne Green's film ""La Sapienza"" is slow, strange and awkward - but stick with it and it may win you over."
-La Sapienza,2265997,2015-06-05,Rob Garratt,4/5,fresh,"Layered with reels of swirling shots of Rome's most beautiful buildings -- all crucially shot from the ground upwards, staring at the heavens-- La Sapienza is visually stunning."
-La Sapienza,2265990,2015-06-05,Boyd van Hoeij,,fresh,"The Sapience juxtaposes insights on how people are emotionally connected with ruminations on the buildings and spaces through which they move, in which they live and, in Alexandre's case, which they also create."
-La Sapienza,2265989,2015-06-05,Robert Horton,3/4,fresh,"If you can groove into this non-realistic mode, the film casts a spell."
-La Sapienza,2265790,2015-06-04,Tom Keogh,3.5/4,fresh,A beautiful space for people and light.
-La Sapienza,2255621,2015-04-09,Wesley Morris,,rotten,This kind of formalism needs to do more than walk through classical wonders. It should want to create cinema that can stand near or beside them. This movie defensively consecrates what's already there. You don't need a film to do that.
-La Sapienza,2255195,2015-04-08,Scott Foundas,,fresh,"An exquisite rumination on life, love and art that tickles the heart and mind in equal measure."
-La Sapienza,2252858,2015-03-23,Richard Brody,,fresh,"Green's richly textured, painterly images fuse with the story to evoke the essence of humane urbanity and the relationships that it fosters, whether educational, familial, or erotic."
-La Sapienza,2252553,2015-03-20,Ignatiy Vishnevetsky,B+,fresh,"Green doesn't so much use his characters as mouthpieces as emotionally invest them in art, turning opinions into feelings."
-La Sapienza,2252541,2015-03-20,Godfrey Cheshire,4/4,fresh,"""La Sapienza"" strikes this reviewer as easily the most astonishing and important movie to emerge from France in quite some time."
-La Sapienza,2252452,2015-03-19,A.O. Scott,,fresh,The movie is an unapologetically rarefied undertaking and at the same time a gracious and inviting film.
-La Sapienza,2252301,2015-03-19,David Noh,,rotten,"Pretentious, stuffy and slow. There's some beautiful scenery here but oh, what you must put up with to earn it!"
-La Sapienza,2252028,2015-03-18,Noel Murray,3/5,fresh,"While La Sapienza is unsatisfying as drama, it's frequently beautiful just as a tour through architecturally significant Italian buildings."
-La Sapienza,2251985,2015-03-17,David Ehrlich,3/5,fresh,La Sapienza  alternately feels like a self-reflexive love story or a haunted history lesson -- its best scenes play like both.
-La Sapienza,2251926,2015-03-17,Zachary Wigon,,fresh,A picture that balances heart and mind with nuance.
-La Sapienza,2251650,2015-03-14,Harvey S. Karten,B+,fresh,"As in ""Who's Afraid of Virginia Woolf,"" both the younger couple and their older mentors are changed from a relationship."
-La Sapienza,2250991,2015-03-12,Ben Sachs,,fresh,"This recalls Manoel de Oliveira and Eric Rohmer in its poker-faced style, deliberately archaic storytelling, and magisterial epiphanies."
-La Sapienza,2225361,2014-09-28,Donald J. Levit,,fresh,"Although a love-fiction crossed with documentary lecture and superb Raphael O'Byrne cinematography, 'La Sapienza' is as close as celluloid can approach to architecture."
-La Sapienza,2222032,2014-09-10,Carson Lund,3/4,fresh,"Eugne Green's mannered direction doesn't work for every situation it's homogenously applied to, but at its most effective it inspires an enhanced sensitivity to the import of every gesture, visual or verbal."
-Uncle Tom,2713732,2020-08-14,Megan Basham,,fresh,Uncle Tom suffers from an overreliance on pundits. Its most compelling insights come from people who've never been quoted in a Twitter or Facebook battle.
-Uncle Tom,2706229,2020-07-19,Matthew Pejkovic,4/5,fresh,"An incredibly relevant and insightful documentary that delves into the past, present, and future of the black American conservative movement."
+title,reviewid,creationdate,criticname,originalscore,reviewstate,reviewtext
+Beavers,1145982,2003-05-23,Ivan M. Lincoln,3.5/4,fresh,"Timed to be just long enough for most youngsters' brief attention spans -- and it's packed with plenty of interesting activity, both on land and under the water."
+Blood Mask,1636744,2007-06-02,The Foywonder,1/5,rotten,"It doesn't matter if a movie costs 300 million or only 300 dollars; good is good and bad is bad, and Bloodmask: The Possession of Nicole Lameroux is just plain bad."
+City Hunter: Shinjuku Private Eyes,2590987,2019-05-28,Reuben Baron,,fresh,"The choreography is so precise and lifelike at points one might wonder whether the movie was rotoscoped, but no live-action reference footage was used. The quality is due to the skill of the animators and Kodama's love for professional wrestling."
+City Hunter: Shinjuku Private Eyes,2558908,2019-02-14,Matt Schley,2.5/5,rotten,The film's out-of-touch attempts at humor may find them hunting for the reason the franchise was so popular in the first place.
+Dangerous Men,2504681,2018-08-29,Pat Padua,,fresh,Its clumsy determination is endearing and sometimes wildly entertaining
+Dangerous Men,2299284,2015-12-13,Eric Melin,4/5,fresh,"With every new minute, there's another head-scratching choice that's bound to elicit some amazing out-loud responses, so this feels like a true party flick."
+Dangerous Men,2295858,2015-11-22,Matt Donato,7/10,fresh,"Emotionless reaction shots, zero characterization, guns that have absolutely no special effects when blasted - Dangerous Men is rare winning dish from a one star restaurant."
+Dangerous Men,2295338,2015-11-19,Peter Keough,0.5/4,rotten,"Conceivably, it could serve as a primer for students on how not to make a movie, and perhaps as a deconstruction of filmic conventions for the more theoretical minded."
+Dangerous Men,2294641,2015-11-16,Jason Wilson,3/10,rotten,"If you're not a fan of garbage cinema, even for the fun of it, Dangerous Men is best to be avoided."
+Dangerous Men,2294129,2015-11-12,Soren Andersen,0/4,rotten,"""Dangerous Men,"" the picture's production notes inform, took 26 years to reach the big screen. After having seen it, I wonder: What was the rush?"
+Dangerous Men,2293902,2015-11-12,Maitland McDonagh,,rotten,Will entertain some viewers and infuriate others with its clunky mix of feminist fury and awkward action sequences.
+Dangerous Men,2293900,2015-11-12,Marjorie Baumgarten,1.5/5,rotten,"This is a bad movie, but one that awakens your senses every so often with flashes of originality and abundant self-belief."
+Dangerous Men,2293815,2015-11-12,Katie Rife,B+,fresh,"Ridiculous, artless, and wildly entertaining, Dangerous Men is more than the sum of its fascinatingly misguided parts, although it will take a special sort of moviegoer to truly appreciate (or endure, depending on your perspective) its charms."
+Dangerous Men,2293605,2015-11-11,Amy Nicholson,C,fresh,To sit through it feels like honoring the dreamers of the world who at least get shit done. Is it terrible? Of course. Is there belly-dancing? Duh.
+Small Town Wisconsin,102711819,2022-07-22,Peter Gray,,fresh,Small Town Wisconsin could hit some home truths for viewers, and though being faced with the truth isn’t always pleasant, it feels necessary in growing towards a happier fruition.
+Small Town Wisconsin,102711545,2022-07-22,Tim Grierson,,fresh,"This low-key drama has lovely interludes and some nicely understated performances, although director Niels Mueller doesn’t glean too many new insights from Jason Naczek’s familiar story..."
+Small Town Wisconsin,102700937,2022-06-16,Sumner Forbes,8.5/10,fresh,"Small Town Wisconsin is a success in almost every regard, and if you can see over the legions of cheeseheads in the rows ahead of you, it shouldn’t be missed."
+Small Town Wisconsin,102699897,2022-06-14,Tara McNamara,3/5,fresh,Just like Wayne, Small Town Wisconsin has flaws, but the poignancy of the story will stick with you for a long time.
+Small Town Wisconsin,102698744,2022-06-10,Rob Thomas,3/4,fresh,It’s a movie with its heart in the right place, and does both small town and big city Wisconsin proud.
+Small Town Wisconsin,102698639,2022-06-10,Todd Jorgenson,,rotten,Despite some intriguing character dynamics and performances that generate sympathy for this fractured family, the film stumbles when it veers into melodrama without the narrative dexterity to tackle its weightier ambitions.
+Small Town Wisconsin,102698482,2022-06-10,Jackie K. Cooper,7/10,fresh,This is the kind of movie that draws you so deeply into its story you are reluctant to let it end.
+Small Town Wisconsin,102698164,2022-06-09,Glenn Kenny,,fresh,"Mueller’s direction is patient and sensitive, the cast is accomplished and committed, and the picture’s comedic aspects sometimes earn a chuckle."
+Small Town Wisconsin,102697854,2022-06-08,Brian Orndorf,B+,fresh,Naczek isn't interested in making a soap opera with this examination of fallibility, going somewhere much more authentic when exploring character aches and pains.
+Small Town Wisconsin,102695788,2022-06-02,Eddie Harrison,4/5,fresh,…a warm-hearted story of everyday life that’s easy to recommend for those who like films about people rather than portals and vortexes…
+Small Town Wisconsin,102695250,2022-05-31,Laura Clifford,C,rotten,Debuting screenwriter Jason Naczek has concocted a manchild redemption story using metaphors as heavy as a hammer and a fairy godmother who makes everything alright with a seeming flip of the switch.
+Small Town Wisconsin,2733251,2020-10-12,Jared Mobarak,B,fresh,Small Town Wisconsin is always proving itself to be more than its familiar premise thanks to Naczek's ability to infuse a lot more drama into the mix than one custody battle.
+Tejano,2564925,2019-03-07,Joe Friar,3/4,fresh,The story of a South Texas ranch hand who gets mixed up with a Mexican cartel moves with pulse-pounding velocity and features top performances from a talented cast of actors with Texas roots.
+Tejano,2557738,2019-02-12,Cary Darling,4/5,fresh,"An entertaining blast of Texas noir that nods toward the work of the Coen brothers, Quentin Tarantino and fellow Austinite Greg Kwedar's 2016 low-budget thriller ""Transpecos"" as well as ""Breaking Bad."""
+Tejano,2547231,2019-01-10,Danielle White,3/5,fresh,The story itself slithers with twists and turns and unexpected betrayals. It's almost ridiculous how many characters die in this film.
+Tejano,2530119,2018-11-08,Chris Salce,9/10,fresh,"Tejano is one of those films that can be described as a hidden gem as it sneaks under the radar and will have you talking, telling your friends about it, and wanting to watch it again."
+Death of a Salesman,2770637,2021-02-23,Michael Dougan,,fresh,"Miller has taken a small, intimate tale and expanded it into a treatise on larger themes, primarily the abuse of the American Dream."
+Death of a Salesman,1950734,2011-01-02,Randy White,5/5,fresh,A classic American tragedy.
+Death of a Salesman,1422415,2005-08-04,Jules Brenner,4/5,fresh,
+Death of a Salesman,1409415,2005-07-05,Emanuel Levy,3/5,fresh,
+Death of a Salesman,839546,2003-02-06,Frederic and Mary Ann Brussat,,fresh,"Death of a Salesman, directed by Volker Schlondorff, draws out the multiple meanings of this Pulitzer Prize-winning play by Arthur Miller about change, family and fatherhood, work and love."
+Death of a Salesman,788410,2002-09-29,Dan Lybarger,4/5,fresh,"Schlndorff's artificial settings and some amazing performances help keep this from looking like a typical ""filmed play."""
+Death of a Salesman,751951,2002-08-08,Cory Cheney,4/5,fresh,
+Death of a Salesman,743794,2002-07-26,Bob Grimm,5/5,fresh,
+Death of a Salesman,743291,2002-07-26,Scott Weinberg,5/5,fresh,They MAKE you watch it in English class for a good reason!
+Sahara,1137710,2003-05-13,Dragan Antulov,5/10,fresh,
+The Debt,2628192,2019-09-20,Diego Batlle,,fresh,A Bresson-esque movie that is always enigmatic. [Full Review in Spanish]
+The Debt,2627988,2019-09-20,Gaspar Zimerman,,fresh,The story [Director Gustavo Fontán] tells is an excuse to give way to the exploration of feelings and sensations that avoid verbality. [Full review in Spanish]
+Peppermint Candy,2725008,2020-09-16,A.S. Hamrah,,fresh,"South Korean political history of the previous twenty years, Peppermint Candy is not tempered by its hysterical edge, which adds unpredictable violence to its vignettes of romantic, domestic, and business failure."
+Peppermint Candy,2541271,2018-12-16,Panos Kotzathanasis,,fresh,"Lee Chang-dong presents a melodrama that stands apart from the plethora of similar productions due to its intense political element, because it doesn't lose its seriousness at any point and because it doesn't become hyperbolic in his effort to draw tears"
+Peppermint Candy,1883708,2010-05-11,Anton Bitel,,fresh,"This is Korea's millennial elegy, filtering its search for times past through a confection no less bittersweet than Proust's madeleine."
+Peppermint Candy,1706014,2008-01-29,Beth Accomando,9/10,fresh,The film offers a heartbreaking drama told in reverse chronology and spanning twenty years in both the life of the main character and the political history of Korea.
+Peppermint Candy,1231988,2003-12-22,Greg Muskewitz,2/5,rotten,
+Peppermint Candy,1187104,2003-08-14,Joshua Tanzer,4/4,fresh,"It's a story about the original sin of a nation as well as one character. There has rarely been a better film made, ever"
+Prison Girls,2475348,2018-05-03,Roger Ebert,,rotten,Prison Girls didn't have a lot of prison sets because it was a big-budget exploitation movie. Maybe.
+Gimme the Power,2575688,2019-04-09,Afroxander,,fresh,"Rubio's film shows ambition where none is required, making Gimme the Power a lot like Molotov's music: politically engaged without having to take itself too seriously."
+Paa,2673089,2020-02-27,Nikhat Kazmi,3.5/5,fresh,"The film, which peters off into vague sub-plots about slum redevelopment and unwarranted media-bashing in the first half, suddenly picks up and scales new heights in the second half."
+Paa,2578129,2019-04-17,Shubhra Gupta,2/5,rotten,"Disappointingly, Paa is not as out-of-the-box as it could have been."
+Paa,2429810,2017-10-24,Anil Sinanan,3/5,rotten,Will Auro survive to know his Pa and reunite his parents? Forget about the disease: this is a vanity vehicle designed to showcase the Big B's versatility.
+Paa,1860476,2009-12-14,Frank Lovece,,rotten,This would-be tearjerker without the musical numbers of typical Bollywood fare is for die-hard Amitabh Bachchan fans only.
+Paa,1860473,2009-12-14,David Chute,,fresh,"The film owes much of its interest to the alertness and sincerity of the younger Bachchan and the luminous Vidya Balan as the anguished parents, and to the soft wash of the tasteful playback songs supplied by Ilaiyaraaja."
+Paa,1858964,2009-12-05,Avi Offer,5.85/10,rotten,"Well-acted, funny and occasionally witty with terrific make-up design. However, it's often convoluted, awkwardly paced and too uneven as a whole."
+Paa,1858853,2009-12-04,Frank Lovece,,fresh,"A would-be tearjerker without the singing-dancing musical numbers of typical Bollywood fare seen in the U.S., the lackluster Paa is for die-hard Amitabh Bachchan fans only%u2014of which there is no small number."
+Paa,1858816,2009-12-04,Rachel Saltz,3/5,fresh,Odd and sometimes oddly affecting.
+Alraune (A Daughter of Destiny) (Mandrake) (Unholy Love),2835964,2021-10-30,Erich Hellmund-Waldow,,fresh,"The acting is not only artistic, it is also as realistic as can be possible in such a film."
+Alraune (A Daughter of Destiny) (Mandrake) (Unholy Love),2357086,2016-10-17,C. Hooper Trask,,fresh,"Aimed straight for the gooseflesh, it strikes directly into the centre of the target."
+Toorbos,2760593,2021-01-29,Neil Young,,fresh,Built around a luminous and intriguing central performance by dancer-actor Elani Dekker.
+Toorbos,2752827,2020-12-21,Guy Lodge,,fresh,"A satisfying marriage of folky period romance and environmental parable from the misty, mossy depths of South Africa's Knysna forest region..."
+Connors' War,1555113,2006-11-09,David Nusair,1.5/4,rotten,"...although Criss does show some potential as a performer, his efforts to step into the shoes of a blind character are laughable."
+Connors' War,1539106,2006-09-19,Scott Weinberg,2/5,rotten,"Standard cable fodder all the way, with only a few solid action scenes and maybe one colorful performance in the whole thing."
+Born to Kill,2710947,2020-08-05,Mike Massie,10/10,fresh,"One of the most acerbic of all films noir, boasting essentially no redeemable characters (or a wealth of deliciously evil villains) while also being utterly enthralling."
+Born to Kill,2340106,2016-07-15,David Nusair,3/4,fresh,...a fairly typical film-noir premise that's employed to watchable yet entirely unmemorable effect by Robert Wise...
+Born to Kill,1507021,2006-05-16,Nick Schager,B,fresh,Competent if slightly too tame for a supposedly sleazy story.
+Born to Kill,1501617,2006-05-01,Fernando F. Croce,,fresh,"The usually meek Robert Wise trades his chameleonic tastefulness for full-on, jazzy misanthropy in this nasty melodrama."
+Born to Kill,1433953,2005-09-09,Jeffrey M. Anderson,3/4,fresh,"Hard to watch, but effective and alluring nonetheless."
+Born to Kill,1123980,2003-04-02,Dennis Schwartz,C,rotten,A revolting B film noir...
+The Soong Sisters,1402087,2005-06-15,Emanuel Levy,3/5,fresh,
+La Sapienza,102772380,2023-01-24,Vadim Rizov,,fresh,"Sapienza is a pretty lovely film. Symmetricities are everywhere, starting with that opening architectural showreel, which deliberately avoids perfect symmetricity..."
+La Sapienza,2767839,2021-02-14,Dustin Chang,,fresh,Their sincere expression of these thoughts rings true and melts away its artificiality in its presentation soon enough. This is the beauty of La Sapienza and Green films in general.
+La Sapienza,2598336,2019-06-18,C.J. Prince,,fresh,"It's a nice entry point into a peculiar cinematic universe, and those willing to open themselves to it will find a lot to enjoy."
+La Sapienza,2503963,2018-08-28,Charles Mudede,,fresh,"If architecture aspires to the condition of music, the acting in La Sapienza aspires to the condition of architecture. You will love the ending of this very original and elegant and arty work."
+La Sapienza,2314368,2016-03-12,Forrest Cardamenis,B,fresh,This startling architectural juxtaposition feels like a wake-up call.
+La Sapienza,2275677,2015-08-03,Nicole Armour,,fresh,"While Green's film is dense with historical fact and theory, it's not averse to plumbing life's mysteries. Suffused with warmth, it expresses a potent admiration for human striving and accomplishment."
+La Sapienza,2273804,2015-07-23,Norman Wilner,2/5,rotten,"The uncomplicated narrative resists stylization; Green's presentation turns everyone into mannequins, rendering their emotions theoretical. That may well be his point, but it didn't work for me."
+La Sapienza,2269287,2015-06-26,Sam Lubell,,fresh,"On the surface, writer-director Eugne Green's film ""La Sapienza"" is slow, strange and awkward - but stick with it and it may win you over."
+La Sapienza,2265997,2015-06-05,Rob Garratt,4/5,fresh,"Layered with reels of swirling shots of Rome's most beautiful buildings -- all crucially shot from the ground upwards, staring at the heavens-- La Sapienza is visually stunning."
+La Sapienza,2265990,2015-06-05,Boyd van Hoeij,,fresh,"The Sapience juxtaposes insights on how people are emotionally connected with ruminations on the buildings and spaces through which they move, in which they live and, in Alexandre's case, which they also create."
+La Sapienza,2265989,2015-06-05,Robert Horton,3/4,fresh,"If you can groove into this non-realistic mode, the film casts a spell."
+La Sapienza,2265790,2015-06-04,Tom Keogh,3.5/4,fresh,A beautiful space for people and light.
+La Sapienza,2255621,2015-04-09,Wesley Morris,,rotten,This kind of formalism needs to do more than walk through classical wonders. It should want to create cinema that can stand near or beside them. This movie defensively consecrates what's already there. You don't need a film to do that.
+La Sapienza,2255195,2015-04-08,Scott Foundas,,fresh,"An exquisite rumination on life, love and art that tickles the heart and mind in equal measure."
+La Sapienza,2252858,2015-03-23,Richard Brody,,fresh,"Green's richly textured, painterly images fuse with the story to evoke the essence of humane urbanity and the relationships that it fosters, whether educational, familial, or erotic."
+La Sapienza,2252553,2015-03-20,Ignatiy Vishnevetsky,B+,fresh,"Green doesn't so much use his characters as mouthpieces as emotionally invest them in art, turning opinions into feelings."
+La Sapienza,2252541,2015-03-20,Godfrey Cheshire,4/4,fresh,"""La Sapienza"" strikes this reviewer as easily the most astonishing and important movie to emerge from France in quite some time."
+La Sapienza,2252452,2015-03-19,A.O. Scott,,fresh,The movie is an unapologetically rarefied undertaking and at the same time a gracious and inviting film.
+La Sapienza,2252301,2015-03-19,David Noh,,rotten,"Pretentious, stuffy and slow. There's some beautiful scenery here but oh, what you must put up with to earn it!"
+La Sapienza,2252028,2015-03-18,Noel Murray,3/5,fresh,"While La Sapienza is unsatisfying as drama, it's frequently beautiful just as a tour through architecturally significant Italian buildings."
+La Sapienza,2251985,2015-03-17,David Ehrlich,3/5,fresh,La Sapienza  alternately feels like a self-reflexive love story or a haunted history lesson -- its best scenes play like both.
+La Sapienza,2251926,2015-03-17,Zachary Wigon,,fresh,A picture that balances heart and mind with nuance.
+La Sapienza,2251650,2015-03-14,Harvey S. Karten,B+,fresh,"As in ""Who's Afraid of Virginia Woolf,"" both the younger couple and their older mentors are changed from a relationship."
+La Sapienza,2250991,2015-03-12,Ben Sachs,,fresh,"This recalls Manoel de Oliveira and Eric Rohmer in its poker-faced style, deliberately archaic storytelling, and magisterial epiphanies."
+La Sapienza,2225361,2014-09-28,Donald J. Levit,,fresh,"Although a love-fiction crossed with documentary lecture and superb Raphael O'Byrne cinematography, 'La Sapienza' is as close as celluloid can approach to architecture."
+La Sapienza,2222032,2014-09-10,Carson Lund,3/4,fresh,"Eugne Green's mannered direction doesn't work for every situation it's homogenously applied to, but at its most effective it inspires an enhanced sensitivity to the import of every gesture, visual or verbal."
+Uncle Tom,2713732,2020-08-14,Megan Basham,,fresh,Uncle Tom suffers from an overreliance on pundits. Its most compelling insights come from people who've never been quoted in a Twitter or Facebook battle.
+Uncle Tom,2706229,2020-07-19,Matthew Pejkovic,4/5,fresh,"An incredibly relevant and insightful documentary that delves into the past, present, and future of the black American conservative movement."
 Uncle Tom,2698525,2020-06-24,Dante James,7/10,fresh,"It's a little misleading in some areas, especially if you know the players involved in this doc, but there are a lot of interesting historical facts about the breakdown of the Black family and how the whole welfare system targeted the Black community."

examples/gptllama.ts

@@ -25,12 +25,12 @@ import { ChatMessage, LlamaDeuce, OpenAI } from "llamaindex";
 
   while (true) {
     const next = history.length % 2 === 1 ? gpt4 : l2;
-    const r = await next.chat(
-      history.map(({ content, role }) => ({
+    const r = await next.chat({
+      messages: history.map(({ content, role }) => ({
         content,
         role: next === l2 ? role : role === "user" ? "assistant" : "user",
       })),
-    );
+    });
     history.push({
       content: r.message.content,
       role: next === l2 ? "assistant" : "user",

examples/huggingface.ts

@@ -32,12 +32,15 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const stream = await queryEngine.query({
+    query: "What did the author do in college?",
+    stream: true,
+  });
 
   // Output response
-  console.log(response.toString());
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.response);
+  }
 }
 
 main().catch(console.error);

examples/jsonExtract.ts

@@ -21,18 +21,20 @@ async function main() {
     action_items: ["action item 1", "action item 2"],
   };
 
-  const response = await llm.chat([
-    {
-      role: "system",
-      content: `You are an expert assistant for summarizing and extracting insights from sales call transcripts.\n\nGenerate a valid JSON in the following format:\n\n${JSON.stringify(
-        example,
-      )}`,
-    },
-    {
-      role: "user",
-      content: `Here is the transcript: \n------\n${transcript}\n------`,
-    },
-  ]);
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "system",
+        content: `You are an expert assistant for summarizing and extracting insights from sales call transcripts.\n\nGenerate a valid JSON in the following format:\n\n${JSON.stringify(
+          example,
+        )}`,
+      },
+      {
+        role: "user",
+        content: `Here is the transcript: \n------\n${transcript}\n------`,
+      },
+    ],
+  });
 
   const json = JSON.parse(response.message.content);
 

examples/keywordIndex.ts

@@ -20,9 +20,9 @@ async function main() {
         mode,
       }),
     });
-    const response = await queryEngine.query(
-      "What did the author do growing up?",
-    );
+    const response = await queryEngine.query({
+      query: "What did the author do growing up?",
+    });
     console.log(response.toString());
   });
 }

examples/llamadeuce.ts

@@ -2,6 +2,8 @@ import { DeuceChatStrategy, LlamaDeuce } from "llamaindex";
 
 (async () => {
   const deuce = new LlamaDeuce({ chatStrategy: DeuceChatStrategy.META });
-  const result = await deuce.chat([{ content: "Hello, world!", role: "user" }]);
+  const result = await deuce.chat({
+    messages: [{ content: "Hello, world!", role: "user" }],
+  });
   console.log(result);
 })();

examples/lowlevel.ts

@@ -27,9 +27,12 @@ import {
     },
   ];
 
-  const response = await responseSynthesizer.synthesize(
-    "What age am I?",
+  const stream = await responseSynthesizer.synthesize({
+    query: "What age am I?",
     nodesWithScore,
-  );
-  console.log(response.response);
+    stream: true,
+  });
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.response);
+  }
 })();

examples/markdown.ts

@@ -11,7 +11,9 @@ async function main() {
   // Query the index
   const queryEngine = index.asQueryEngine();
 
-  const response = await queryEngine.query("What does the example code do?");
+  const response = await queryEngine.query({
+    query: "What does the example code do?",
+  });
 
   // Output response
   console.log(response.toString());

examples/mistral.ts

@@ -27,7 +27,7 @@ async function rag(llm: LLM, embedModel: BaseEmbedding, query: string) {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(query);
+  const response = await queryEngine.query({ query });
   return response.response;
 }
 
@@ -43,19 +43,20 @@ async function rag(llm: LLM, embedModel: BaseEmbedding, query: string) {
 
   // chat api (non-streaming)
   const llm = new MistralAI({ model: "mistral-tiny" });
-  const response = await llm.chat([
-    { content: "What is the best French cheese?", role: "user" },
-  ]);
+  const response = await llm.chat({
+    messages: [{ content: "What is the best French cheese?", role: "user" }],
+  });
   console.log(response.message.content);
 
   // chat api (streaming)
-  const stream = await llm.chat(
-    [{ content: "Who is the most renowned French painter?", role: "user" }],
-    undefined,
-    true,
-  );
+  const stream = await llm.chat({
+    messages: [
+      { content: "Who is the most renowned French painter?", role: "user" },
+    ],
+    stream: true,
+  });
   for await (const chunk of stream) {
-    process.stdout.write(chunk);
+    process.stdout.write(chunk.delta);
   }
 
   // rag

examples/mongo.ts

@@ -54,7 +54,7 @@ async function main() {
       break;
     }
 
-    const response = await queryEngine.query(query);
+    const response = await queryEngine.query({ query });
 
     // Output response
     console.log(response.toString());

examples/mongodb/3_query.ts

@@ -24,9 +24,9 @@ async function query() {
 
   const retriever = index.asRetriever({ similarityTopK: 20 });
   const queryEngine = index.asQueryEngine({ retriever });
-  const result = await queryEngine.query(
-    "What does the author think of web frameworks?",
-  );
+  const result = await queryEngine.query({
+    query: "What does the author think of web frameworks?",
+  });
   console.log(result.response);
   await client.close();
 }

examples/multimodal/rag.ts

@@ -46,9 +46,9 @@ async function main() {
     responseSynthesizer: new MultiModalResponseSynthesizer({ serviceContext }),
     retriever: index.asRetriever({ similarityTopK: 3, imageSimilarityTopK: 1 }),
   });
-  const result = await queryEngine.query(
-    "Tell me more about Vincent van Gogh's famous paintings",
-  );
+  const result = await queryEngine.query({
+    query: "Tell me more about Vincent van Gogh's famous paintings",
+  });
   console.log(result.response, "\n");
   images.forEach((image) =>
     console.log(`Image retrieved and used in inference: ${image.toString()}`),

examples/ollama.ts

@@ -3,32 +3,34 @@ import { Ollama } from "llamaindex";
 (async () => {
   const llm = new Ollama({ model: "llama2", temperature: 0.75 });
   {
-    const response = await llm.chat([
-      { content: "Tell me a joke.", role: "user" },
-    ]);
+    const response = await llm.chat({
+      messages: [{ content: "Tell me a joke.", role: "user" }],
+    });
     console.log("Response 1:", response.message.content);
   }
   {
-    const response = await llm.complete("How are you?");
-    console.log("Response 2:", response.message.content);
+    const response = await llm.complete({ prompt: "How are you?" });
+    console.log("Response 2:", response.text);
   }
   {
-    const response = await llm.chat(
-      [{ content: "Tell me a joke.", role: "user" }],
-      undefined,
-      true,
-    );
+    const response = await llm.chat({
+      messages: [{ content: "Tell me a joke.", role: "user" }],
+      stream: true,
+    });
     console.log("Response 3:");
     for await (const message of response) {
-      process.stdout.write(message); // no newline
+      process.stdout.write(message.delta); // no newline
     }
     console.log(); // newline
   }
   {
-    const response = await llm.complete("How are you?", undefined, true);
+    const response = await llm.complete({
+      prompt: "How are you?",
+      stream: true,
+    });
     console.log("Response 4:");
     for await (const message of response) {
-      process.stdout.write(message); // no newline
+      process.stdout.write(message.text); // no newline
     }
     console.log(); // newline
   }

examples/openai.ts

@@ -4,12 +4,12 @@ import { OpenAI } from "llamaindex";
   const llm = new OpenAI({ model: "gpt-4-1106-preview", temperature: 0.1 });
 
   // complete api
-  const response1 = await llm.complete("How are you?");
-  console.log(response1.message.content);
+  const response1 = await llm.complete({ prompt: "How are you?" });
+  console.log(response1.text);
 
   // chat api
-  const response2 = await llm.chat([
-    { content: "Tell me a joke.", role: "user" },
-  ]);
+  const response2 = await llm.chat({
+    messages: [{ content: "Tell me a joke.", role: "user" }],
+  });
   console.log(response2.message.content);
 })();

examples/package.json

@@ -3,12 +3,13 @@
   "private": true,
   "name": "examples",
   "dependencies": {
-    "@notionhq/client": "^2.2.13",
     "@datastax/astra-db-ts": "^0.1.2",
+    "@notionhq/client": "^2.2.14",
     "@pinecone-database/pinecone": "^1.1.2",
+    "chromadb": "^1.7.3",
     "commander": "^11.1.0",
-    "llamaindex": "latest",
     "dotenv": "^16.3.1",
+    "llamaindex": "latest",
     "mongodb": "^6.2.0"
   },
   "devDependencies": {

examples/pg-vector-store/query.ts

@@ -31,7 +31,7 @@ async function main() {
       }
 
       try {
-        const answer = await queryEngine.query(question);
+        const answer = await queryEngine.query({ query: question });
         console.log(answer.response);
       } catch (error) {
         console.error("Error:", error);

examples/pinecone-vector-store/query.ts

@@ -29,7 +29,7 @@ async function main() {
       }
 
       try {
-        const answer = await queryEngine.query(question);
+        const answer = await queryEngine.query({ query: question });
         console.log(answer.response);
       } catch (error) {
         console.error("Error:", error);

examples/portkey.ts

@@ -1,7 +1,6 @@
 import { Portkey } from "llamaindex";
 
 (async () => {
-  const llms = [{}];
   const portkey = new Portkey({
     mode: "single",
     llms: [
@@ -13,11 +12,14 @@ import { Portkey } from "llamaindex";
       },
     ],
   });
-  const result = portkey.stream_chat([
-    { role: "system", content: "You are a helpful assistant." },
-    { role: "user", content: "Tell me a joke." },
-  ]);
+  const result = await portkey.chat({
+    messages: [
+      { role: "system", content: "You are a helpful assistant." },
+      { role: "user", content: "Tell me a joke." },
+    ],
+    stream: true,
+  });
   for await (const res of result) {
-    process.stdout.write(res);
+    process.stdout.write(res.delta);
   }
 })();

examples/readers/load-assemblyai.ts

@@ -48,7 +48,7 @@ program
         break;
       }
 
-      const response = await queryEngine.query(query);
+      const response = await queryEngine.query({ query });
 
       console.log(response.toString());
     }

examples/readers/load-csv.ts

@@ -38,9 +38,9 @@ Given the CSV file, generate me Typescript code to answer the question: ${query}
   const queryEngine = index.asQueryEngine({ responseSynthesizer });
 
   // Query the index
-  const response = await queryEngine.query(
-    "What is the correlation between survival and age?",
-  );
+  const response = await queryEngine.query({
+    query: "What is the correlation between survival and age?",
+  });
 
   // Output response
   console.log(response.toString());

examples/readers/load-docx.ts

@@ -15,7 +15,7 @@ async function main() {
 
   // Test query
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(SAMPLE_QUERY);
+  const response = await queryEngine.query({ query: SAMPLE_QUERY });
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 

examples/readers/load-html.ts

@@ -10,9 +10,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What were the notable changes in 18.1?",
-  );
+  const response = await queryEngine.query({
+    query: "What were the notable changes in 18.1?",
+  });
 
   // Output response
   console.log(response.toString());

examples/readers/load-md.ts

@@ -15,7 +15,7 @@ async function main() {
 
   // Test query
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(SAMPLE_QUERY);
+  const response = await queryEngine.query({ query: SAMPLE_QUERY });
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 

examples/readers/load-notion.ts

@@ -79,7 +79,7 @@ program
         break;
       }
 
-      const response = await queryEngine.query(query);
+      const response = await queryEngine.query({ query });
 
       // Output response
       console.log(response.toString());

examples/readers/load-pdf.ts

@@ -10,7 +10,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query("What mistakes did they make?");
+  const response = await queryEngine.query({
+    query: "What mistakes did they make?",
+  });
 
   // Output response
   console.log(response.toString());

examples/sentenceWindow.ts

@@ -31,9 +31,9 @@ async function main() {
   const queryEngine = index.asQueryEngine({
     nodePostprocessors: [new MetadataReplacementPostProcessor("window")],
   });
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/storageContext.ts

@@ -20,9 +20,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());
@@ -35,9 +35,9 @@ async function main() {
     storageContext: secondStorageContext,
   });
   const loadedQueryEngine = loadedIndex.asQueryEngine();
-  const loadedResponse = await loadedQueryEngine.query(
-    "What did the author do growing up?",
-  );
+  const loadedResponse = await loadedQueryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(loadedResponse.toString());
 }
 

examples/subquestion.ts

@@ -18,9 +18,9 @@ import essay from "./essay";
     ],
   });
 
-  const response = await queryEngine.query(
-    "How was Paul Grahams life different before and after YC?",
-  );
+  const response = await queryEngine.query({
+    query: "How was Paul Grahams life different before and after YC?",
+  });
 
   console.log(response.toString());
 })();

examples/summaryIndex.ts

@@ -20,9 +20,9 @@ async function main() {
   const queryEngine = index.asQueryEngine({
     retriever: index.asRetriever({ mode: SummaryRetrieverMode.LLM }),
   });
-  const response = await queryEngine.query(
-    "What did the author do growing up?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(response.toString());
 }
 

examples/together-ai.ts

@@ -0,0 +1,29 @@
+import { TogetherEmbedding, TogetherLLM } from "llamaindex";
+
+// process.env.TOGETHER_API_KEY is required
+const together = new TogetherLLM({
+  model: "mistralai/Mixtral-8x7B-Instruct-v0.1",
+});
+
+(async () => {
+  const generator = await together.chat({
+    messages: [
+      {
+        role: "system",
+        content: "You are an AI assistant",
+      },
+      {
+        role: "user",
+        content: "Tell me about San Francisco",
+      },
+    ],
+    stream: true,
+  });
+  console.log("Chatting with Together AI...");
+  for await (const message of generator) {
+    process.stdout.write(message.delta);
+  }
+  const embedding = new TogetherEmbedding();
+  const vector = await embedding.getTextEmbedding("Hello world!");
+  console.log("vector:", vector);
+})();

examples/together-ai/vector-index.ts

@@ -0,0 +1,39 @@
+import fs from "node:fs/promises";
+
+import {
+  Document,
+  TogetherEmbedding,
+  TogetherLLM,
+  VectorStoreIndex,
+  serviceContextFromDefaults,
+} from "llamaindex";
+
+async function main() {
+  const apiKey = process.env.TOGETHER_API_KEY;
+  if (!apiKey) {
+    throw new Error("Missing TOGETHER_API_KEY");
+  }
+  const path = require.resolve("llamaindex/examples/abramov.txt");
+  const essay = await fs.readFile(path, "utf-8");
+
+  const document = new Document({ text: essay, id_: path });
+
+  const serviceContext = serviceContextFromDefaults({
+    llm: new TogetherLLM({ model: "mistralai/Mixtral-8x7B-Instruct-v0.1" }),
+    embedModel: new TogetherEmbedding(),
+  });
+
+  const index = await VectorStoreIndex.fromDocuments([document], {
+    serviceContext,
+  });
+
+  const queryEngine = index.asQueryEngine();
+
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);

examples/tsconfig.json

@@ -1,4 +1,5 @@
 {
+  "extends": "../tsconfig.json",
   "compilerOptions": {
     "target": "es2016",
     "module": "commonjs",

examples/vectorIndex.ts

@@ -16,9 +16,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vectorIndexAnthropic.ts

@@ -35,9 +35,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine({ responseSynthesizer });
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vectorIndexCustomize.ts

@@ -33,9 +33,9 @@ async function main() {
     [nodePostprocessor],
   );
 
-  const response = await queryEngine.query(
-    "What did the author do growing up?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do growing up?",
+  });
   console.log(response.response);
 }
 

examples/vectorIndexFromVectorStore.ts

@@ -189,7 +189,9 @@ async function main() {
     },
   });
 
-  const response = await queryEngine.query("How many results do you have?");
+  const response = await queryEngine.query({
+    query: "How many results do you have?",
+  });
 
   console.log(response.toString());
 }

examples/vectorIndexGPT4.ts

@@ -25,9 +25,9 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query(
-    "What did the author do in college?",
-  );
+  const response = await queryEngine.query({
+    query: "What did the author do in college?",
+  });
 
   // Output response
   console.log(response.toString());

examples/vision.ts

@@ -4,12 +4,12 @@ import { OpenAI } from "llamaindex";
   const llm = new OpenAI({ model: "gpt-4-vision-preview", temperature: 0.1 });
 
   // complete api
-  const response1 = await llm.complete("How are you?");
-  console.log(response1.message.content);
+  const response1 = await llm.complete({ prompt: "How are you?" });
+  console.log(response1.text);
 
   // chat api
-  const response2 = await llm.chat([
-    { content: "Tell me a joke!", role: "user" },
-  ]);
+  const response2 = await llm.chat({
+    messages: [{ content: "Tell me a joke!", role: "user" }],
+  });
   console.log(response2.message.content);
 })();

package.json

@@ -8,6 +8,7 @@
     "lint": "turbo run lint",
     "prepare": "husky install",
     "test": "turbo run test",
+    "type-check": "tsc -b --diagnostics",
     "new-version": "turbo run build lint test --filter=\"!docs\" && changeset version",
     "new-snapshot": "turbo run build lint test --filter=\"!docs\" && changeset version --snapshot"
   },
@@ -23,7 +24,8 @@
     "prettier": "^3.1.1",
     "prettier-plugin-organize-imports": "^3.2.4",
     "ts-jest": "^29.1.1",
-    "turbo": "^1.11.2"
+    "turbo": "^1.11.2",
+    "typescript": "^5.3.3"
   },
   "packageManager": "pnpm@8.10.5+sha256.a4bd9bb7b48214bbfcd95f264bd75bb70d100e5d4b58808f5cd6ab40c6ac21c5",
   "pnpm": {

packages/core/CHANGELOG.md

@@ -1,5 +1,38 @@
 # llamaindex
 
+## 0.0.48
+
+### Patch Changes
+
+- 34a26e5: Remove HistoryChatEngine and use ChatHistory for all chat engines
+
+## 0.0.47
+
+### Patch Changes
+
+- 844029d: Add streaming support for QueryEngine (and unify streaming interface with ChatEngine)
+- 844029d: Breaking: Use parameter object for query and chat methods of ChatEngine and QueryEngine
+
+## 0.0.46
+
+### Patch Changes
+
+- 977f284: fixing import statement
+- 5d3bb66: fix: class SimpleKVStore might throw error in ES module
+- f18c9f6: refactor: Updated low-level streaming interface
+
+## 0.0.45
+
+### Patch Changes
+
+- 2e6b36e: feat: support together AI
+
+## 0.0.44
+
+### Patch Changes
+
+- 648482b: Feat: Add support for Chroma DB as a vector store
+
 ## 0.0.43
 
 ### Patch Changes

packages/core/README.md

@@ -1,5 +1,10 @@
 # LlamaIndex.TS
 
+[![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
+
 LlamaIndex is a data framework for your LLM application.
 
 Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.
@@ -12,7 +17,7 @@ LlamaIndex.TS aims to be a lightweight, easy to use set of libraries to help you
 
 ## Getting started with an example:
 
-LlamaIndex.TS requries Node v18 or higher. You can download it from https://nodejs.org or use https://nvm.sh (our preferred option).
+LlamaIndex.TS requires Node v18 or higher. You can download it from https://nodejs.org or use https://nvm.sh (our preferred option).
 
 In a new folder:
 
@@ -84,11 +89,37 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
 
 - [SimplePrompt](/packages/core/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
 
+## Note: NextJS:
+
+If you're using NextJS App Router, you'll need to use the NodeJS runtime (default) and add the following config to your next.config.js to have it use imports/exports in the same way Node does.
+
+```js
+export const runtime = "nodejs"; // default
+```
+
+```js
+// next.config.js
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  webpack: (config) => {
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      sharp$: false,
+      "onnxruntime-node$": false,
+    };
+    return config;
+  },
+};
+
+module.exports = nextConfig;
+```
+
 ## Supported LLMs:
 
 - OpenAI GPT-3.5-turbo and GPT-4
 - Anthropic Claude Instant and Claude 2
 - Llama2 Chat LLMs (70B, 13B, and 7B parameters)
+- MistralAI Chat LLMs
 
 ## Contributing:
 

packages/core/jest.config.cjs

@@ -2,4 +2,5 @@
 module.exports = {
   preset: "ts-jest",
   testEnvironment: "node",
+  testPathIgnorePatterns: ["/lib/"],
 };

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.0.43",
+  "version": "0.0.48",
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.9.1",
@@ -10,6 +10,7 @@
     "@pinecone-database/pinecone": "^1.1.2",
     "@xenova/transformers": "^2.10.0",
     "assemblyai": "^4.0.0",
+    "chromadb": "^1.7.3",
     "file-type": "^18.7.0",
     "js-tiktoken": "^1.0.8",
     "lodash": "^4.17.21",
@@ -26,31 +27,47 @@
     "rake-modified": "^1.0.8",
     "replicate": "^0.21.1",
     "string-strip-html": "^13.4.3",
-    "uuid": "^9.0.1",
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
     "@types/jest": "^29.5.11",
     "@types/lodash": "^4.14.202",
-    "@types/node": "^18.19.2",
+    "@types/node": "^18.19.6",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.10.9",
-    "@types/uuid": "^9.0.7",
+    "bunchee": "^4.3.3",
     "node-stdlib-browser": "^1.2.0",
-    "tsup": "^7.2.0",
-    "typescript": "^5.3.2"
+    "typescript": "^5.3.3"
   },
   "engines": {
     "node": ">=18.0.0"
   },
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
-  "module": "./dist/index.mjs",
-  "repository": "run-llama/LlamaIndexTS",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "./examples/*": "./examples/*"
+  },
+  "files": [
+    "dist",
+    "examples",
+    "src",
+    "types",
+    "CHANGELOG.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/core"
+  },
   "scripts": {
     "lint": "eslint .",
     "test": "jest",
-    "build": "tsup src/index.ts --format esm,cjs --dts",
-    "dev": "tsup src/index.ts --format esm,cjs --dts --watch"
+    "build": "bunchee",
+    "dev": "bunchee -w"
   }
 }

packages/core/src/ChatEngine.ts

@@ -1,452 +0,0 @@
-import { v4 as uuidv4 } from "uuid";
-import { ChatHistory } from "./ChatHistory";
-import { NodeWithScore, TextNode } from "./Node";
-import {
-  CondenseQuestionPrompt,
-  ContextSystemPrompt,
-  defaultCondenseQuestionPrompt,
-  defaultContextSystemPrompt,
-  messagesToHistoryStr,
-} from "./Prompt";
-import { BaseQueryEngine } from "./QueryEngine";
-import { Response } from "./Response";
-import { BaseRetriever } from "./Retriever";
-import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { Event } from "./callbacks/CallbackManager";
-import { ChatMessage, LLM, OpenAI } from "./llm";
-import { BaseNodePostprocessor } from "./postprocessors";
-
-/**
- * A ChatEngine is used to handle back and forth chats between the application and the LLM.
- */
-export interface ChatEngine {
-  /**
-   * Send message along with the class's current chat history to the LLM.
-   * @param message
-   * @param chatHistory optional chat history if you want to customize the chat history
-   * @param streaming optional streaming flag, which auto-sets the return value if True.
-   */
-  chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-    streaming?: T,
-  ): Promise<R>;
-
-  /**
-   * Resets the chat history so that it's empty.
-   */
-  reset(): void;
-}
-
-/**
- * SimpleChatEngine is the simplest possible chat engine. Useful for using your own custom prompts.
- */
-export class SimpleChatEngine implements ChatEngine {
-  chatHistory: ChatMessage[];
-  llm: LLM;
-
-  constructor(init?: Partial<SimpleChatEngine>) {
-    this.chatHistory = init?.chatHistory ?? [];
-    this.llm = init?.llm ?? new OpenAI();
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-    streaming?: T,
-  ): Promise<R> {
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-
-    //Non-streaming option
-    chatHistory = chatHistory ?? this.chatHistory;
-    chatHistory.push({ content: message, role: "user" });
-    const response = await this.llm.chat(chatHistory, undefined);
-    chatHistory.push(response.message);
-    this.chatHistory = chatHistory;
-    return new Response(response.message.content) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory?: ChatMessage[],
-  ): AsyncGenerator<string, void, unknown> {
-    chatHistory = chatHistory ?? this.chatHistory;
-    chatHistory.push({ content: message, role: "user" });
-    const response_generator = await this.llm.chat(
-      chatHistory,
-      undefined,
-      true,
-    );
-
-    var accumulator: string = "";
-    for await (const part of response_generator) {
-      accumulator += part;
-      yield part;
-    }
-
-    chatHistory.push({ content: accumulator, role: "assistant" });
-    this.chatHistory = chatHistory;
-    return;
-  }
-
-  reset() {
-    this.chatHistory = [];
-  }
-}
-
-/**
- * CondenseQuestionChatEngine is used in conjunction with a Index (for example VectorStoreIndex).
- * It does two steps on taking a user's chat message: first, it condenses the chat message
- * with the previous chat history into a question with more context.
- * Then, it queries the underlying Index using the new question with context and returns
- * the response.
- * CondenseQuestionChatEngine performs well when the input is primarily questions about the
- * underlying data. It performs less well when the chat messages are not questions about the
- * data, or are very referential to previous context.
- */
-export class CondenseQuestionChatEngine implements ChatEngine {
-  queryEngine: BaseQueryEngine;
-  chatHistory: ChatMessage[];
-  serviceContext: ServiceContext;
-  condenseMessagePrompt: CondenseQuestionPrompt;
-
-  constructor(init: {
-    queryEngine: BaseQueryEngine;
-    chatHistory: ChatMessage[];
-    serviceContext?: ServiceContext;
-    condenseMessagePrompt?: CondenseQuestionPrompt;
-  }) {
-    this.queryEngine = init.queryEngine;
-    this.chatHistory = init?.chatHistory ?? [];
-    this.serviceContext =
-      init?.serviceContext ?? serviceContextFromDefaults({});
-    this.condenseMessagePrompt =
-      init?.condenseMessagePrompt ?? defaultCondenseQuestionPrompt;
-  }
-
-  private async condenseQuestion(chatHistory: ChatMessage[], question: string) {
-    const chatHistoryStr = messagesToHistoryStr(chatHistory);
-
-    return this.serviceContext.llm.complete(
-      defaultCondenseQuestionPrompt({
-        question: question,
-        chatHistory: chatHistoryStr,
-      }),
-    );
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    const condensedQuestion = (
-      await this.condenseQuestion(chatHistory, extractText(message))
-    ).message.content;
-
-    const response = await this.queryEngine.query(condensedQuestion);
-
-    chatHistory.push({ content: message, role: "user" });
-    chatHistory.push({ content: response.response, role: "assistant" });
-
-    return response as R;
-  }
-
-  reset() {
-    this.chatHistory = [];
-  }
-}
-
-export interface Context {
-  message: ChatMessage;
-  nodes: NodeWithScore[];
-}
-
-export interface ContextGenerator {
-  generate(message: string, parentEvent?: Event): Promise<Context>;
-}
-
-export class DefaultContextGenerator implements ContextGenerator {
-  retriever: BaseRetriever;
-  contextSystemPrompt: ContextSystemPrompt;
-  nodePostprocessors: BaseNodePostprocessor[];
-
-  constructor(init: {
-    retriever: BaseRetriever;
-    contextSystemPrompt?: ContextSystemPrompt;
-    nodePostprocessors?: BaseNodePostprocessor[];
-  }) {
-    this.retriever = init.retriever;
-    this.contextSystemPrompt =
-      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
-    this.nodePostprocessors = init.nodePostprocessors || [];
-  }
-
-  private applyNodePostprocessors(nodes: NodeWithScore[]) {
-    return this.nodePostprocessors.reduce(
-      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
-      nodes,
-    );
-  }
-
-  async generate(message: string, parentEvent?: Event): Promise<Context> {
-    if (!parentEvent) {
-      parentEvent = {
-        id: uuidv4(),
-        type: "wrapper",
-        tags: ["final"],
-      };
-    }
-    const sourceNodesWithScore = await this.retriever.retrieve(
-      message,
-      parentEvent,
-    );
-
-    const nodes = this.applyNodePostprocessors(sourceNodesWithScore);
-
-    return {
-      message: {
-        content: this.contextSystemPrompt({
-          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
-        }),
-        role: "system",
-      },
-      nodes,
-    };
-  }
-}
-
-/**
- * ContextChatEngine uses the Index to get the appropriate context for each query.
- * The context is stored in the system prompt, and the chat history is preserved,
- * ideally allowing the appropriate context to be surfaced for each query.
- */
-export class ContextChatEngine implements ChatEngine {
-  chatModel: LLM;
-  chatHistory: ChatMessage[];
-  contextGenerator: ContextGenerator;
-
-  constructor(init: {
-    retriever: BaseRetriever;
-    chatModel?: LLM;
-    chatHistory?: ChatMessage[];
-    contextSystemPrompt?: ContextSystemPrompt;
-    nodePostprocessors?: BaseNodePostprocessor[];
-  }) {
-    this.chatModel =
-      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
-    this.chatHistory = init?.chatHistory ?? [];
-    this.contextGenerator = new DefaultContextGenerator({
-      retriever: init.retriever,
-      contextSystemPrompt: init?.contextSystemPrompt,
-    });
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-
-    const parentEvent: Event = {
-      id: uuidv4(),
-      type: "wrapper",
-      tags: ["final"],
-    };
-    const context = await this.contextGenerator.generate(
-      extractText(message),
-      parentEvent,
-    );
-
-    chatHistory.push({ content: message, role: "user" });
-
-    const response = await this.chatModel.chat(
-      [context.message, ...chatHistory],
-      parentEvent,
-    );
-    chatHistory.push(response.message);
-
-    this.chatHistory = chatHistory;
-
-    return new Response(
-      response.message.content,
-      context.nodes.map((r) => r.node),
-    ) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory?: ChatMessage[] | undefined,
-  ): AsyncGenerator<string, void, unknown> {
-    chatHistory = chatHistory ?? this.chatHistory;
-
-    const parentEvent: Event = {
-      id: uuidv4(),
-      type: "wrapper",
-      tags: ["final"],
-    };
-    const context = await this.contextGenerator.generate(
-      extractText(message),
-      parentEvent,
-    );
-
-    chatHistory.push({ content: message, role: "user" });
-
-    const response_stream = await this.chatModel.chat(
-      [context.message, ...chatHistory],
-      parentEvent,
-      true,
-    );
-    var accumulator: string = "";
-    for await (const part of response_stream) {
-      accumulator += part;
-      yield part;
-    }
-
-    chatHistory.push({ content: accumulator, role: "assistant" });
-
-    this.chatHistory = chatHistory;
-
-    return;
-  }
-
-  reset() {
-    this.chatHistory = [];
-  }
-}
-
-export interface MessageContentDetail {
-  type: "text" | "image_url";
-  text?: string;
-  image_url?: { url: string };
-}
-
-/**
- * Extended type for the content of a message that allows for multi-modal messages.
- */
-export type MessageContent = string | MessageContentDetail[];
-
-/**
- * Extracts just the text from a multi-modal message or the message itself if it's just text.
- *
- * @param message The message to extract text from.
- * @returns The extracted text
- */
-function extractText(message: MessageContent): string {
-  if (Array.isArray(message)) {
-    // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
-    // so we can pass them to the context generator
-    return (message as MessageContentDetail[])
-      .filter((c) => c.type === "text")
-      .map((c) => c.text)
-      .join("\n\n");
-  }
-  return message;
-}
-
-/**
- * HistoryChatEngine is a ChatEngine that uses a `ChatHistory` object
- * to keeps track of chat's message history.
- * A `ChatHistory` object is passed as a parameter for each call to the `chat` method,
- * so the state of the chat engine is preserved between calls.
- * Optionally, a `ContextGenerator` can be used to generate an additional context for each call to `chat`.
- */
-export class HistoryChatEngine {
-  llm: LLM;
-  contextGenerator?: ContextGenerator;
-
-  constructor(init?: Partial<HistoryChatEngine>) {
-    this.llm = init?.llm ?? new OpenAI();
-    this.contextGenerator = init?.contextGenerator;
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : Response,
-  >(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-    streaming?: T,
-  ): Promise<R> {
-    //Streaming option
-    if (streaming) {
-      return this.streamChat(message, chatHistory) as R;
-    }
-    const requestMessages = await this.prepareRequestMessages(
-      message,
-      chatHistory,
-    );
-    const response = await this.llm.chat(requestMessages);
-    chatHistory.addMessage(response.message);
-    return new Response(response.message.content) as R;
-  }
-
-  protected async *streamChat(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-  ): AsyncGenerator<string, void, unknown> {
-    const requestMessages = await this.prepareRequestMessages(
-      message,
-      chatHistory,
-    );
-    const response_stream = await this.llm.chat(
-      requestMessages,
-      undefined,
-      true,
-    );
-
-    var accumulator = "";
-    for await (const part of response_stream) {
-      accumulator += part;
-      yield part;
-    }
-    chatHistory.addMessage({
-      content: accumulator,
-      role: "assistant",
-    });
-    return;
-  }
-
-  private async prepareRequestMessages(
-    message: MessageContent,
-    chatHistory: ChatHistory,
-  ) {
-    chatHistory.addMessage({
-      content: message,
-      role: "user",
-    });
-    let requestMessages;
-    let context;
-    if (this.contextGenerator) {
-      const textOnly = extractText(message);
-      context = await this.contextGenerator.generate(textOnly);
-    }
-    requestMessages = await chatHistory.requestMessages(
-      context ? [context.message] : undefined,
-    );
-    return requestMessages;
-  }
-}

packages/core/src/ChatHistory.ts

@@ -1,4 +1,5 @@
-import { ChatMessage, LLM, MessageType, OpenAI } from "./llm/LLM";
+import { OpenAI } from "./llm/LLM";
+import { ChatMessage, LLM, MessageType } from "./llm/types";
 import {
   defaultSummaryPrompt,
   messagesToHistoryStr,
@@ -8,35 +9,38 @@ import {
 /**
  * A ChatHistory is used to keep the state of back and forth chat messages
  */
-export interface ChatHistory {
-  messages: ChatMessage[];
+export abstract class ChatHistory {
+  abstract get messages(): ChatMessage[];
   /**
    * Adds a message to the chat history.
    * @param message
    */
-  addMessage(message: ChatMessage): void;
+  abstract addMessage(message: ChatMessage): void;
 
   /**
    * Returns the messages that should be used as input to the LLM.
    */
-  requestMessages(transientMessages?: ChatMessage[]): Promise<ChatMessage[]>;
+  abstract requestMessages(
+    transientMessages?: ChatMessage[],
+  ): Promise<ChatMessage[]>;
 
   /**
    * Resets the chat history so that it's empty.
    */
-  reset(): void;
+  abstract reset(): void;
 
   /**
    * Returns the new messages since the last call to this function (or since calling the constructor)
    */
-  newMessages(): ChatMessage[];
+  abstract newMessages(): ChatMessage[];
 }
 
-export class SimpleChatHistory implements ChatHistory {
+export class SimpleChatHistory extends ChatHistory {
   messages: ChatMessage[];
   private messagesBefore: number;
 
   constructor(init?: Partial<SimpleChatHistory>) {
+    super();
     this.messages = init?.messages ?? [];
     this.messagesBefore = this.messages.length;
   }
@@ -60,7 +64,7 @@ export class SimpleChatHistory implements ChatHistory {
   }
 }
 
-export class SummaryChatHistory implements ChatHistory {
+export class SummaryChatHistory extends ChatHistory {
   tokensToSummarize: number;
   messages: ChatMessage[];
   summaryPrompt: SummaryPrompt;
@@ -68,6 +72,7 @@ export class SummaryChatHistory implements ChatHistory {
   private messagesBefore: number;
 
   constructor(init?: Partial<SummaryChatHistory>) {
+    super();
     this.messages = init?.messages ?? [];
     this.messagesBefore = this.messages.length;
     this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
@@ -79,6 +84,11 @@ export class SummaryChatHistory implements ChatHistory {
     }
     this.tokensToSummarize =
       this.llm.metadata.contextWindow - this.llm.metadata.maxTokens;
+    if (this.tokensToSummarize < this.llm.metadata.contextWindow * 0.25) {
+      throw new Error(
+        "The number of tokens that trigger the summarize process are less than 25% of the context window. Try lowering maxTokens or use a model with a larger context window.",
+      );
+    }
   }
 
   private async summarize(): Promise<ChatMessage> {
@@ -99,7 +109,7 @@ export class SummaryChatHistory implements ChatHistory {
       messagesToSummarize.shift();
     } while (this.llm.tokens(promptMessages) > this.tokensToSummarize);
 
-    const response = await this.llm.chat(promptMessages);
+    const response = await this.llm.chat({ messages: promptMessages });
     return { content: response.message.content, role: "memory" };
   }
 
@@ -119,6 +129,11 @@ export class SummaryChatHistory implements ChatHistory {
     return this.messages.length - 1 - index;
   }
 
+  public getLastSummary(): ChatMessage | null {
+    const lastSummaryIndex = this.getLastSummaryIndex();
+    return lastSummaryIndex ? this.messages[lastSummaryIndex] : null;
+  }
+
   private get systemMessages() {
     // get array of all system messages
     return this.messages.filter((message) => message.role === "system");
@@ -198,3 +213,12 @@ export class SummaryChatHistory implements ChatHistory {
     return newMessages;
   }
 }
+
+export function getHistory(
+  chatHistory?: ChatMessage[] | ChatHistory,
+): ChatHistory {
+  if (chatHistory instanceof ChatHistory) {
+    return chatHistory;
+  }
+  return new SimpleChatHistory({ messages: chatHistory });
+}

packages/core/src/GlobalsHelper.ts

@@ -1,6 +1,6 @@
 import { encodingForModel } from "js-tiktoken";
 
-import { v4 as uuidv4 } from "uuid";
+import { randomUUID } from "node:crypto";
 import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
 
 export enum Tokenizers {
@@ -64,7 +64,7 @@ class GlobalsHelper {
     tags?: EventTag[];
   }): Event {
     return {
-      id: uuidv4(),
+      id: randomUUID(),
       type,
       // inherit parent tags if tags not set
       tags: tags || parentEvent?.tags,

packages/core/src/Node.ts

@@ -1,7 +1,6 @@
 import _ from "lodash";
-import { createHash } from "node:crypto";
-import path from "path";
-import { v4 as uuidv4 } from "uuid";
+import { createHash, randomUUID } from "node:crypto";
+import path from "node:path";
 
 export enum NodeRelationship {
   SOURCE = "SOURCE",
@@ -49,7 +48,7 @@ export abstract class BaseNode<T extends Metadata = Metadata> {
    *
    * Set to a UUID by default.
    */
-  id_: string = uuidv4();
+  id_: string = randomUUID();
   embedding?: number[];
 
   // Metadata fields

packages/core/src/Prompt.ts

@@ -1,4 +1,4 @@
-import { ChatMessage } from "./llm/LLM";
+import { ChatMessage } from "./llm/types";
 import { SubQuestion } from "./QuestionGenerator";
 import { ToolMetadata } from "./Tool";
 

packages/core/src/PromptHelper.ts

@@ -37,6 +37,7 @@ export class PromptHelper {
   tokenizer: (text: string) => Uint32Array;
   separator = " ";
 
+  // eslint-disable-next-line max-params
   constructor(
     contextWindow = DEFAULT_CONTEXT_WINDOW,
     numOutput = DEFAULT_NUM_OUTPUTS,

packages/core/src/QueryEngine.ts

@@ -1,4 +1,4 @@
-import { v4 as uuidv4 } from "uuid";
+import { randomUUID } from "node:crypto";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   BaseQuestionGenerator,
@@ -17,16 +17,32 @@ import {
   ResponseSynthesizer,
 } from "./synthesizers";
 
+/**
+ * Parameters for sending a query.
+ */
+export interface QueryEngineParamsBase {
+  query: string;
+  parentEvent?: Event;
+}
+
+export interface QueryEngineParamsStreaming extends QueryEngineParamsBase {
+  stream: true;
+}
+
+export interface QueryEngineParamsNonStreaming extends QueryEngineParamsBase {
+  stream?: false | null;
+}
+
 /**
  * A query engine is a question answerer that can use one or more steps.
  */
 export interface BaseQueryEngine {
   /**
    * Query the query engine and get a response.
-   * @param query
-   * @param parentEvent
+   * @param params
    */
-  query(query: string, parentEvent?: Event): Promise<Response>;
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
 }
 
 /**
@@ -70,14 +86,31 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
     return this.applyNodePostprocessors(nodes);
   }
 
-  async query(query: string, parentEvent?: Event) {
-    const _parentEvent: Event = parentEvent || {
-      id: uuidv4(),
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { query, stream } = params;
+    const parentEvent: Event = params.parentEvent || {
+      id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
     };
-    const nodes = await this.retrieve(query, _parentEvent);
-    return this.responseSynthesizer.synthesize(query, nodes, _parentEvent);
+    const nodesWithScore = await this.retrieve(query, parentEvent);
+    if (stream) {
+      return this.responseSynthesizer.synthesize({
+        query,
+        nodesWithScore,
+        parentEvent,
+        stream: true,
+      });
+    }
+    return this.responseSynthesizer.synthesize({
+      query,
+      nodesWithScore,
+      parentEvent,
+    });
   }
 }
 
@@ -131,19 +164,24 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
     });
   }
 
-  async query(query: string): Promise<Response> {
+  query(params: QueryEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<Response>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { query, stream } = params;
     const subQuestions = await this.questionGen.generate(this.metadatas, query);
 
     // groups final retrieval+synthesis operation
-    const parentEvent: Event = {
-      id: uuidv4(),
+    const parentEvent: Event = params.parentEvent || {
+      id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
     };
 
     // groups all sub-queries
     const subQueryParentEvent: Event = {
-      id: uuidv4(),
+      id: randomUUID(),
       parentId: parentEvent.id,
       type: "wrapper",
       tags: ["intermediate"],
@@ -153,10 +191,22 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
       subQuestions.map((subQ) => this.querySubQ(subQ, subQueryParentEvent)),
     );
 
-    const nodes = subQNodes
+    const nodesWithScore = subQNodes
       .filter((node) => node !== null)
       .map((node) => node as NodeWithScore);
-    return this.responseSynthesizer.synthesize(query, nodes, parentEvent);
+    if (stream) {
+      return this.responseSynthesizer.synthesize({
+        query,
+        nodesWithScore,
+        parentEvent,
+        stream: true,
+      });
+    }
+    return this.responseSynthesizer.synthesize({
+      query,
+      nodesWithScore,
+      parentEvent,
+    });
   }
 
   private async querySubQ(
@@ -167,7 +217,10 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
       const question = subQ.subQuestion;
       const queryEngine = this.queryEngines[subQ.toolName];
 
-      const response = await queryEngine.query(question, parentEvent);
+      const response = await queryEngine.query({
+        query: question,
+        parentEvent,
+      });
       const responseText = response.response;
       const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
       const node = new TextNode({ text: nodeText });

packages/core/src/QuestionGenerator.ts

@@ -9,7 +9,8 @@ import {
   defaultSubQuestionPrompt,
 } from "./Prompt";
 import { ToolMetadata } from "./Tool";
-import { LLM, OpenAI } from "./llm/LLM";
+import { OpenAI } from "./llm/LLM";
+import { LLM } from "./llm/types";
 
 export interface SubQuestion {
   subQuestion: string;
@@ -41,13 +42,13 @@ export class LLMQuestionGenerator implements BaseQuestionGenerator {
     const toolsStr = buildToolsText(tools);
     const queryStr = query;
     const prediction = (
-      await this.llm.complete(
-        this.prompt({
+      await this.llm.complete({
+        prompt: this.prompt({
           toolsStr,
           queryStr,
         }),
-      )
-    ).message.content;
+      })
+    ).text;
 
     const structuredOutput = this.outputParser.parse(prediction);
 

packages/core/src/Response.ts

@@ -1,7 +1,7 @@
 import { BaseNode } from "./Node";
 
 /**
- * Respone is the output of a LLM
+ * Response is the output of a LLM
  */
 export class Response {
   response: string;

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -14,7 +14,7 @@ export enum OpenAIEmbeddingModelType {
 }
 
 export class OpenAIEmbedding extends BaseEmbedding {
-  model: OpenAIEmbeddingModelType;
+  model: OpenAIEmbeddingModelType | string;
 
   // OpenAI session params
   apiKey?: string = undefined;

packages/core/src/embeddings/index.ts

@@ -3,5 +3,6 @@ export * from "./HuggingFaceEmbedding";
 export * from "./MistralAIEmbedding";
 export * from "./MultiModalEmbedding";
 export * from "./OpenAIEmbedding";
+export { TogetherEmbedding } from "./together";
 export * from "./types";
 export * from "./utils";

packages/core/src/embeddings/together.ts

@@ -0,0 +1,16 @@
+import { OpenAIEmbedding } from "./OpenAIEmbedding";
+
+export class TogetherEmbedding extends OpenAIEmbedding {
+  override model: string;
+  constructor(init?: Partial<OpenAIEmbedding>) {
+    super({
+      apiKey: process.env.TOGETHER_API_KEY,
+      ...init,
+      additionalSessionOptions: {
+        ...init?.additionalSessionOptions,
+        baseURL: "https://api.together.xyz/v1",
+      },
+    });
+    this.model = init?.model ?? "togethercomputer/m2-bert-80M-32k-retrieval";
+  }
+}

packages/core/src/embeddings/utils.ts

@@ -66,6 +66,7 @@ export function similarity(
  * @param similarityCutoff minimum similarity score
  * @returns
  */
+// eslint-disable-next-line max-params
 export function getTopKEmbeddings(
   queryEmbedding: number[],
   embeddings: number[][],
@@ -108,6 +109,7 @@ export function getTopKEmbeddings(
   return [resultSimilarities, resultIds];
 }
 
+// eslint-disable-next-line max-params
 export function getTopKEmbeddingsLearner(
   queryEmbedding: number[],
   embeddings: number[][],
@@ -120,6 +122,7 @@ export function getTopKEmbeddingsLearner(
   // https://github.com/mljs/libsvm which itself hasn't been updated in a while
 }
 
+// eslint-disable-next-line max-params
 export function getTopKMMREmbeddings(
   queryEmbedding: number[],
   embeddings: number[][],

packages/core/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -0,0 +1,104 @@
+import { ChatHistory, getHistory } from "../../ChatHistory";
+import {
+  CondenseQuestionPrompt,
+  defaultCondenseQuestionPrompt,
+  messagesToHistoryStr,
+} from "../../Prompt";
+import { BaseQueryEngine } from "../../QueryEngine";
+import { Response } from "../../Response";
+import {
+  ServiceContext,
+  serviceContextFromDefaults,
+} from "../../ServiceContext";
+import { ChatMessage, LLM } from "../../llm";
+import { extractText, streamReducer } from "../../llm/utils";
+import {
+  ChatEngine,
+  ChatEngineParamsNonStreaming,
+  ChatEngineParamsStreaming,
+} from "./types";
+
+/**
+ * CondenseQuestionChatEngine is used in conjunction with a Index (for example VectorStoreIndex).
+ * It does two steps on taking a user's chat message: first, it condenses the chat message
+ * with the previous chat history into a question with more context.
+ * Then, it queries the underlying Index using the new question with context and returns
+ * the response.
+ * CondenseQuestionChatEngine performs well when the input is primarily questions about the
+ * underlying data. It performs less well when the chat messages are not questions about the
+ * data, or are very referential to previous context.
+ */
+
+export class CondenseQuestionChatEngine implements ChatEngine {
+  queryEngine: BaseQueryEngine;
+  chatHistory: ChatHistory;
+  llm: LLM;
+  condenseMessagePrompt: CondenseQuestionPrompt;
+
+  constructor(init: {
+    queryEngine: BaseQueryEngine;
+    chatHistory: ChatMessage[];
+    serviceContext?: ServiceContext;
+    condenseMessagePrompt?: CondenseQuestionPrompt;
+  }) {
+    this.queryEngine = init.queryEngine;
+    this.chatHistory = getHistory(init?.chatHistory);
+    this.llm = init?.serviceContext?.llm ?? serviceContextFromDefaults().llm;
+    this.condenseMessagePrompt =
+      init?.condenseMessagePrompt ?? defaultCondenseQuestionPrompt;
+  }
+
+  private async condenseQuestion(chatHistory: ChatHistory, question: string) {
+    const chatHistoryStr = messagesToHistoryStr(
+      await chatHistory.requestMessages(),
+    );
+
+    return this.llm.complete({
+      prompt: defaultCondenseQuestionPrompt({
+        question: question,
+        chatHistory: chatHistoryStr,
+      }),
+    });
+  }
+
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+    const chatHistory = params.chatHistory
+      ? getHistory(params.chatHistory)
+      : this.chatHistory;
+
+    const condensedQuestion = (
+      await this.condenseQuestion(chatHistory, extractText(message))
+    ).text;
+    chatHistory.addMessage({ content: message, role: "user" });
+
+    if (stream) {
+      const stream = await this.queryEngine.query({
+        query: condensedQuestion,
+        stream: true,
+      });
+      return streamReducer({
+        stream,
+        initialValue: "",
+        reducer: (accumulator, part) => (accumulator += part.response),
+        finished: (accumulator) => {
+          chatHistory.addMessage({ content: accumulator, role: "assistant" });
+        },
+      });
+    }
+    const response = await this.queryEngine.query({
+      query: condensedQuestion,
+    });
+    chatHistory.addMessage({ content: response.response, role: "assistant" });
+
+    return response;
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+}

packages/core/src/engines/chat/ContextChatEngine.ts

@@ -0,0 +1,113 @@
+import { randomUUID } from "node:crypto";
+import { ChatHistory, getHistory } from "../../ChatHistory";
+import { ContextSystemPrompt } from "../../Prompt";
+import { Response } from "../../Response";
+import { BaseRetriever } from "../../Retriever";
+import { Event } from "../../callbacks/CallbackManager";
+import { ChatMessage, ChatResponseChunk, LLM, OpenAI } from "../../llm";
+import { MessageContent } from "../../llm/types";
+import { extractText, streamConverter, streamReducer } from "../../llm/utils";
+import { BaseNodePostprocessor } from "../../postprocessors";
+import { DefaultContextGenerator } from "./DefaultContextGenerator";
+import {
+  ChatEngine,
+  ChatEngineParamsNonStreaming,
+  ChatEngineParamsStreaming,
+  ContextGenerator,
+} from "./types";
+
+/**
+ * ContextChatEngine uses the Index to get the appropriate context for each query.
+ * The context is stored in the system prompt, and the chat history is preserved,
+ * ideally allowing the appropriate context to be surfaced for each query.
+ */
+export class ContextChatEngine implements ChatEngine {
+  chatModel: LLM;
+  chatHistory: ChatHistory;
+  contextGenerator: ContextGenerator;
+
+  constructor(init: {
+    retriever: BaseRetriever;
+    chatModel?: LLM;
+    chatHistory?: ChatMessage[];
+    contextSystemPrompt?: ContextSystemPrompt;
+    nodePostprocessors?: BaseNodePostprocessor[];
+  }) {
+    this.chatModel =
+      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
+    this.chatHistory = getHistory(init?.chatHistory);
+    this.contextGenerator = new DefaultContextGenerator({
+      retriever: init.retriever,
+      contextSystemPrompt: init?.contextSystemPrompt,
+      nodePostprocessors: init?.nodePostprocessors,
+    });
+  }
+
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+    const chatHistory = params.chatHistory
+      ? getHistory(params.chatHistory)
+      : this.chatHistory;
+    const parentEvent: Event = {
+      id: randomUUID(),
+      type: "wrapper",
+      tags: ["final"],
+    };
+    const requestMessages = await this.prepareRequestMessages(
+      message,
+      chatHistory,
+      parentEvent,
+    );
+
+    if (stream) {
+      const stream = await this.chatModel.chat({
+        messages: requestMessages.messages,
+        parentEvent,
+        stream: true,
+      });
+      return streamConverter(
+        streamReducer({
+          stream,
+          initialValue: "",
+          reducer: (accumulator, part) => (accumulator += part.delta),
+          finished: (accumulator) => {
+            chatHistory.addMessage({ content: accumulator, role: "assistant" });
+          },
+        }),
+        (r: ChatResponseChunk) => new Response(r.delta, requestMessages.nodes),
+      );
+    }
+    const response = await this.chatModel.chat({
+      messages: requestMessages.messages,
+      parentEvent,
+    });
+    chatHistory.addMessage(response.message);
+    return new Response(response.message.content, requestMessages.nodes);
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+
+  private async prepareRequestMessages(
+    message: MessageContent,
+    chatHistory: ChatHistory,
+    parentEvent?: Event,
+  ) {
+    chatHistory.addMessage({
+      content: message,
+      role: "user",
+    });
+    const textOnly = extractText(message);
+    const context = await this.contextGenerator.generate(textOnly, parentEvent);
+    const nodes = context.nodes.map((r) => r.node);
+    const messages = await chatHistory.requestMessages(
+      context ? [context.message] : undefined,
+    );
+    return { nodes, messages };
+  }
+}

packages/core/src/engines/chat/DefaultContextGenerator.ts

@@ -0,0 +1,57 @@
+import { randomUUID } from "node:crypto";
+import { NodeWithScore, TextNode } from "../../Node";
+import { ContextSystemPrompt, defaultContextSystemPrompt } from "../../Prompt";
+import { BaseRetriever } from "../../Retriever";
+import { Event } from "../../callbacks/CallbackManager";
+import { BaseNodePostprocessor } from "../../postprocessors";
+import { Context, ContextGenerator } from "./types";
+
+export class DefaultContextGenerator implements ContextGenerator {
+  retriever: BaseRetriever;
+  contextSystemPrompt: ContextSystemPrompt;
+  nodePostprocessors: BaseNodePostprocessor[];
+
+  constructor(init: {
+    retriever: BaseRetriever;
+    contextSystemPrompt?: ContextSystemPrompt;
+    nodePostprocessors?: BaseNodePostprocessor[];
+  }) {
+    this.retriever = init.retriever;
+    this.contextSystemPrompt =
+      init?.contextSystemPrompt ?? defaultContextSystemPrompt;
+    this.nodePostprocessors = init.nodePostprocessors || [];
+  }
+
+  private applyNodePostprocessors(nodes: NodeWithScore[]) {
+    return this.nodePostprocessors.reduce(
+      (nodes, nodePostprocessor) => nodePostprocessor.postprocessNodes(nodes),
+      nodes,
+    );
+  }
+
+  async generate(message: string, parentEvent?: Event): Promise<Context> {
+    if (!parentEvent) {
+      parentEvent = {
+        id: randomUUID(),
+        type: "wrapper",
+        tags: ["final"],
+      };
+    }
+    const sourceNodesWithScore = await this.retriever.retrieve(
+      message,
+      parentEvent,
+    );
+
+    const nodes = this.applyNodePostprocessors(sourceNodesWithScore);
+
+    return {
+      message: {
+        content: this.contextSystemPrompt({
+          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
+        }),
+        role: "system",
+      },
+      nodes,
+    };
+  }
+}

packages/core/src/engines/chat/SimpleChatEngine.ts

@@ -0,0 +1,64 @@
+import { ChatHistory, getHistory } from "../../ChatHistory";
+import { Response } from "../../Response";
+import { ChatResponseChunk, LLM, OpenAI } from "../../llm";
+import { streamConverter, streamReducer } from "../../llm/utils";
+import {
+  ChatEngine,
+  ChatEngineParamsNonStreaming,
+  ChatEngineParamsStreaming,
+} from "./types";
+
+/**
+ * SimpleChatEngine is the simplest possible chat engine. Useful for using your own custom prompts.
+ */
+
+export class SimpleChatEngine implements ChatEngine {
+  chatHistory: ChatHistory;
+  llm: LLM;
+
+  constructor(init?: Partial<SimpleChatEngine>) {
+    this.chatHistory = getHistory(init?.chatHistory);
+    this.llm = init?.llm ?? new OpenAI();
+  }
+
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+  async chat(
+    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+  ): Promise<Response | AsyncIterable<Response>> {
+    const { message, stream } = params;
+
+    const chatHistory = params.chatHistory
+      ? getHistory(params.chatHistory)
+      : this.chatHistory;
+    chatHistory.addMessage({ content: message, role: "user" });
+
+    if (stream) {
+      const stream = await this.llm.chat({
+        messages: await chatHistory.requestMessages(),
+        stream: true,
+      });
+      return streamConverter(
+        streamReducer({
+          stream,
+          initialValue: "",
+          reducer: (accumulator, part) => (accumulator += part.delta),
+          finished: (accumulator) => {
+            chatHistory.addMessage({ content: accumulator, role: "assistant" });
+          },
+        }),
+        (r: ChatResponseChunk) => new Response(r.delta),
+      );
+    }
+
+    const response = await this.llm.chat({
+      messages: await chatHistory.requestMessages(),
+    });
+    chatHistory.addMessage(response.message);
+    return new Response(response.message.content);
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+}

packages/core/src/engines/chat/index.ts

@@ -0,0 +1,4 @@
+export { CondenseQuestionChatEngine } from "./CondenseQuestionChatEngine";
+export { ContextChatEngine } from "./ContextChatEngine";
+export { SimpleChatEngine } from "./SimpleChatEngine";
+export * from "./types";

packages/core/src/engines/chat/types.ts

@@ -0,0 +1,54 @@
+import { ChatHistory } from "../../ChatHistory";
+import { NodeWithScore } from "../../Node";
+import { Response } from "../../Response";
+import { Event } from "../../callbacks/CallbackManager";
+import { ChatMessage } from "../../llm";
+import { MessageContent } from "../../llm/types";
+
+/**
+ * Represents the base parameters for ChatEngine.
+ */
+export interface ChatEngineParamsBase {
+  message: MessageContent;
+  /**
+   * Optional chat history if you want to customize the chat history.
+   */
+  chatHistory?: ChatMessage[] | ChatHistory;
+}
+
+export interface ChatEngineParamsStreaming extends ChatEngineParamsBase {
+  stream: true;
+}
+
+export interface ChatEngineParamsNonStreaming extends ChatEngineParamsBase {
+  stream?: false | null;
+}
+
+/**
+ * A ChatEngine is used to handle back and forth chats between the application and the LLM.
+ */
+export interface ChatEngine {
+  /**
+   * Send message along with the class's current chat history to the LLM.
+   * @param params
+   */
+  chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;
+  chat(params: ChatEngineParamsNonStreaming): Promise<Response>;
+
+  /**
+   * Resets the chat history so that it's empty.
+   */
+  reset(): void;
+}
+
+export interface Context {
+  message: ChatMessage;
+  nodes: NodeWithScore[];
+}
+
+/**
+ * A ContextGenerator is used to generate a context based on a message's text content
+ */
+export interface ContextGenerator {
+  generate(message: string, parentEvent?: Event): Promise<Context>;
+}

packages/core/src/index.ts

@@ -1,4 +1,3 @@
-export * from "./ChatEngine";
 export * from "./ChatHistory";
 export * from "./GlobalsHelper";
 export * from "./Node";
@@ -15,6 +14,7 @@ export * from "./Tool";
 export * from "./callbacks/CallbackManager";
 export * from "./constants";
 export * from "./embeddings";
+export * from "./engines/chat";
 export * from "./indices";
 export * from "./llm";
 export * from "./nodeParsers";

packages/core/src/indices/BaseIndex.ts

@@ -1,4 +1,4 @@
-import { v4 as uuidv4 } from "uuid";
+import { randomUUID } from "node:crypto";
 import { BaseNode, Document, jsonToNode } from "../Node";
 import { BaseQueryEngine } from "../QueryEngine";
 import { BaseRetriever } from "../Retriever";
@@ -16,7 +16,7 @@ export abstract class IndexStruct {
   indexId: string;
   summary?: string;
 
-  constructor(indexId = uuidv4(), summary = undefined) {
+  constructor(indexId = randomUUID(), summary = undefined) {
     this.indexId = indexId;
     this.summary = summary;
   }

packages/core/src/indices/keyword/KeywordTableIndex.ts

@@ -149,12 +149,12 @@ export class KeywordTableIndex extends BaseIndex<KeywordTable> {
     text: string,
     serviceContext: ServiceContext,
   ): Promise<Set<string>> {
-    const response = await serviceContext.llm.complete(
-      defaultKeywordExtractPrompt({
+    const response = await serviceContext.llm.complete({
+      prompt: defaultKeywordExtractPrompt({
         context: text,
       }),
-    );
-    return extractKeywordsGivenResponse(response.message.content, "KEYWORDS:");
+    });
+    return extractKeywordsGivenResponse(response.text, "KEYWORDS:");
   }
 
   /**

packages/core/src/indices/keyword/KeywordTableIndexRetriever.ts

@@ -86,16 +86,13 @@ abstract class BaseKeywordTableRetriever implements BaseRetriever {
 // Extracts keywords using LLMs.
 export class KeywordTableLLMRetriever extends BaseKeywordTableRetriever {
   async getKeywords(query: string): Promise<string[]> {
-    const response = await this.serviceContext.llm.complete(
-      this.queryKeywordExtractTemplate({
+    const response = await this.serviceContext.llm.complete({
+      prompt: this.queryKeywordExtractTemplate({
         question: query,
         maxKeywords: this.maxKeywordsPerQuery,
       }),
-    );
-    const keywords = extractKeywordsGivenResponse(
-      response.message.content,
-      "KEYWORDS:",
-    );
+    });
+    const keywords = extractKeywordsGivenResponse(response.text, "KEYWORDS:");
     return [...keywords];
   }
 }

packages/core/src/indices/summary/SummaryIndexRetriever.ts

@@ -61,6 +61,7 @@ export class SummaryIndexLLMRetriever implements BaseRetriever {
   parseChoiceSelectAnswerFn: ChoiceSelectParserFunction;
   serviceContext: ServiceContext;
 
+  // eslint-disable-next-line max-params
   constructor(
     index: SummaryIndex,
     choiceSelectPrompt?: ChoiceSelectPrompt,
@@ -89,8 +90,10 @@ export class SummaryIndexLLMRetriever implements BaseRetriever {
       const fmtBatchStr = this.formatNodeBatchFn(nodesBatch);
       const input = { context: fmtBatchStr, query: query };
       const rawResponse = (
-        await this.serviceContext.llm.complete(this.choiceSelectPrompt(input))
-      ).message.content;
+        await this.serviceContext.llm.complete({
+          prompt: this.choiceSelectPrompt(input),
+        })
+      ).text;
 
       // parseResult is a map from doc number to relevance score
       const parseResult = this.parseChoiceSelectAnswerFn(

packages/core/src/llm/LLM.ts

@@ -10,8 +10,7 @@ import {
 
 import { ChatCompletionMessageParam } from "openai/resources";
 import { LLMOptions } from "portkey-ai";
-import { MessageContent } from "../ChatEngine";
-import { globalsHelper, Tokenizers } from "../GlobalsHelper";
+import { Tokenizers, globalsHelper } from "../GlobalsHelper";
 import {
   ANTHROPIC_AI_PROMPT,
   ANTHROPIC_HUMAN_PROMPT,
@@ -25,81 +24,19 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure";
-import { getOpenAISession, OpenAISession } from "./openai";
-import { getPortkeySession, PortkeySession } from "./portkey";
+import { BaseLLM } from "./base";
+import { OpenAISession, getOpenAISession } from "./openai";
+import { PortkeySession, getPortkeySession } from "./portkey";
 import { ReplicateSession } from "./replicate";
-
-export type MessageType =
-  | "user"
-  | "assistant"
-  | "system"
-  | "generic"
-  | "function"
-  | "memory";
-
-export interface ChatMessage {
-  content: any;
-  role: MessageType;
-}
-
-export interface ChatResponse {
-  message: ChatMessage;
-  raw?: Record<string, any>;
-  delta?: string;
-}
-
-// NOTE in case we need CompletionResponse to diverge from ChatResponse in the future
-export type CompletionResponse = ChatResponse;
-
-export interface LLMMetadata {
-  model: string;
-  temperature: number;
-  topP: number;
-  maxTokens?: number;
-  contextWindow: number;
-  tokenizer: Tokenizers | undefined;
-}
-
-/**
- * Unified language model interface
- */
-export interface LLM {
-  metadata: LLMMetadata;
-  // Whether a LLM has streaming support
-  hasStreaming: boolean;
-  /**
-   * Get a chat response from the LLM
-   * @param messages
-   *
-   * The return type of chat() and complete() are set by the "streaming" parameter being set to True.
-   */
-  chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-    streaming?: T,
-  ): Promise<R>;
-
-  /**
-   * Get a prompt completion from the LLM
-   * @param prompt the prompt to complete
-   */
-  complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: MessageContent,
-    parentEvent?: Event,
-    streaming?: T,
-  ): Promise<R>;
-
-  /**
-   * Calculates the number of tokens needed for the given chat messages
-   */
-  tokens(messages: ChatMessage[]): number;
-}
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMMetadata,
+  MessageType,
+} from "./types";
 
 export const GPT4_MODELS = {
   "gpt-4": { contextWindow: 8192 },
@@ -125,17 +62,15 @@ export const ALL_AVAILABLE_OPENAI_MODELS = {
 /**
  * OpenAI LLM implementation
  */
-export class OpenAI implements LLM {
-  hasStreaming: boolean = true;
-
+export class OpenAI extends BaseLLM {
   // Per completion OpenAI params
-  model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS;
+  model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
   temperature: number;
   topP: number;
   maxTokens?: number;
   additionalChatOptions?: Omit<
     Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
-    "max_tokens" | "messages" | "model" | "temperature" | "top_p" | "streaming"
+    "max_tokens" | "messages" | "model" | "temperature" | "top_p" | "stream"
   >;
 
   // OpenAI session params
@@ -155,6 +90,7 @@ export class OpenAI implements LLM {
       azure?: AzureOpenAIConfig;
     },
   ) {
+    super();
     this.model = init?.model ?? "gpt-3.5-turbo";
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 1;
@@ -205,12 +141,16 @@ export class OpenAI implements LLM {
   }
 
   get metadata() {
+    const contextWindow =
+      ALL_AVAILABLE_OPENAI_MODELS[
+        this.model as keyof typeof ALL_AVAILABLE_OPENAI_MODELS
+      ]?.contextWindow ?? 1024;
     return {
       model: this.model,
       temperature: this.temperature,
       topP: this.topP,
       maxTokens: this.maxTokens,
-      contextWindow: ALL_AVAILABLE_OPENAI_MODELS[this.model].contextWindow,
+      contextWindow,
       tokenizer: Tokenizers.CL100K_BASE,
     };
   }
@@ -247,10 +187,14 @@ export class OpenAI implements LLM {
     }
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(messages: ChatMessage[], parentEvent?: Event, streaming?: T): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
       model: this.model,
       temperature: this.temperature,
@@ -266,11 +210,8 @@ export class OpenAI implements LLM {
       ...this.additionalChatOptions,
     };
     // Streaming
-    if (streaming) {
-      if (!this.hasStreaming) {
-        throw Error("No streaming support for this LLM.");
-      }
-      return this.streamChat(messages, parentEvent) as R;
+    if (stream) {
+      return this.streamChat(params);
     }
     // Non-streaming
     const response = await this.session.openai.chat.completions.create({
@@ -281,27 +222,13 @@ export class OpenAI implements LLM {
     const content = response.choices[0].message?.content ?? "";
     return {
       message: { content, role: response.choices[0].message.role },
-    } as R;
+    };
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    );
-  }
-
-  //We can wrap a stream in a generator to add some additional logging behavior
-  //For future edits: syntax for generator type is <typeof Yield, typeof Return, typeof Accept>
-  //"typeof Accept" refers to what types you'll accept when you manually call generator.next(<AcceptType>)
-  protected async *streamChat(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
+  protected async *streamChat({
+    messages,
+    parentEvent,
+  }: LLMChatParamsStreaming): AsyncIterable<ChatResponseChunk> {
     const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
       model: this.model,
       temperature: this.temperature,
@@ -335,6 +262,7 @@ export class OpenAI implements LLM {
           type: "llmPredict" as EventType,
         };
 
+    // TODO: add callback to streamConverter and use streamConverter here
     //Indices
     var idx_counter: number = 0;
     for await (const part of chunk_stream) {
@@ -356,18 +284,12 @@ export class OpenAI implements LLM {
 
       idx_counter++;
 
-      yield part.choices[0].delta.content ? part.choices[0].delta.content : "";
+      yield {
+        delta: part.choices[0].delta.content ?? "",
+      };
     }
     return;
   }
-
-  //streamComplete doesn't need to be async because it's child function is already async
-  protected streamComplete(
-    query: string,
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: query, role: "user" }], parentEvent);
-  }
 }
 
 export const ALL_AVAILABLE_LLAMADEUCE_MODELS = {
@@ -425,16 +347,16 @@ export enum DeuceChatStrategy {
 /**
  * Llama2 LLM implementation
  */
-export class LlamaDeuce implements LLM {
+export class LlamaDeuce extends BaseLLM {
   model: keyof typeof ALL_AVAILABLE_LLAMADEUCE_MODELS;
   chatStrategy: DeuceChatStrategy;
   temperature: number;
   topP: number;
   maxTokens?: number;
   replicateSession: ReplicateSession;
-  hasStreaming: boolean;
 
   constructor(init?: Partial<LlamaDeuce>) {
+    super();
     this.model = init?.model ?? "Llama-2-70b-chat-4bit";
     this.chatStrategy =
       init?.chatStrategy ??
@@ -447,7 +369,6 @@ export class LlamaDeuce implements LLM {
       init?.maxTokens ??
       ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model].contextWindow; // For Replicate, the default is 500 tokens which is too low.
     this.replicateSession = init?.replicateSession ?? new ReplicateSession();
-    this.hasStreaming = init?.hasStreaming ?? false;
   }
 
   tokens(messages: ChatMessage[]): number {
@@ -592,10 +513,14 @@ If a question does not make any sense, or is not factually coherent, explain why
     };
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(messages: ChatMessage[], _parentEvent?: Event, streaming?: T): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     const api = ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model]
       .replicateApi as `${string}/${string}:${string}`;
 
@@ -617,6 +542,9 @@ If a question does not make any sense, or is not factually coherent, explain why
     }
 
     //TODO: Add streaming for this
+    if (stream) {
+      throw new Error("Streaming not supported for LlamaDeuce");
+    }
 
     //Non-streaming
     const response = await this.replicateSession.replicate.run(
@@ -629,14 +557,7 @@ If a question does not make any sense, or is not factually coherent, explain why
         //^ We need to do this because Replicate returns a list of strings (for streaming functionality which is not exposed by the run function)
         role: "assistant",
       },
-    } as R;
-  }
-
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
-    return this.chat([{ content: prompt, role: "user" }], parentEvent);
+    };
   }
 }
 
@@ -650,9 +571,7 @@ export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
  * Anthropic LLM implementation
  */
 
-export class Anthropic implements LLM {
-  hasStreaming: boolean = true;
-
+export class Anthropic extends BaseLLM {
   // Per completion Anthropic params
   model: keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS;
   temperature: number;
@@ -668,6 +587,7 @@ export class Anthropic implements LLM {
   callbackManager?: CallbackManager;
 
   constructor(init?: Partial<Anthropic>) {
+    super();
     this.model = init?.model ?? "claude-2";
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 0.999; // Per Ben Mann
@@ -719,20 +639,17 @@ export class Anthropic implements LLM {
     );
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     //Streaming
-    if (streaming) {
-      if (!this.hasStreaming) {
-        throw Error("No streaming support for this LLM.");
-      }
-      return this.streamChat(messages, parentEvent) as R;
+    if (stream) {
+      return this.streamChat(messages, parentEvent);
     }
 
     //Non-streaming
@@ -748,13 +665,13 @@ export class Anthropic implements LLM {
       message: { content: response.completion.trimStart(), role: "assistant" },
       //^ We're trimming the start because Anthropic often starts with a space in the response
       // That space will be re-added when we generate the next prompt.
-    } as R;
+    };
   }
 
   protected async *streamChat(
     messages: ChatMessage[],
     parentEvent?: Event | undefined,
-  ): AsyncGenerator<string, void, unknown> {
+  ): AsyncIterable<ChatResponseChunk> {
     // AsyncIterable<AnthropicStreamToken>
     const stream: AsyncIterable<AnthropicStreamToken> =
       await this.session.anthropic.completions.create({
@@ -771,40 +688,13 @@ export class Anthropic implements LLM {
       //TODO: LLM Stream Callback, pending re-work.
 
       idx_counter++;
-      yield part.completion;
+      yield { delta: part.completion };
     }
     return;
   }
-
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: string,
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    if (streaming) {
-      return this.streamComplete(prompt, parentEvent) as R;
-    }
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    ) as R;
-  }
-
-  protected streamComplete(
-    prompt: string,
-    parentEvent?: Event | undefined,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: prompt, role: "user" }], parentEvent);
-  }
 }
 
-export class Portkey implements LLM {
-  hasStreaming: boolean = true;
-
+export class Portkey extends BaseLLM {
   apiKey?: string = undefined;
   baseURL?: string = undefined;
   mode?: string = undefined;
@@ -813,6 +703,7 @@ export class Portkey implements LLM {
   callbackManager?: CallbackManager;
 
   constructor(init?: Partial<Portkey>) {
+    super();
     this.apiKey = init?.apiKey;
     this.baseURL = init?.baseURL;
     this.mode = init?.mode;
@@ -834,50 +725,34 @@ export class Portkey implements LLM {
     throw new Error("metadata not implemented for Portkey");
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-    streaming?: T,
-    params?: Record<string, any>,
-  ): Promise<R> {
-    if (streaming) {
-      return this.streamChat(messages, parentEvent, params) as R;
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream, extraParams } = params;
+    if (stream) {
+      return this.streamChat(messages, parentEvent, extraParams);
     } else {
-      const resolvedParams = params || {};
+      const bodyParams = extraParams || {};
       const response = await this.session.portkey.chatCompletions.create({
         messages,
-        ...resolvedParams,
+        ...bodyParams,
       });
 
       const content = response.choices[0].message?.content ?? "";
       const role = response.choices[0].message?.role || "assistant";
-      return { message: { content, role: role as MessageType } } as R;
+      return { message: { content, role: role as MessageType } };
     }
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: string,
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    );
-  }
-
   async *streamChat(
     messages: ChatMessage[],
     parentEvent?: Event,
     params?: Record<string, any>,
-  ): AsyncGenerator<string, void, unknown> {
+  ): AsyncIterable<ChatResponseChunk> {
     // Wrapping the stream in a callback.
     const onLLMStream = this.callbackManager?.onLLMStream
       ? this.callbackManager.onLLMStream
@@ -915,15 +790,8 @@ export class Portkey implements LLM {
 
       idx_counter++;
 
-      yield part.choices[0].delta?.content ?? "";
+      yield { delta: part.choices[0].delta?.content ?? "" };
     }
     return;
   }
-
-  streamComplete(
-    query: string,
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: query, role: "user" }], parentEvent);
-  }
 }

packages/core/src/llm/base.ts

@@ -0,0 +1,53 @@
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+} from "./types";
+import { streamConverter } from "./utils";
+
+export abstract class BaseLLM implements LLM {
+  abstract metadata: LLMMetadata;
+
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const { prompt, parentEvent, stream } = params;
+    if (stream) {
+      const stream = await this.chat({
+        messages: [{ content: prompt, role: "user" }],
+        parentEvent,
+        stream: true,
+      });
+      return streamConverter(stream, (chunk) => {
+        return {
+          text: chunk.delta,
+        };
+      });
+    }
+    const chatResponse = await this.chat({
+      messages: [{ content: prompt, role: "user" }],
+      parentEvent,
+    });
+    return { text: chatResponse.message.content as string };
+  }
+
+  abstract chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  abstract chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+
+  abstract tokens(messages: ChatMessage[]): number;
+}

packages/core/src/llm/index.ts

@@ -1,3 +1,5 @@
 export * from "./LLM";
 export * from "./mistral";
 export { Ollama } from "./ollama";
+export { TogetherLLM } from "./together";
+export * from "./types";

packages/core/src/llm/mistral.ts

@@ -4,7 +4,14 @@ import {
   EventType,
   StreamCallbackResponse,
 } from "../callbacks/CallbackManager";
-import { ChatMessage, ChatResponse, LLM } from "./LLM";
+import { BaseLLM } from "./base";
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+} from "./types";
 
 export const ALL_AVAILABLE_MISTRAL_MODELS = {
   "mistral-tiny": { contextWindow: 32000 },
@@ -41,9 +48,7 @@ export class MistralAISession {
 /**
  * MistralAI LLM implementation
  */
-export class MistralAI implements LLM {
-  hasStreaming: boolean = true;
-
+export class MistralAI extends BaseLLM {
   // Per completion MistralAI params
   model: keyof typeof ALL_AVAILABLE_MISTRAL_MODELS;
   temperature: number;
@@ -57,6 +62,7 @@ export class MistralAI implements LLM {
   private session: MistralAISession;
 
   constructor(init?: Partial<MistralAI>) {
+    super();
     this.model = init?.model ?? "mistral-small";
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 1;
@@ -94,16 +100,17 @@ export class MistralAI implements LLM {
     };
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(messages: ChatMessage[], parentEvent?: Event, streaming?: T): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, stream } = params;
     // Streaming
-    if (streaming) {
-      if (!this.hasStreaming) {
-        throw Error("No streaming support for this LLM.");
-      }
-      return this.streamChat(messages, parentEvent) as R;
+    if (stream) {
+      return this.streamChat(params);
     }
     // Non-streaming
     const client = await this.session.getClient();
@@ -111,24 +118,13 @@ export class MistralAI implements LLM {
     const message = response.choices[0].message;
     return {
       message,
-    } as R;
+    };
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    );
-  }
-
-  protected async *streamChat(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
+  protected async *streamChat({
+    messages,
+    parentEvent,
+  }: LLMChatParamsStreaming): AsyncIterable<ChatResponseChunk> {
     //Now let's wrap our stream in a callback
     const onLLMStream = this.callbackManager?.onLLMStream
       ? this.callbackManager.onLLMStream
@@ -163,16 +159,10 @@ export class MistralAI implements LLM {
 
       idx_counter++;
 
-      yield part.choices[0].delta.content ?? "";
+      yield {
+        delta: part.choices[0].delta.content ?? "",
+      };
     }
     return;
   }
-
-  //streamComplete doesn't need to be async because it's child function is already async
-  protected streamComplete(
-    query: string,
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: query, role: "user" }], parentEvent);
-  }
 }

packages/core/src/llm/ollama.ts

@@ -1,11 +1,27 @@
 import { ok } from "node:assert";
-import { MessageContent } from "../ChatEngine";
 import { CallbackManager, Event } from "../callbacks/CallbackManager";
 import { BaseEmbedding } from "../embeddings";
-import { ChatMessage, ChatResponse, LLM, LLMMetadata } from "./LLM";
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+} from "./types";
 
-const messageAccessor = (data: any) => data.message.content;
-const completionAccessor = (data: any) => data.response;
+const messageAccessor = (data: any): ChatResponseChunk => {
+  return {
+    delta: data.message.content,
+  };
+};
+const completionAccessor = (data: any): CompletionResponse => {
+  return { text: data.response };
+};
 
 // https://github.com/jmorganca/ollama
 export class Ollama extends BaseEmbedding implements LLM {
@@ -43,21 +59,21 @@ export class Ollama extends BaseEmbedding implements LLM {
     };
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     const payload = {
       model: this.model,
       messages: messages.map((message) => ({
         role: message.role,
         content: message.content,
       })),
-      stream: !!streaming,
+      stream: !!stream,
       options: {
         temperature: this.temperature,
         num_ctx: this.contextWindow,
@@ -73,7 +89,7 @@ export class Ollama extends BaseEmbedding implements LLM {
         "Content-Type": "application/json",
       },
     });
-    if (!streaming) {
+    if (!stream) {
       const raw = await response.json();
       const { message } = raw;
       return {
@@ -82,20 +98,20 @@ export class Ollama extends BaseEmbedding implements LLM {
           content: message.content,
         },
         raw,
-      } satisfies ChatResponse as R;
+      };
     } else {
       const stream = response.body;
       ok(stream, "stream is null");
       ok(stream instanceof ReadableStream, "stream is not readable");
-      return this.streamChat(stream, messageAccessor, parentEvent) as R;
+      return this.streamChat(stream, messageAccessor, parentEvent);
     }
   }
 
-  private async *streamChat(
+  private async *streamChat<T>(
     stream: ReadableStream<Uint8Array>,
-    accessor: (data: any) => string,
+    accessor: (data: any) => T,
     parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
+  ): AsyncIterable<T> {
     const reader = stream.getReader();
     while (true) {
       const { done, value } = await reader.read();
@@ -119,18 +135,20 @@ export class Ollama extends BaseEmbedding implements LLM {
     }
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: MessageContent,
-    parentEvent?: Event | undefined,
-    streaming?: T | undefined,
-  ): Promise<R> {
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const { prompt, parentEvent, stream } = params;
     const payload = {
       model: this.model,
       prompt: prompt,
-      stream: !!streaming,
+      stream: !!stream,
       options: {
         temperature: this.temperature,
         num_ctx: this.contextWindow,
@@ -146,20 +164,17 @@ export class Ollama extends BaseEmbedding implements LLM {
         "Content-Type": "application/json",
       },
     });
-    if (!streaming) {
+    if (!stream) {
       const raw = await response.json();
       return {
-        message: {
-          role: "assistant",
-          content: raw.response,
-        },
+        text: raw.response,
         raw,
-      } satisfies ChatResponse as R;
+      };
     } else {
       const stream = response.body;
       ok(stream, "stream is null");
       ok(stream instanceof ReadableStream, "stream is not readable");
-      return this.streamChat(stream, completionAccessor, parentEvent) as R;
+      return this.streamChat(stream, completionAccessor, parentEvent);
     }
   }
 

packages/core/src/llm/together.ts

@@ -0,0 +1,14 @@
+import { OpenAI } from "./LLM";
+
+export class TogetherLLM extends OpenAI {
+  constructor(init?: Partial<OpenAI>) {
+    super({
+      ...init,
+      apiKey: process.env.TOGETHER_API_KEY,
+      additionalSessionOptions: {
+        ...init?.additionalSessionOptions,
+        baseURL: "https://api.together.xyz/v1",
+      },
+    });
+  }
+}

packages/core/src/llm/types.ts

@@ -0,0 +1,110 @@
+import { Tokenizers } from "../GlobalsHelper";
+import { Event } from "../callbacks/CallbackManager";
+
+/**
+ * Unified language model interface
+ */
+export interface LLM {
+  metadata: LLMMetadata;
+  /**
+   * Get a chat response from the LLM
+   *
+   * @param params
+   */
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+
+  /**
+   * Get a prompt completion from the LLM
+   * @param params
+   */
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+
+  /**
+   * Calculates the number of tokens needed for the given chat messages
+   */
+  tokens(messages: ChatMessage[]): number;
+}
+
+export type MessageType =
+  | "user"
+  | "assistant"
+  | "system"
+  | "generic"
+  | "function"
+  | "memory";
+
+export interface ChatMessage {
+  // TODO: use MessageContent
+  content: any;
+  role: MessageType;
+}
+
+export interface ChatResponse {
+  message: ChatMessage;
+  raw?: Record<string, any>;
+}
+
+export interface ChatResponseChunk {
+  delta: string;
+}
+
+export interface CompletionResponse {
+  text: string;
+  raw?: Record<string, any>;
+}
+
+export interface LLMMetadata {
+  model: string;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  contextWindow: number;
+  tokenizer: Tokenizers | undefined;
+}
+
+export interface LLMChatParamsBase {
+  messages: ChatMessage[];
+  parentEvent?: Event;
+  extraParams?: Record<string, any>;
+}
+
+export interface LLMChatParamsStreaming extends LLMChatParamsBase {
+  stream: true;
+}
+
+export interface LLMChatParamsNonStreaming extends LLMChatParamsBase {
+  stream?: false | null;
+}
+
+export interface LLMCompletionParamsBase {
+  prompt: any;
+  parentEvent?: Event;
+}
+
+export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
+  stream: true;
+}
+
+export interface LLMCompletionParamsNonStreaming
+  extends LLMCompletionParamsBase {
+  stream?: false | null;
+}
+
+export interface MessageContentDetail {
+  type: "text" | "image_url";
+  text?: string;
+  image_url?: { url: string };
+}
+
+/**
+ * Extended type for the content of a message that allows for multi-modal messages.
+ */
+export type MessageContent = string | MessageContentDetail[];

packages/core/src/llm/utils.ts

@@ -0,0 +1,44 @@
+import { MessageContent, MessageContentDetail } from "./types";
+
+export async function* streamConverter<S, D>(
+  stream: AsyncIterable<S>,
+  converter: (s: S) => D,
+): AsyncIterable<D> {
+  for await (const data of stream) {
+    yield converter(data);
+  }
+}
+
+export async function* streamReducer<S, D>(params: {
+  stream: AsyncIterable<S>;
+  reducer: (previousValue: D, currentValue: S) => D;
+  initialValue: D;
+  finished?: (value: D | undefined) => void;
+}): AsyncIterable<S> {
+  let value = params.initialValue;
+  for await (const data of params.stream) {
+    value = params.reducer(value, data);
+    yield data;
+  }
+  if (params.finished) {
+    params.finished(value);
+  }
+}
+/**
+ * Extracts just the text from a multi-modal message or the message itself if it's just text.
+ *
+ * @param message The message to extract text from.
+ * @returns The extracted text
+ */
+
+export function extractText(message: MessageContent): string {
+  if (Array.isArray(message)) {
+    // message is of type MessageContentDetail[] - retrieve just the text parts and concatenate them
+    // so we can pass them to the context generator
+    return (message as MessageContentDetail[])
+      .filter((c) => c.type === "text")
+      .map((c) => c.text)
+      .join("\n\n");
+  }
+  return message;
+}

packages/core/src/readers/SimpleMongoReader.ts

@@ -36,6 +36,7 @@ export class SimpleMongoReader implements BaseReader {
    * @returns {Promise<Document[]>}
    * @throws If a field specified in fieldNames or metadataNames is not found in a MongoDB document.
    */
+  // eslint-disable-next-line max-params
   public async loadData(
     dbName: string,
     collectionName: string,

packages/core/src/storage/StorageContext.ts

@@ -1,4 +1,4 @@
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "./FileSystem";
 import {
   DEFAULT_FS,

packages/core/src/storage/docStore/SimpleDocumentStore.ts

@@ -1,5 +1,5 @@
 import _ from "lodash";
-import * as path from "path";
+import path from "path";
 import { GenericFileSystem } from "../FileSystem";
 import {
   DEFAULT_DOC_STORE_PERSIST_FILENAME,