RELEASING: Releasing 2 package(s)

Releases: create-llama@0.0.14 llamaindex@0.0.45 [skip ci]
docs: update README.md (#376 )
2026-07-14 20:48:32 -04:00 · 2024-01-14 18:14:55 -06:00 · 2024-01-14 18:10:55 -06:00 · 2024-01-14 18:10:33 -06:00 · 2024-01-12 18:49:53 -06:00 · 2024-01-12 18:41:48 -06:00
73 changed files with 1653 additions and 1366 deletions
@@ -1,7 +1,7 @@
 {
  "$schema": "https://unpkg.com/@changesets/config@2.3.1/schema.json",
  "changelog": "@changesets/cli/changelog",
-  "commit": false,
+  "commit": true,
  "fixed": [],
  "linked": [],
  "access": "public",
@@ -18,3 +18,21 @@ jobs:
        run: pnpm install
      - name: Run tests
        run: pnpm run test
+  typecheck:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+      - name: Install dependencies
+        run: pnpm install
+      - name: Build
+        run: pnpm run build
+        working-directory: ./packages/core
+      - name: Run Type Check
+        run: pnpm run type-check
@@ -37,6 +37,7 @@ yarn-error.log*
 .vercel

 dist/
+lib/

 # vs code
 .vscode/launch.json
@@ -1,3 +1,4 @@
 apps/docs/i18n
 pnpm-lock.yaml
-
+lib/
+dist/
@@ -1,5 +1,10 @@
 # LlamaIndex.TS

+[![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
+
 LlamaIndex is a data framework for your LLM application.

 Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.
@@ -7,6 +7,7 @@
 # Generated files
 .docusaurus
 .cache-loader
+lib

 # Misc
 .DS_Store
@@ -28,8 +28,10 @@
  },
  "devDependencies": {
    "@docusaurus/module-type-aliases": "2.4.3",
+    "@docusaurus/theme-classic": "^2.4.3",
    "@docusaurus/types": "^2.4.3",
    "@tsconfig/docusaurus": "^2.0.1",
+    "@types/node": "^18.19.6",
    "docusaurus-plugin-typedoc": "^0.19.2",
    "typedoc": "^0.24.8",
    "typedoc-plugin-markdown": "^3.16.0",
@@ -1,7 +1,11 @@
 {
  // This file is not used in compilation. It is here just for a nice editor experience.
-  "extends": "@tsconfig/docusaurus/tsconfig.json",
+  "extends": "./node_modules/@tsconfig/docusaurus/tsconfig.json",
  "compilerOptions": {
-    "baseUrl": "."
+    "baseUrl": ".",
+    "composite": true,
+    "incremental": true,
+    "outDir": "./lib",
+    "tsBuildInfoFile": "./lib/.tsbuildinfo"
  }
 }
@@ -10,7 +10,7 @@ const collectionName = "movie_reviews";
 async function main() {
  try {
    const reader = new PapaCSVReader(false);
-    const docs = await reader.loadData("astradb/data/movie_reviews.csv");
+    const docs = await reader.loadData("../data/movie_reviews.csv");

    const astraVS = new AstraDBVectorStore();
    await astraVS.create(collectionName, {
@@ -0,0 +1,13 @@
+# Chroma Vector Store Example
+
+How to run `examples/chromadb/test.ts`:
+
+Export your OpenAI API Key using `export OPEN_API_KEY=insert your api key here`
+
+If you haven't installed chromadb, run `pip install chromadb`. Start the server using `chroma run`.
+
+Now, open a new terminal window and inside `examples`, run `pnpx ts-node chromadb/test.ts`.
+
+Here's the output for the input query `Tell me about Godfrey Cheshire's rating of La Sapienza.`:
+
+`Godfrey Cheshire gave La Sapienza a rating of 4 out of 4, describing it as fresh and the most astonishing and important movie to emerge from France in quite some time.`
@@ -0,0 +1,40 @@
+import {
+  ChromaVectorStore,
+  PapaCSVReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+
+const collectionName = "movie_reviews";
+
+async function main() {
+  const sourceFile: string = "./data/movie_reviews.csv";
+
+  try {
+    console.log(`Loading data from ${sourceFile}`);
+    const reader = new PapaCSVReader(false, ", ", "\n", {
+      header: true,
+    });
+    const docs = await reader.loadData(sourceFile);
+
+    console.log("Creating ChromaDB vector store");
+    const chromaVS = new ChromaVectorStore({ collectionName });
+    const ctx = await storageContextFromDefaults({ vectorStore: chromaVS });
+
+    console.log("Embedding documents and adding to index");
+    const index = await VectorStoreIndex.fromDocuments(docs, {
+      storageContext: ctx,
+    });
+
+    console.log("Querying index");
+    const queryEngine = index.asQueryEngine();
+    const response = await queryEngine.query(
+      "Tell me about Godfrey Cheshire's rating of La Sapienza.",
+    );
+    console.log(response.toString());
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+main();
@@ -1,101 +1,101 @@
-title,reviewid,creationdate,criticname,originalscore,reviewstate,reviewtext
-Beavers,1145982,2003-05-23,Ivan M. Lincoln,3.5/4,fresh,"Timed to be just long enough for most youngsters' brief attention spans -- and it's packed with plenty of interesting activity, both on land and under the water."
-Blood Mask,1636744,2007-06-02,The Foywonder,1/5,rotten,"It doesn't matter if a movie costs 300 million or only 300 dollars; good is good and bad is bad, and Bloodmask: The Possession of Nicole Lameroux is just plain bad."
-City Hunter: Shinjuku Private Eyes,2590987,2019-05-28,Reuben Baron,,fresh,"The choreography is so precise and lifelike at points one might wonder whether the movie was rotoscoped, but no live-action reference footage was used. The quality is due to the skill of the animators and Kodama's love for professional wrestling."
-City Hunter: Shinjuku Private Eyes,2558908,2019-02-14,Matt Schley,2.5/5,rotten,The film's out-of-touch attempts at humor may find them hunting for the reason the franchise was so popular in the first place.
-Dangerous Men,2504681,2018-08-29,Pat Padua,,fresh,Its clumsy determination is endearing and sometimes wildly entertaining
-Dangerous Men,2299284,2015-12-13,Eric Melin,4/5,fresh,"With every new minute, there's another head-scratching choice that's bound to elicit some amazing out-loud responses, so this feels like a true party flick."
-Dangerous Men,2295858,2015-11-22,Matt Donato,7/10,fresh,"Emotionless reaction shots, zero characterization, guns that have absolutely no special effects when blasted - Dangerous Men is rare winning dish from a one star restaurant."
-Dangerous Men,2295338,2015-11-19,Peter Keough,0.5/4,rotten,"Conceivably, it could serve as a primer for students on how not to make a movie, and perhaps as a deconstruction of filmic conventions for the more theoretical minded."
-Dangerous Men,2294641,2015-11-16,Jason Wilson,3/10,rotten,"If you're not a fan of garbage cinema, even for the fun of it, Dangerous Men is best to be avoided."
-Dangerous Men,2294129,2015-11-12,Soren Andersen,0/4,rotten,"""Dangerous Men,"" the picture's production notes inform, took 26 years to reach the big screen. After having seen it, I wonder: What was the rush?"
-Dangerous Men,2293902,2015-11-12,Maitland McDonagh,,rotten,Will entertain some viewers and infuriate others with its clunky mix of feminist fury and awkward action sequences.
-Dangerous Men,2293900,2015-11-12,Marjorie Baumgarten,1.5/5,rotten,"This is a bad movie, but one that awakens your senses every so often with flashes of originality and abundant self-belief."
-Dangerous Men,2293815,2015-11-12,Katie Rife,B+,fresh,"Ridiculous, artless, and wildly entertaining, Dangerous Men is more than the sum of its fascinatingly misguided parts, although it will take a special sort of moviegoer to truly appreciate (or endure, depending on your perspective) its charms."
-Dangerous Men,2293605,2015-11-11,Amy Nicholson,C,fresh,To sit through it feels like honoring the dreamers of the world who at least get shit done. Is it terrible? Of course. Is there belly-dancing? Duh.
-Small Town Wisconsin,102711819,2022-07-22,Peter Gray,,fresh,Small Town Wisconsin could hit some home truths for viewers&#44; and though being faced with the truth isn&#8217;t always pleasant&#44; it feels necessary in growing towards a happier fruition&#46;
-Small Town Wisconsin,102711545,2022-07-22,Tim Grierson,,fresh,"This low-key drama has lovely interludes and some nicely understated performances, although director Niels Mueller doesn’t glean too many new insights from Jason Naczek’s familiar story..."
-Small Town Wisconsin,102700937,2022-06-16,Sumner Forbes,8.5/10,fresh,"Small Town Wisconsin is a success in almost every regard, and if you can see over the legions of cheeseheads in the rows ahead of you, it shouldn’t be missed."
-Small Town Wisconsin,102699897,2022-06-14,Tara McNamara,3/5,fresh,Just like Wayne&#44; Small Town Wisconsin has flaws&#44; but the poignancy of the story will stick with you for a long time&#46;
-Small Town Wisconsin,102698744,2022-06-10,Rob Thomas,3/4,fresh,It&#8217;s a movie with its heart in the right place&#44; and does both small town and big city Wisconsin proud&#46;
-Small Town Wisconsin,102698639,2022-06-10,Todd Jorgenson,,rotten,Despite some intriguing character dynamics and performances that generate sympathy for this fractured family&#44; the film stumbles when it veers into melodrama without the narrative dexterity to tackle its weightier ambitions&#46;
-Small Town Wisconsin,102698482,2022-06-10,Jackie K. Cooper,7/10,fresh,This is the kind of movie that draws you so deeply into its story you are reluctant to let it end&#46;
-Small Town Wisconsin,102698164,2022-06-09,Glenn Kenny,,fresh,"Mueller’s direction is patient and sensitive, the cast is accomplished and committed, and the picture’s comedic aspects sometimes earn a chuckle."
-Small Town Wisconsin,102697854,2022-06-08,Brian Orndorf,B+,fresh,Naczek isn&apos;t interested in making a soap opera with this examination of fallibility&#44; going somewhere much more authentic when exploring character aches and pains&#46;
-Small Town Wisconsin,102695788,2022-06-02,Eddie Harrison,4/5,fresh,&#8230;a warm-hearted story of everyday life that&#8217;s easy to recommend for those who like films about people rather than portals and vortexes&#8230;
-Small Town Wisconsin,102695250,2022-05-31,Laura Clifford,C,rotten,Debuting screenwriter Jason Naczek has concocted a manchild redemption story using metaphors as heavy as a hammer and a fairy godmother who makes everything alright with a seeming flip of the switch&#46;
-Small Town Wisconsin,2733251,2020-10-12,Jared Mobarak,B,fresh,Small Town Wisconsin is always proving itself to be more than its familiar premise thanks to Naczek's ability to infuse a lot more drama into the mix than one custody battle.
-Tejano,2564925,2019-03-07,Joe Friar,3/4,fresh,The story of a South Texas ranch hand who gets mixed up with a Mexican cartel moves with pulse-pounding velocity and features top performances from a talented cast of actors with Texas roots.
-Tejano,2557738,2019-02-12,Cary Darling,4/5,fresh,"An entertaining blast of Texas noir that nods toward the work of the Coen brothers, Quentin Tarantino and fellow Austinite Greg Kwedar's 2016 low-budget thriller ""Transpecos"" as well as ""Breaking Bad."""
-Tejano,2547231,2019-01-10,Danielle White,3/5,fresh,The story itself slithers with twists and turns and unexpected betrayals. It's almost ridiculous how many characters die in this film.
-Tejano,2530119,2018-11-08,Chris Salce,9/10,fresh,"Tejano is one of those films that can be described as a hidden gem as it sneaks under the radar and will have you talking, telling your friends about it, and wanting to watch it again."
-Death of a Salesman,2770637,2021-02-23,Michael Dougan,,fresh,"Miller has taken a small, intimate tale and expanded it into a treatise on larger themes, primarily the abuse of the American Dream."
-Death of a Salesman,1950734,2011-01-02,Randy White,5/5,fresh,A classic American tragedy.
-Death of a Salesman,1422415,2005-08-04,Jules Brenner,4/5,fresh,
-Death of a Salesman,1409415,2005-07-05,Emanuel Levy,3/5,fresh,
-Death of a Salesman,839546,2003-02-06,Frederic and Mary Ann Brussat,,fresh,"Death of a Salesman, directed by Volker Schlondorff, draws out the multiple meanings of this Pulitzer Prize-winning play by Arthur Miller about change, family and fatherhood, work and love."
-Death of a Salesman,788410,2002-09-29,Dan Lybarger,4/5,fresh,"Schlndorff's artificial settings and some amazing performances help keep this from looking like a typical ""filmed play."""
-Death of a Salesman,751951,2002-08-08,Cory Cheney,4/5,fresh,
-Death of a Salesman,743794,2002-07-26,Bob Grimm,5/5,fresh,
-Death of a Salesman,743291,2002-07-26,Scott Weinberg,5/5,fresh,They MAKE you watch it in English class for a good reason!
-Sahara,1137710,2003-05-13,Dragan Antulov,5/10,fresh,
-The Debt,2628192,2019-09-20,Diego Batlle,,fresh,A Bresson-esque movie that is always enigmatic. [Full Review in Spanish]
-The Debt,2627988,2019-09-20,Gaspar Zimerman,,fresh,The story [Director Gustavo Fontán] tells is an excuse to give way to the exploration of feelings and sensations that avoid verbality. [Full review in Spanish]
-Peppermint Candy,2725008,2020-09-16,A.S. Hamrah,,fresh,"South Korean political history of the previous twenty years, Peppermint Candy is not tempered by its hysterical edge, which adds unpredictable violence to its vignettes of romantic, domestic, and business failure."
-Peppermint Candy,2541271,2018-12-16,Panos Kotzathanasis,,fresh,"Lee Chang-dong presents a melodrama that stands apart from the plethora of similar productions due to its intense political element, because it doesn't lose its seriousness at any point and because it doesn't become hyperbolic in his effort to draw tears"
-Peppermint Candy,1883708,2010-05-11,Anton Bitel,,fresh,"This is Korea's millennial elegy, filtering its search for times past through a confection no less bittersweet than Proust's madeleine."
-Peppermint Candy,1706014,2008-01-29,Beth Accomando,9/10,fresh,The film offers a heartbreaking drama told in reverse chronology and spanning twenty years in both the life of the main character and the political history of Korea.
-Peppermint Candy,1231988,2003-12-22,Greg Muskewitz,2/5,rotten,
-Peppermint Candy,1187104,2003-08-14,Joshua Tanzer,4/4,fresh,"It's a story about the original sin of a nation as well as one character. There has rarely been a better film made, ever"
-Prison Girls,2475348,2018-05-03,Roger Ebert,,rotten,Prison Girls didn't have a lot of prison sets because it was a big-budget exploitation movie. Maybe.
-Gimme the Power,2575688,2019-04-09,Afroxander,,fresh,"Rubio's film shows ambition where none is required, making Gimme the Power a lot like Molotov's music: politically engaged without having to take itself too seriously."
-Paa,2673089,2020-02-27,Nikhat Kazmi,3.5/5,fresh,"The film, which peters off into vague sub-plots about slum redevelopment and unwarranted media-bashing in the first half, suddenly picks up and scales new heights in the second half."
-Paa,2578129,2019-04-17,Shubhra Gupta,2/5,rotten,"Disappointingly, Paa is not as out-of-the-box as it could have been."
-Paa,2429810,2017-10-24,Anil Sinanan,3/5,rotten,Will Auro survive to know his Pa and reunite his parents? Forget about the disease: this is a vanity vehicle designed to showcase the Big B's versatility.
-Paa,1860476,2009-12-14,Frank Lovece,,rotten,This would-be tearjerker without the musical numbers of typical Bollywood fare is for die-hard Amitabh Bachchan fans only.
-Paa,1860473,2009-12-14,David Chute,,fresh,"The film owes much of its interest to the alertness and sincerity of the younger Bachchan and the luminous Vidya Balan as the anguished parents, and to the soft wash of the tasteful playback songs supplied by Ilaiyaraaja."
-Paa,1858964,2009-12-05,Avi Offer,5.85/10,rotten,"Well-acted, funny and occasionally witty with terrific make-up design. However, it's often convoluted, awkwardly paced and too uneven as a whole."
-Paa,1858853,2009-12-04,Frank Lovece,,fresh,"A would-be tearjerker without the singing-dancing musical numbers of typical Bollywood fare seen in the U.S., the lackluster Paa is for die-hard Amitabh Bachchan fans only%u2014of which there is no small number."
-Paa,1858816,2009-12-04,Rachel Saltz,3/5,fresh,Odd and sometimes oddly affecting.
-Alraune (A Daughter of Destiny) (Mandrake) (Unholy Love),2835964,2021-10-30,Erich Hellmund-Waldow,,fresh,"The acting is not only artistic, it is also as realistic as can be possible in such a film."
-Alraune (A Daughter of Destiny) (Mandrake) (Unholy Love),2357086,2016-10-17,C. Hooper Trask,,fresh,"Aimed straight for the gooseflesh, it strikes directly into the centre of the target."
-Toorbos,2760593,2021-01-29,Neil Young,,fresh,Built around a luminous and intriguing central performance by dancer-actor Elani Dekker.
-Toorbos,2752827,2020-12-21,Guy Lodge,,fresh,"A satisfying marriage of folky period romance and environmental parable from the misty, mossy depths of South Africa's Knysna forest region..."
-Connors' War,1555113,2006-11-09,David Nusair,1.5/4,rotten,"...although Criss does show some potential as a performer, his efforts to step into the shoes of a blind character are laughable."
-Connors' War,1539106,2006-09-19,Scott Weinberg,2/5,rotten,"Standard cable fodder all the way, with only a few solid action scenes and maybe one colorful performance in the whole thing."
-Born to Kill,2710947,2020-08-05,Mike Massie,10/10,fresh,"One of the most acerbic of all films noir, boasting essentially no redeemable characters (or a wealth of deliciously evil villains) while also being utterly enthralling."
-Born to Kill,2340106,2016-07-15,David Nusair,3/4,fresh,...a fairly typical film-noir premise that's employed to watchable yet entirely unmemorable effect by Robert Wise...
-Born to Kill,1507021,2006-05-16,Nick Schager,B,fresh,Competent if slightly too tame for a supposedly sleazy story.
-Born to Kill,1501617,2006-05-01,Fernando F. Croce,,fresh,"The usually meek Robert Wise trades his chameleonic tastefulness for full-on, jazzy misanthropy in this nasty melodrama."
-Born to Kill,1433953,2005-09-09,Jeffrey M. Anderson,3/4,fresh,"Hard to watch, but effective and alluring nonetheless."
-Born to Kill,1123980,2003-04-02,Dennis Schwartz,C,rotten,A revolting B film noir...
-The Soong Sisters,1402087,2005-06-15,Emanuel Levy,3/5,fresh,
-La Sapienza,102772380,2023-01-24,Vadim Rizov,,fresh,"Sapienza is a pretty lovely film. Symmetricities are everywhere, starting with that opening architectural showreel, which deliberately avoids perfect symmetricity..."
-La Sapienza,2767839,2021-02-14,Dustin Chang,,fresh,Their sincere expression of these thoughts rings true and melts away its artificiality in its presentation soon enough. This is the beauty of La Sapienza and Green films in general.
-La Sapienza,2598336,2019-06-18,C.J. Prince,,fresh,"It's a nice entry point into a peculiar cinematic universe, and those willing to open themselves to it will find a lot to enjoy."
-La Sapienza,2503963,2018-08-28,Charles Mudede,,fresh,"If architecture aspires to the condition of music, the acting in La Sapienza aspires to the condition of architecture. You will love the ending of this very original and elegant and arty work."
-La Sapienza,2314368,2016-03-12,Forrest Cardamenis,B,fresh,This startling architectural juxtaposition feels like a wake-up call.
-La Sapienza,2275677,2015-08-03,Nicole Armour,,fresh,"While Green's film is dense with historical fact and theory, it's not averse to plumbing life's mysteries. Suffused with warmth, it expresses a potent admiration for human striving and accomplishment."
-La Sapienza,2273804,2015-07-23,Norman Wilner,2/5,rotten,"The uncomplicated narrative resists stylization; Green's presentation turns everyone into mannequins, rendering their emotions theoretical. That may well be his point, but it didn't work for me."
-La Sapienza,2269287,2015-06-26,Sam Lubell,,fresh,"On the surface, writer-director Eugne Green's film ""La Sapienza"" is slow, strange and awkward - but stick with it and it may win you over."
-La Sapienza,2265997,2015-06-05,Rob Garratt,4/5,fresh,"Layered with reels of swirling shots of Rome's most beautiful buildings -- all crucially shot from the ground upwards, staring at the heavens-- La Sapienza is visually stunning."
-La Sapienza,2265990,2015-06-05,Boyd van Hoeij,,fresh,"The Sapience juxtaposes insights on how people are emotionally connected with ruminations on the buildings and spaces through which they move, in which they live and, in Alexandre's case, which they also create."
-La Sapienza,2265989,2015-06-05,Robert Horton,3/4,fresh,"If you can groove into this non-realistic mode, the film casts a spell."
-La Sapienza,2265790,2015-06-04,Tom Keogh,3.5/4,fresh,A beautiful space for people and light.
-La Sapienza,2255621,2015-04-09,Wesley Morris,,rotten,This kind of formalism needs to do more than walk through classical wonders. It should want to create cinema that can stand near or beside them. This movie defensively consecrates what's already there. You don't need a film to do that.
-La Sapienza,2255195,2015-04-08,Scott Foundas,,fresh,"An exquisite rumination on life, love and art that tickles the heart and mind in equal measure."
-La Sapienza,2252858,2015-03-23,Richard Brody,,fresh,"Green's richly textured, painterly images fuse with the story to evoke the essence of humane urbanity and the relationships that it fosters, whether educational, familial, or erotic."
-La Sapienza,2252553,2015-03-20,Ignatiy Vishnevetsky,B+,fresh,"Green doesn't so much use his characters as mouthpieces as emotionally invest them in art, turning opinions into feelings."
-La Sapienza,2252541,2015-03-20,Godfrey Cheshire,4/4,fresh,"""La Sapienza"" strikes this reviewer as easily the most astonishing and important movie to emerge from France in quite some time."
-La Sapienza,2252452,2015-03-19,A.O. Scott,,fresh,The movie is an unapologetically rarefied undertaking and at the same time a gracious and inviting film.
-La Sapienza,2252301,2015-03-19,David Noh,,rotten,"Pretentious, stuffy and slow. There's some beautiful scenery here but oh, what you must put up with to earn it!"
-La Sapienza,2252028,2015-03-18,Noel Murray,3/5,fresh,"While La Sapienza is unsatisfying as drama, it's frequently beautiful just as a tour through architecturally significant Italian buildings."
-La Sapienza,2251985,2015-03-17,David Ehrlich,3/5,fresh,La Sapienza  alternately feels like a self-reflexive love story or a haunted history lesson -- its best scenes play like both.
-La Sapienza,2251926,2015-03-17,Zachary Wigon,,fresh,A picture that balances heart and mind with nuance.
-La Sapienza,2251650,2015-03-14,Harvey S. Karten,B+,fresh,"As in ""Who's Afraid of Virginia Woolf,"" both the younger couple and their older mentors are changed from a relationship."
-La Sapienza,2250991,2015-03-12,Ben Sachs,,fresh,"This recalls Manoel de Oliveira and Eric Rohmer in its poker-faced style, deliberately archaic storytelling, and magisterial epiphanies."
-La Sapienza,2225361,2014-09-28,Donald J. Levit,,fresh,"Although a love-fiction crossed with documentary lecture and superb Raphael O'Byrne cinematography, 'La Sapienza' is as close as celluloid can approach to architecture."
-La Sapienza,2222032,2014-09-10,Carson Lund,3/4,fresh,"Eugne Green's mannered direction doesn't work for every situation it's homogenously applied to, but at its most effective it inspires an enhanced sensitivity to the import of every gesture, visual or verbal."
-Uncle Tom,2713732,2020-08-14,Megan Basham,,fresh,Uncle Tom suffers from an overreliance on pundits. Its most compelling insights come from people who've never been quoted in a Twitter or Facebook battle.
-Uncle Tom,2706229,2020-07-19,Matthew Pejkovic,4/5,fresh,"An incredibly relevant and insightful documentary that delves into the past, present, and future of the black American conservative movement."
+title,reviewid,creationdate,criticname,originalscore,reviewstate,reviewtext
+Beavers,1145982,2003-05-23,Ivan M. Lincoln,3.5/4,fresh,"Timed to be just long enough for most youngsters' brief attention spans -- and it's packed with plenty of interesting activity, both on land and under the water."
+Blood Mask,1636744,2007-06-02,The Foywonder,1/5,rotten,"It doesn't matter if a movie costs 300 million or only 300 dollars; good is good and bad is bad, and Bloodmask: The Possession of Nicole Lameroux is just plain bad."
+City Hunter: Shinjuku Private Eyes,2590987,2019-05-28,Reuben Baron,,fresh,"The choreography is so precise and lifelike at points one might wonder whether the movie was rotoscoped, but no live-action reference footage was used. The quality is due to the skill of the animators and Kodama's love for professional wrestling."
+City Hunter: Shinjuku Private Eyes,2558908,2019-02-14,Matt Schley,2.5/5,rotten,The film's out-of-touch attempts at humor may find them hunting for the reason the franchise was so popular in the first place.
+Dangerous Men,2504681,2018-08-29,Pat Padua,,fresh,Its clumsy determination is endearing and sometimes wildly entertaining
+Dangerous Men,2299284,2015-12-13,Eric Melin,4/5,fresh,"With every new minute, there's another head-scratching choice that's bound to elicit some amazing out-loud responses, so this feels like a true party flick."
+Dangerous Men,2295858,2015-11-22,Matt Donato,7/10,fresh,"Emotionless reaction shots, zero characterization, guns that have absolutely no special effects when blasted - Dangerous Men is rare winning dish from a one star restaurant."
+Dangerous Men,2295338,2015-11-19,Peter Keough,0.5/4,rotten,"Conceivably, it could serve as a primer for students on how not to make a movie, and perhaps as a deconstruction of filmic conventions for the more theoretical minded."
+Dangerous Men,2294641,2015-11-16,Jason Wilson,3/10,rotten,"If you're not a fan of garbage cinema, even for the fun of it, Dangerous Men is best to be avoided."
+Dangerous Men,2294129,2015-11-12,Soren Andersen,0/4,rotten,"""Dangerous Men,"" the picture's production notes inform, took 26 years to reach the big screen. After having seen it, I wonder: What was the rush?"
+Dangerous Men,2293902,2015-11-12,Maitland McDonagh,,rotten,Will entertain some viewers and infuriate others with its clunky mix of feminist fury and awkward action sequences.
+Dangerous Men,2293900,2015-11-12,Marjorie Baumgarten,1.5/5,rotten,"This is a bad movie, but one that awakens your senses every so often with flashes of originality and abundant self-belief."
+Dangerous Men,2293815,2015-11-12,Katie Rife,B+,fresh,"Ridiculous, artless, and wildly entertaining, Dangerous Men is more than the sum of its fascinatingly misguided parts, although it will take a special sort of moviegoer to truly appreciate (or endure, depending on your perspective) its charms."
+Dangerous Men,2293605,2015-11-11,Amy Nicholson,C,fresh,To sit through it feels like honoring the dreamers of the world who at least get shit done. Is it terrible? Of course. Is there belly-dancing? Duh.
+Small Town Wisconsin,102711819,2022-07-22,Peter Gray,,fresh,Small Town Wisconsin could hit some home truths for viewers&#44; and though being faced with the truth isn&#8217;t always pleasant&#44; it feels necessary in growing towards a happier fruition&#46;
+Small Town Wisconsin,102711545,2022-07-22,Tim Grierson,,fresh,"This low-key drama has lovely interludes and some nicely understated performances, although director Niels Mueller doesn’t glean too many new insights from Jason Naczek’s familiar story..."
+Small Town Wisconsin,102700937,2022-06-16,Sumner Forbes,8.5/10,fresh,"Small Town Wisconsin is a success in almost every regard, and if you can see over the legions of cheeseheads in the rows ahead of you, it shouldn’t be missed."
+Small Town Wisconsin,102699897,2022-06-14,Tara McNamara,3/5,fresh,Just like Wayne&#44; Small Town Wisconsin has flaws&#44; but the poignancy of the story will stick with you for a long time&#46;
+Small Town Wisconsin,102698744,2022-06-10,Rob Thomas,3/4,fresh,It&#8217;s a movie with its heart in the right place&#44; and does both small town and big city Wisconsin proud&#46;
+Small Town Wisconsin,102698639,2022-06-10,Todd Jorgenson,,rotten,Despite some intriguing character dynamics and performances that generate sympathy for this fractured family&#44; the film stumbles when it veers into melodrama without the narrative dexterity to tackle its weightier ambitions&#46;
+Small Town Wisconsin,102698482,2022-06-10,Jackie K. Cooper,7/10,fresh,This is the kind of movie that draws you so deeply into its story you are reluctant to let it end&#46;
+Small Town Wisconsin,102698164,2022-06-09,Glenn Kenny,,fresh,"Mueller’s direction is patient and sensitive, the cast is accomplished and committed, and the picture’s comedic aspects sometimes earn a chuckle."
+Small Town Wisconsin,102697854,2022-06-08,Brian Orndorf,B+,fresh,Naczek isn&apos;t interested in making a soap opera with this examination of fallibility&#44; going somewhere much more authentic when exploring character aches and pains&#46;
+Small Town Wisconsin,102695788,2022-06-02,Eddie Harrison,4/5,fresh,&#8230;a warm-hearted story of everyday life that&#8217;s easy to recommend for those who like films about people rather than portals and vortexes&#8230;
+Small Town Wisconsin,102695250,2022-05-31,Laura Clifford,C,rotten,Debuting screenwriter Jason Naczek has concocted a manchild redemption story using metaphors as heavy as a hammer and a fairy godmother who makes everything alright with a seeming flip of the switch&#46;
+Small Town Wisconsin,2733251,2020-10-12,Jared Mobarak,B,fresh,Small Town Wisconsin is always proving itself to be more than its familiar premise thanks to Naczek's ability to infuse a lot more drama into the mix than one custody battle.
+Tejano,2564925,2019-03-07,Joe Friar,3/4,fresh,The story of a South Texas ranch hand who gets mixed up with a Mexican cartel moves with pulse-pounding velocity and features top performances from a talented cast of actors with Texas roots.
+Tejano,2557738,2019-02-12,Cary Darling,4/5,fresh,"An entertaining blast of Texas noir that nods toward the work of the Coen brothers, Quentin Tarantino and fellow Austinite Greg Kwedar's 2016 low-budget thriller ""Transpecos"" as well as ""Breaking Bad."""
+Tejano,2547231,2019-01-10,Danielle White,3/5,fresh,The story itself slithers with twists and turns and unexpected betrayals. It's almost ridiculous how many characters die in this film.
+Tejano,2530119,2018-11-08,Chris Salce,9/10,fresh,"Tejano is one of those films that can be described as a hidden gem as it sneaks under the radar and will have you talking, telling your friends about it, and wanting to watch it again."
+Death of a Salesman,2770637,2021-02-23,Michael Dougan,,fresh,"Miller has taken a small, intimate tale and expanded it into a treatise on larger themes, primarily the abuse of the American Dream."
+Death of a Salesman,1950734,2011-01-02,Randy White,5/5,fresh,A classic American tragedy.
+Death of a Salesman,1422415,2005-08-04,Jules Brenner,4/5,fresh,
+Death of a Salesman,1409415,2005-07-05,Emanuel Levy,3/5,fresh,
+Death of a Salesman,839546,2003-02-06,Frederic and Mary Ann Brussat,,fresh,"Death of a Salesman, directed by Volker Schlondorff, draws out the multiple meanings of this Pulitzer Prize-winning play by Arthur Miller about change, family and fatherhood, work and love."
+Death of a Salesman,788410,2002-09-29,Dan Lybarger,4/5,fresh,"Schlndorff's artificial settings and some amazing performances help keep this from looking like a typical ""filmed play."""
+Death of a Salesman,751951,2002-08-08,Cory Cheney,4/5,fresh,
+Death of a Salesman,743794,2002-07-26,Bob Grimm,5/5,fresh,
+Death of a Salesman,743291,2002-07-26,Scott Weinberg,5/5,fresh,They MAKE you watch it in English class for a good reason!
+Sahara,1137710,2003-05-13,Dragan Antulov,5/10,fresh,
+The Debt,2628192,2019-09-20,Diego Batlle,,fresh,A Bresson-esque movie that is always enigmatic. [Full Review in Spanish]
+The Debt,2627988,2019-09-20,Gaspar Zimerman,,fresh,The story [Director Gustavo Fontán] tells is an excuse to give way to the exploration of feelings and sensations that avoid verbality. [Full review in Spanish]
+Peppermint Candy,2725008,2020-09-16,A.S. Hamrah,,fresh,"South Korean political history of the previous twenty years, Peppermint Candy is not tempered by its hysterical edge, which adds unpredictable violence to its vignettes of romantic, domestic, and business failure."
+Peppermint Candy,2541271,2018-12-16,Panos Kotzathanasis,,fresh,"Lee Chang-dong presents a melodrama that stands apart from the plethora of similar productions due to its intense political element, because it doesn't lose its seriousness at any point and because it doesn't become hyperbolic in his effort to draw tears"
+Peppermint Candy,1883708,2010-05-11,Anton Bitel,,fresh,"This is Korea's millennial elegy, filtering its search for times past through a confection no less bittersweet than Proust's madeleine."
+Peppermint Candy,1706014,2008-01-29,Beth Accomando,9/10,fresh,The film offers a heartbreaking drama told in reverse chronology and spanning twenty years in both the life of the main character and the political history of Korea.
+Peppermint Candy,1231988,2003-12-22,Greg Muskewitz,2/5,rotten,
+Peppermint Candy,1187104,2003-08-14,Joshua Tanzer,4/4,fresh,"It's a story about the original sin of a nation as well as one character. There has rarely been a better film made, ever"
+Prison Girls,2475348,2018-05-03,Roger Ebert,,rotten,Prison Girls didn't have a lot of prison sets because it was a big-budget exploitation movie. Maybe.
+Gimme the Power,2575688,2019-04-09,Afroxander,,fresh,"Rubio's film shows ambition where none is required, making Gimme the Power a lot like Molotov's music: politically engaged without having to take itself too seriously."
+Paa,2673089,2020-02-27,Nikhat Kazmi,3.5/5,fresh,"The film, which peters off into vague sub-plots about slum redevelopment and unwarranted media-bashing in the first half, suddenly picks up and scales new heights in the second half."
+Paa,2578129,2019-04-17,Shubhra Gupta,2/5,rotten,"Disappointingly, Paa is not as out-of-the-box as it could have been."
+Paa,2429810,2017-10-24,Anil Sinanan,3/5,rotten,Will Auro survive to know his Pa and reunite his parents? Forget about the disease: this is a vanity vehicle designed to showcase the Big B's versatility.
+Paa,1860476,2009-12-14,Frank Lovece,,rotten,This would-be tearjerker without the musical numbers of typical Bollywood fare is for die-hard Amitabh Bachchan fans only.
+Paa,1860473,2009-12-14,David Chute,,fresh,"The film owes much of its interest to the alertness and sincerity of the younger Bachchan and the luminous Vidya Balan as the anguished parents, and to the soft wash of the tasteful playback songs supplied by Ilaiyaraaja."
+Paa,1858964,2009-12-05,Avi Offer,5.85/10,rotten,"Well-acted, funny and occasionally witty with terrific make-up design. However, it's often convoluted, awkwardly paced and too uneven as a whole."
+Paa,1858853,2009-12-04,Frank Lovece,,fresh,"A would-be tearjerker without the singing-dancing musical numbers of typical Bollywood fare seen in the U.S., the lackluster Paa is for die-hard Amitabh Bachchan fans only%u2014of which there is no small number."
+Paa,1858816,2009-12-04,Rachel Saltz,3/5,fresh,Odd and sometimes oddly affecting.
+Alraune (A Daughter of Destiny) (Mandrake) (Unholy Love),2835964,2021-10-30,Erich Hellmund-Waldow,,fresh,"The acting is not only artistic, it is also as realistic as can be possible in such a film."
+Alraune (A Daughter of Destiny) (Mandrake) (Unholy Love),2357086,2016-10-17,C. Hooper Trask,,fresh,"Aimed straight for the gooseflesh, it strikes directly into the centre of the target."
+Toorbos,2760593,2021-01-29,Neil Young,,fresh,Built around a luminous and intriguing central performance by dancer-actor Elani Dekker.
+Toorbos,2752827,2020-12-21,Guy Lodge,,fresh,"A satisfying marriage of folky period romance and environmental parable from the misty, mossy depths of South Africa's Knysna forest region..."
+Connors' War,1555113,2006-11-09,David Nusair,1.5/4,rotten,"...although Criss does show some potential as a performer, his efforts to step into the shoes of a blind character are laughable."
+Connors' War,1539106,2006-09-19,Scott Weinberg,2/5,rotten,"Standard cable fodder all the way, with only a few solid action scenes and maybe one colorful performance in the whole thing."
+Born to Kill,2710947,2020-08-05,Mike Massie,10/10,fresh,"One of the most acerbic of all films noir, boasting essentially no redeemable characters (or a wealth of deliciously evil villains) while also being utterly enthralling."
+Born to Kill,2340106,2016-07-15,David Nusair,3/4,fresh,...a fairly typical film-noir premise that's employed to watchable yet entirely unmemorable effect by Robert Wise...
+Born to Kill,1507021,2006-05-16,Nick Schager,B,fresh,Competent if slightly too tame for a supposedly sleazy story.
+Born to Kill,1501617,2006-05-01,Fernando F. Croce,,fresh,"The usually meek Robert Wise trades his chameleonic tastefulness for full-on, jazzy misanthropy in this nasty melodrama."
+Born to Kill,1433953,2005-09-09,Jeffrey M. Anderson,3/4,fresh,"Hard to watch, but effective and alluring nonetheless."
+Born to Kill,1123980,2003-04-02,Dennis Schwartz,C,rotten,A revolting B film noir...
+The Soong Sisters,1402087,2005-06-15,Emanuel Levy,3/5,fresh,
+La Sapienza,102772380,2023-01-24,Vadim Rizov,,fresh,"Sapienza is a pretty lovely film. Symmetricities are everywhere, starting with that opening architectural showreel, which deliberately avoids perfect symmetricity..."
+La Sapienza,2767839,2021-02-14,Dustin Chang,,fresh,Their sincere expression of these thoughts rings true and melts away its artificiality in its presentation soon enough. This is the beauty of La Sapienza and Green films in general.
+La Sapienza,2598336,2019-06-18,C.J. Prince,,fresh,"It's a nice entry point into a peculiar cinematic universe, and those willing to open themselves to it will find a lot to enjoy."
+La Sapienza,2503963,2018-08-28,Charles Mudede,,fresh,"If architecture aspires to the condition of music, the acting in La Sapienza aspires to the condition of architecture. You will love the ending of this very original and elegant and arty work."
+La Sapienza,2314368,2016-03-12,Forrest Cardamenis,B,fresh,This startling architectural juxtaposition feels like a wake-up call.
+La Sapienza,2275677,2015-08-03,Nicole Armour,,fresh,"While Green's film is dense with historical fact and theory, it's not averse to plumbing life's mysteries. Suffused with warmth, it expresses a potent admiration for human striving and accomplishment."
+La Sapienza,2273804,2015-07-23,Norman Wilner,2/5,rotten,"The uncomplicated narrative resists stylization; Green's presentation turns everyone into mannequins, rendering their emotions theoretical. That may well be his point, but it didn't work for me."
+La Sapienza,2269287,2015-06-26,Sam Lubell,,fresh,"On the surface, writer-director Eugne Green's film ""La Sapienza"" is slow, strange and awkward - but stick with it and it may win you over."
+La Sapienza,2265997,2015-06-05,Rob Garratt,4/5,fresh,"Layered with reels of swirling shots of Rome's most beautiful buildings -- all crucially shot from the ground upwards, staring at the heavens-- La Sapienza is visually stunning."
+La Sapienza,2265990,2015-06-05,Boyd van Hoeij,,fresh,"The Sapience juxtaposes insights on how people are emotionally connected with ruminations on the buildings and spaces through which they move, in which they live and, in Alexandre's case, which they also create."
+La Sapienza,2265989,2015-06-05,Robert Horton,3/4,fresh,"If you can groove into this non-realistic mode, the film casts a spell."
+La Sapienza,2265790,2015-06-04,Tom Keogh,3.5/4,fresh,A beautiful space for people and light.
+La Sapienza,2255621,2015-04-09,Wesley Morris,,rotten,This kind of formalism needs to do more than walk through classical wonders. It should want to create cinema that can stand near or beside them. This movie defensively consecrates what's already there. You don't need a film to do that.
+La Sapienza,2255195,2015-04-08,Scott Foundas,,fresh,"An exquisite rumination on life, love and art that tickles the heart and mind in equal measure."
+La Sapienza,2252858,2015-03-23,Richard Brody,,fresh,"Green's richly textured, painterly images fuse with the story to evoke the essence of humane urbanity and the relationships that it fosters, whether educational, familial, or erotic."
+La Sapienza,2252553,2015-03-20,Ignatiy Vishnevetsky,B+,fresh,"Green doesn't so much use his characters as mouthpieces as emotionally invest them in art, turning opinions into feelings."
+La Sapienza,2252541,2015-03-20,Godfrey Cheshire,4/4,fresh,"""La Sapienza"" strikes this reviewer as easily the most astonishing and important movie to emerge from France in quite some time."
+La Sapienza,2252452,2015-03-19,A.O. Scott,,fresh,The movie is an unapologetically rarefied undertaking and at the same time a gracious and inviting film.
+La Sapienza,2252301,2015-03-19,David Noh,,rotten,"Pretentious, stuffy and slow. There's some beautiful scenery here but oh, what you must put up with to earn it!"
+La Sapienza,2252028,2015-03-18,Noel Murray,3/5,fresh,"While La Sapienza is unsatisfying as drama, it's frequently beautiful just as a tour through architecturally significant Italian buildings."
+La Sapienza,2251985,2015-03-17,David Ehrlich,3/5,fresh,La Sapienza  alternately feels like a self-reflexive love story or a haunted history lesson -- its best scenes play like both.
+La Sapienza,2251926,2015-03-17,Zachary Wigon,,fresh,A picture that balances heart and mind with nuance.
+La Sapienza,2251650,2015-03-14,Harvey S. Karten,B+,fresh,"As in ""Who's Afraid of Virginia Woolf,"" both the younger couple and their older mentors are changed from a relationship."
+La Sapienza,2250991,2015-03-12,Ben Sachs,,fresh,"This recalls Manoel de Oliveira and Eric Rohmer in its poker-faced style, deliberately archaic storytelling, and magisterial epiphanies."
+La Sapienza,2225361,2014-09-28,Donald J. Levit,,fresh,"Although a love-fiction crossed with documentary lecture and superb Raphael O'Byrne cinematography, 'La Sapienza' is as close as celluloid can approach to architecture."
+La Sapienza,2222032,2014-09-10,Carson Lund,3/4,fresh,"Eugne Green's mannered direction doesn't work for every situation it's homogenously applied to, but at its most effective it inspires an enhanced sensitivity to the import of every gesture, visual or verbal."
+Uncle Tom,2713732,2020-08-14,Megan Basham,,fresh,Uncle Tom suffers from an overreliance on pundits. Its most compelling insights come from people who've never been quoted in a Twitter or Facebook battle.
+Uncle Tom,2706229,2020-07-19,Matthew Pejkovic,4/5,fresh,"An incredibly relevant and insightful documentary that delves into the past, present, and future of the black American conservative movement."
 Uncle Tom,2698525,2020-06-24,Dante James,7/10,fresh,"It's a little misleading in some areas, especially if you know the players involved in this doc, but there are a lot of interesting historical facts about the breakdown of the Black family and how the whole welfare system targeted the Black community."
@@ -0,0 +1,49 @@
+---
+title: "Planets in the Solar System"
+author: "Your Name"
+date: "January 10, 2024"
+---
+
+# Introduction
+
+Our Solar System comprises several diverse and fascinating planets. Let's explore them below.
+
+## Sun
+
+The Sun is the central star of the solar system, holding all planets and other objects in space through gravitational force.
+
+## Mercury
+
+Mercury is the closest planet to the Sun and also the smallest in the solar system.
+
+## Venus
+
+Venus is a planet similar in size and structure to Earth but has a thick atmosphere and high temperatures.
+
+## Earth
+
+Earth is the only known planet with life. It has water and an atmosphere that supports living organisms.
+
+## Mars
+
+Mars is known for its red appearance. Research indicates the possibility of water in liquid and ice forms.
+
+## Jupiter
+
+Jupiter is the largest planet in the solar system and has a complex system of natural satellites and a prominent ring system.
+
+## Saturn
+
+Saturn is famous for its beautiful atmospheric rings and has multiple ring systems.
+
+## Uranus
+
+Uranus rotates on its side, creating a unique appearance in the solar system.
+
+## Neptune
+
+Neptune, the last large planet in the solar system, has an atmosphere rich in methane gas.
+
+# Conclusion
+
+The planets in the solar system form a complex and diverse system. Each planet has unique and interesting characteristics, making the solar system an attractive subject for research and exploration.
@@ -3,12 +3,13 @@
  "private": true,
  "name": "examples",
  "dependencies": {
-    "@notionhq/client": "^2.2.13",
    "@datastax/astra-db-ts": "^0.1.2",
+    "@notionhq/client": "^2.2.14",
    "@pinecone-database/pinecone": "^1.1.2",
+    "chromadb": "^1.7.3",
    "commander": "^11.1.0",
-    "llamaindex": "latest",
    "dotenv": "^16.3.1",
+    "llamaindex": "latest",
    "mongodb": "^6.2.0"
  },
  "devDependencies": {
@@ -1,7 +1,6 @@
 import { Portkey } from "llamaindex";

 (async () => {
-  const llms = [{}];
  const portkey = new Portkey({
    mode: "single",
    llms: [
@@ -13,7 +12,7 @@ import { Portkey } from "llamaindex";
      },
    ],
  });
-  const result = portkey.stream_chat([
+  const result = portkey.streamChat([
    { role: "system", content: "You are a helpful assistant." },
    { role: "user", content: "Tell me a joke." },
  ]);
@@ -0,0 +1,61 @@
+## Reader Examples
+
+These examples show how to use a specific reader class by loading a document and running a test query.
+
+1. Make sure you are in `examples` directory
+
+```bash
+cd ./examples
+```
+
+2. Prepare `OPENAI_API_KEY` environment variable:
+
+```bash
+export OPENAI_API_KEY=your_openai_api_key
+```
+
+3. Run the following command to load documents and test query:
+
+- MarkdownReader Example
+
+```bash
+npx ts-node readers/load-md.ts
+```
+
+- DocxReader Example
+
+```bash
+npx ts-node readers/load-docx.ts
+```
+
+- PdfReader Example
+
+```bash
+npx ts-node readers/load-pdf.ts
+```
+
+- HtmlReader Example
+
+```bash
+npx ts-node readers/load-html.ts
+```
+
+- CsvReader Example
+
+```bash
+npx ts-node readers/load-csv.ts
+```
+
+- NotionReader Example
+
+```bash
+export NOTION_TOKEN=your_notion_token
+npx ts-node readers/load-notion.ts
+```
+
+- AssemblyAI Example
+
+```bash
+export ASSEMBLYAI_API_KEY=your_assemblyai_api_key
+npx ts-node readers/load-assemblyai.ts
+```
@@ -0,0 +1,22 @@
+import { DocxReader, VectorStoreIndex } from "llamaindex";
+
+const FILE_PATH = "./data/stars.docx";
+const SAMPLE_QUERY = "Information about Zodiac";
+
+async function main() {
+  // Load docx file
+  console.log("Loading data...");
+  const reader = new DocxReader();
+  const documents = await reader.loadData(FILE_PATH);
+
+  // Create embeddings
+  console.log("Creating embeddings...");
+  const index = await VectorStoreIndex.fromDocuments(documents);
+
+  // Test query
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query(SAMPLE_QUERY);
+  console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
+}
+
+main();
@@ -0,0 +1,22 @@
+import { MarkdownReader, VectorStoreIndex } from "llamaindex";
+
+const FILE_PATH = "./data/planets.md";
+const SAMPLE_QUERY = "List all planets";
+
+async function main() {
+  // Load markdown file
+  console.log("Loading data...");
+  const reader = new MarkdownReader();
+  const documents = await reader.loadData(FILE_PATH);
+
+  // Create embeddings
+  console.log("Creating embeddings...");
+  const index = await VectorStoreIndex.fromDocuments(documents);
+
+  // Test query
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query(SAMPLE_QUERY);
+  console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
+}
+
+main();
@@ -0,0 +1,30 @@
+import { TogetherEmbedding, TogetherLLM } from "llamaindex";
+
+// process.env.TOGETHER_API_KEY is required
+const together = new TogetherLLM({
+  model: "mistralai/Mixtral-8x7B-Instruct-v0.1",
+});
+
+(async () => {
+  const generator = await together.chat(
+    [
+      {
+        role: "system",
+        content: "You are an AI assistant",
+      },
+      {
+        role: "user",
+        content: "Tell me about San Francisco",
+      },
+    ],
+    undefined,
+    true,
+  );
+  console.log("Chatting with Together AI...");
+  for await (const message of generator) {
+    process.stdout.write(message);
+  }
+  const embedding = new TogetherEmbedding();
+  const vector = await embedding.getTextEmbedding("Hello world!");
+  console.log("vector:", vector);
+})();
@@ -1,4 +1,5 @@
 {
+  "extends": "../tsconfig.json",
  "compilerOptions": {
    "target": "es2016",
    "module": "commonjs",
@@ -8,8 +8,9 @@
    "lint": "turbo run lint",
    "prepare": "husky install",
    "test": "turbo run test",
-    "publish-packages": "turbo run build lint test --filter=\"!docs\" && changeset version && changeset publish",
-    "publish-snapshot": "turbo run build lint test --filter=\"!docs\" && changeset version --snapshot && changeset publish"
+    "type-check": "tsc -b --diagnostics",
+    "new-version": "turbo run build lint test --filter=\"!docs\" && changeset version",
+    "new-snapshot": "turbo run build lint test --filter=\"!docs\" && changeset version --snapshot"
  },
  "devDependencies": {
    "@changesets/cli": "^2.27.1",
@@ -23,7 +24,8 @@
    "prettier": "^3.1.1",
    "prettier-plugin-organize-imports": "^3.2.4",
    "ts-jest": "^29.1.1",
-    "turbo": "^1.11.2"
+    "turbo": "^1.11.2",
+    "typescript": "^5.3.3"
  },
  "packageManager": "pnpm@8.10.5+sha256.a4bd9bb7b48214bbfcd95f264bd75bb70d100e5d4b58808f5cd6ab40c6ac21c5",
  "pnpm": {
@@ -1,5 +1,23 @@
 # llamaindex

+## 0.0.45
+
+### Patch Changes
+
+- 2e6b36e: feat: support together AI
+
+## 0.0.44
+
+### Patch Changes
+
+- 648482b: Feat: Add support for Chroma DB as a vector store
+
+## 0.0.43
+
+### Patch Changes
+
+- Fix performance issue parsing nodes: use regex to split texts
+
 ## 0.0.42

 ### Patch Changes
@@ -1,5 +1,10 @@
 # LlamaIndex.TS

+[![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
+
 LlamaIndex is a data framework for your LLM application.

 Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.
@@ -12,7 +17,7 @@ LlamaIndex.TS aims to be a lightweight, easy to use set of libraries to help you

 ## Getting started with an example:

-LlamaIndex.TS requries Node v18 or higher. You can download it from https://nodejs.org or use https://nvm.sh (our preferred option).
+LlamaIndex.TS requires Node v18 or higher. You can download it from https://nodejs.org or use https://nvm.sh (our preferred option).

 In a new folder:

@@ -84,11 +89,38 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou

 - [SimplePrompt](/packages/core/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.

+## Note: NextJS:
+
+If you're using NextJS App Router, you'll need to use the NodeJS runtime (default) and add the following config to your next.config.js to have it use imports/exports in the same way Node does.
+
+```js
+export const runtime = "nodejs"; // default
+```
+
+```js
+// next.config.js
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  webpack: (config) => {
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      sharp$: false,
+      "onnxruntime-node$": false,
+      mongodb$: false,
+    };
+    return config;
+  },
+};
+
+module.exports = nextConfig;
+```
+
 ## Supported LLMs:

 - OpenAI GPT-3.5-turbo and GPT-4
 - Anthropic Claude Instant and Claude 2
 - Llama2 Chat LLMs (70B, 13B, and 7B parameters)
+- MistralAI Chat LLMs

 ## Contributing:

@@ -1,6 +1,6 @@
 {
  "name": "llamaindex",
-  "version": "0.0.42",
+  "version": "0.0.45",
  "license": "MIT",
  "dependencies": {
    "@anthropic-ai/sdk": "^0.9.1",
@@ -10,7 +10,7 @@
    "@pinecone-database/pinecone": "^1.1.2",
    "@xenova/transformers": "^2.10.0",
    "assemblyai": "^4.0.0",
-    "compromise": "^14.10.1",
+    "chromadb": "^1.7.3",
    "file-type": "^18.7.0",
    "js-tiktoken": "^1.0.8",
    "lodash": "^4.17.21",
@@ -27,31 +27,46 @@
    "rake-modified": "^1.0.8",
    "replicate": "^0.21.1",
    "string-strip-html": "^13.4.3",
-    "uuid": "^9.0.1",
    "wink-nlp": "^1.14.3"
  },
  "devDependencies": {
    "@types/jest": "^29.5.11",
    "@types/lodash": "^4.14.202",
-    "@types/node": "^18.19.2",
+    "@types/node": "^18.19.6",
    "@types/papaparse": "^5.3.14",
    "@types/pg": "^8.10.9",
-    "@types/uuid": "^9.0.7",
+    "bunchee": "^4.3.3",
    "node-stdlib-browser": "^1.2.0",
-    "tsup": "^7.2.0",
-    "typescript": "^5.3.2"
+    "typescript": "^5.3.3"
  },
  "engines": {
    "node": ">=18.0.0"
  },
  "types": "./dist/index.d.ts",
  "main": "./dist/index.js",
-  "module": "./dist/index.mjs",
-  "repository": "run-llama/LlamaIndexTS",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "examples",
+    "src",
+    "types",
+    "CHANGELOG.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/core"
+  },
  "scripts": {
    "lint": "eslint .",
    "test": "jest",
-    "build": "tsup src/index.ts --format esm,cjs --dts",
-    "dev": "tsup src/index.ts --format esm,cjs --dts --watch"
+    "build": "bunchee",
+    "dev": "bunchee -w"
  }
 }
@@ -1,4 +1,4 @@
-import { v4 as uuidv4 } from "uuid";
+import { randomUUID } from "node:crypto";
 import { ChatHistory } from "./ChatHistory";
 import { NodeWithScore, TextNode } from "./Node";
 import {
@@ -206,7 +206,7 @@ export class DefaultContextGenerator implements ContextGenerator {
  async generate(message: string, parentEvent?: Event): Promise<Context> {
    if (!parentEvent) {
      parentEvent = {
-        id: uuidv4(),
+        id: randomUUID(),
        type: "wrapper",
        tags: ["final"],
      };
@@ -272,7 +272,7 @@ export class ContextChatEngine implements ChatEngine {
    }

    const parentEvent: Event = {
-      id: uuidv4(),
+      id: randomUUID(),
      type: "wrapper",
      tags: ["final"],
    };
@@ -304,7 +304,7 @@ export class ContextChatEngine implements ChatEngine {
    chatHistory = chatHistory ?? this.chatHistory;

    const parentEvent: Event = {
-      id: uuidv4(),
+      id: randomUUID(),
      type: "wrapper",
      tags: ["final"],
    };
@@ -1,6 +1,6 @@
 import { encodingForModel } from "js-tiktoken";

-import { v4 as uuidv4 } from "uuid";
+import { randomUUID } from "node:crypto";
 import { Event, EventTag, EventType } from "./callbacks/CallbackManager";

 export enum Tokenizers {
@@ -64,7 +64,7 @@ class GlobalsHelper {
    tags?: EventTag[];
  }): Event {
    return {
-      id: uuidv4(),
+      id: randomUUID(),
      type,
      // inherit parent tags if tags not set
      tags: tags || parentEvent?.tags,
@@ -1,7 +1,6 @@
 import _ from "lodash";
-import { createHash } from "node:crypto";
-import path from "path";
-import { v4 as uuidv4 } from "uuid";
+import { createHash, randomUUID } from "node:crypto";
+import path from "node:path";

 export enum NodeRelationship {
  SOURCE = "SOURCE",
@@ -49,7 +48,7 @@ export abstract class BaseNode<T extends Metadata = Metadata> {
   *
   * Set to a UUID by default.
   */
-  id_: string = uuidv4();
+  id_: string = randomUUID();
  embedding?: number[];

  // Metadata fields
@@ -1,4 +1,4 @@
-import { v4 as uuidv4 } from "uuid";
+import { randomUUID } from "node:crypto";
 import { NodeWithScore, TextNode } from "./Node";
 import {
  BaseQuestionGenerator,
@@ -72,7 +72,7 @@ export class RetrieverQueryEngine implements BaseQueryEngine {

  async query(query: string, parentEvent?: Event) {
    const _parentEvent: Event = parentEvent || {
-      id: uuidv4(),
+      id: randomUUID(),
      type: "wrapper",
      tags: ["final"],
    };
@@ -136,14 +136,14 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {

    // groups final retrieval+synthesis operation
    const parentEvent: Event = {
-      id: uuidv4(),
+      id: randomUUID(),
      type: "wrapper",
      tags: ["final"],
    };

    // groups all sub-queries
    const subQueryParentEvent: Event = {
-      id: uuidv4(),
+      id: randomUUID(),
      parentId: parentEvent.id,
      type: "wrapper",
      tags: ["intermediate"],
@@ -1,4 +1,3 @@
-import nlp from "compromise";
 import { EOL } from "node:os";
 // GitHub translated
 import { globalsHelper } from "./GlobalsHelper";
@@ -19,11 +18,17 @@ class TextSplit {

 type SplitRep = { text: string; numTokens: number };

+const defaultregex = /[.?!][\])'"`’”]*(?:\s|$)/g;
 export const defaultSentenceTokenizer = (text: string): string[] => {
-  return nlp(text)
-    .sentences()
-    .json()
-    .map((sentence: any) => sentence.text);
+  const slist = [];
+  const iter = text.matchAll(defaultregex);
+  let lastIdx = 0;
+  for (const match of iter) {
+    slist.push(text.slice(lastIdx, match.index! + 1));
+    lastIdx = match.index! + 1;
+  }
+  slist.push(text.slice(lastIdx));
+  return slist.filter((s) => s.length > 0);
 };

 // Refs: https://github.com/fxsjy/jieba/issues/575#issuecomment-359637511
@@ -14,7 +14,7 @@ export enum OpenAIEmbeddingModelType {
 }

 export class OpenAIEmbedding extends BaseEmbedding {
-  model: OpenAIEmbeddingModelType;
+  model: OpenAIEmbeddingModelType | string;

  // OpenAI session params
  apiKey?: string = undefined;
@@ -3,5 +3,6 @@ export * from "./HuggingFaceEmbedding";
 export * from "./MistralAIEmbedding";
 export * from "./MultiModalEmbedding";
 export * from "./OpenAIEmbedding";
+export { TogetherEmbedding } from "./together";
 export * from "./types";
 export * from "./utils";
@@ -0,0 +1,16 @@
+import { OpenAIEmbedding } from "./OpenAIEmbedding";
+
+export class TogetherEmbedding extends OpenAIEmbedding {
+  override model: string;
+  constructor(init?: Partial<OpenAIEmbedding>) {
+    super({
+      apiKey: process.env.TOGETHER_API_KEY,
+      ...init,
+      additionalSessionOptions: {
+        ...init?.additionalSessionOptions,
+        baseURL: "https://api.together.xyz/v1",
+      },
+    });
+    this.model = init?.model ?? "togethercomputer/m2-bert-80M-32k-retrieval";
+  }
+}
@@ -21,6 +21,7 @@ export * from "./nodeParsers";
 export * from "./postprocessors";
 export * from "./readers/AssemblyAI";
 export * from "./readers/CSVReader";
+export * from "./readers/DocxReader";
 export * from "./readers/HTMLReader";
 export * from "./readers/MarkdownReader";
 export * from "./readers/NotionReader";
@@ -1,4 +1,4 @@
-import { v4 as uuidv4 } from "uuid";
+import { randomUUID } from "node:crypto";
 import { BaseNode, Document, jsonToNode } from "../Node";
 import { BaseQueryEngine } from "../QueryEngine";
 import { BaseRetriever } from "../Retriever";
@@ -16,7 +16,7 @@ export abstract class IndexStruct {
  indexId: string;
  summary?: string;

-  constructor(indexId = uuidv4(), summary = undefined) {
+  constructor(indexId = randomUUID(), summary = undefined) {
    this.indexId = indexId;
    this.summary = summary;
  }
@@ -129,7 +129,7 @@ export class OpenAI implements LLM {
  hasStreaming: boolean = true;

  // Per completion OpenAI params
-  model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS;
+  model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
  temperature: number;
  topP: number;
  maxTokens?: number;
@@ -205,12 +205,16 @@ export class OpenAI implements LLM {
  }

  get metadata() {
+    const contextWindow =
+      ALL_AVAILABLE_OPENAI_MODELS[
+        this.model as keyof typeof ALL_AVAILABLE_OPENAI_MODELS
+      ]?.contextWindow ?? 1024;
    return {
      model: this.model,
      temperature: this.temperature,
      topP: this.topP,
      maxTokens: this.maxTokens,
-      contextWindow: ALL_AVAILABLE_OPENAI_MODELS[this.model].contextWindow,
+      contextWindow,
      tokenizer: Tokenizers.CL100K_BASE,
    };
  }
@@ -1,3 +1,4 @@
 export * from "./LLM";
 export * from "./mistral";
 export { Ollama } from "./ollama";
+export { TogetherLLM } from "./together";
@@ -0,0 +1,14 @@
+import { OpenAI } from "./LLM";
+
+export class TogetherLLM extends OpenAI {
+  constructor(init?: Partial<OpenAI>) {
+    super({
+      ...init,
+      apiKey: process.env.TOGETHER_API_KEY,
+      additionalSessionOptions: {
+        ...init?.additionalSessionOptions,
+        baseURL: "https://api.together.xyz/v1",
+      },
+    });
+  }
+}
@@ -8,6 +8,7 @@ export * from "./indexStore/types";
 export { SimpleKVStore } from "./kvStore/SimpleKVStore";
 export * from "./kvStore/types";
 export { AstraDBVectorStore } from "./vectorStore/AstraDBVectorStore";
+export { ChromaVectorStore } from "./vectorStore/ChromaVectorStore";
 export { MongoDBAtlasVectorSearch } from "./vectorStore/MongoDBAtlasVectorStore";
 export { PGVectorStore } from "./vectorStore/PGVectorStore";
 export { PineconeVectorStore } from "./vectorStore/PineconeVectorStore";
@@ -0,0 +1,148 @@
+import {
+  AddParams,
+  ChromaClient,
+  ChromaClientParams,
+  Collection,
+  IncludeEnum,
+  QueryResponse,
+  Where,
+  WhereDocument,
+} from "chromadb";
+import { BaseNode, MetadataMode } from "../../Node";
+import {
+  VectorStore,
+  VectorStoreQuery,
+  VectorStoreQueryMode,
+  VectorStoreQueryResult,
+} from "./types";
+import { metadataDictToNode, nodeToMetadata } from "./utils";
+
+type ChromaDeleteOptions = {
+  where?: Where;
+  whereDocument?: WhereDocument;
+};
+
+type ChromaQueryOptions = {
+  whereDocument?: WhereDocument;
+};
+
+const DEFAULT_TEXT_KEY = "text";
+
+export class ChromaVectorStore implements VectorStore {
+  storesText: boolean = true;
+  flatMetadata: boolean = true;
+  textKey: string;
+  private chromaClient: ChromaClient;
+  private collection: Collection | null = null;
+  private collectionName: string;
+
+  constructor(init: {
+    collectionName: string;
+    textKey?: string;
+    chromaClientParams?: ChromaClientParams;
+  }) {
+    this.collectionName = init.collectionName;
+    this.chromaClient = new ChromaClient(init.chromaClientParams);
+    this.textKey = init.textKey ?? DEFAULT_TEXT_KEY;
+  }
+
+  client(): ChromaClient {
+    return this.chromaClient;
+  }
+
+  async getCollection(): Promise<Collection> {
+    if (!this.collection) {
+      const coll = await this.chromaClient.createCollection({
+        name: this.collectionName,
+      });
+      this.collection = coll;
+    }
+    return this.collection;
+  }
+
+  private getDataToInsert(nodes: BaseNode[]): AddParams {
+    const metadatas = nodes.map((node) =>
+      nodeToMetadata(node, true, this.textKey, this.flatMetadata),
+    );
+    return {
+      embeddings: nodes.map((node) => node.getEmbedding()),
+      ids: nodes.map((node) => node.id_),
+      metadatas,
+      documents: nodes.map((node) => node.getContent(MetadataMode.NONE)),
+    };
+  }
+
+  async add(nodes: BaseNode[]): Promise<string[]> {
+    if (!nodes || nodes.length === 0) {
+      return [];
+    }
+
+    const dataToInsert = this.getDataToInsert(nodes);
+    const collection = await this.getCollection();
+    await collection.add(dataToInsert);
+    return nodes.map((node) => node.id_);
+  }
+
+  async delete(
+    refDocId: string,
+    deleteOptions?: ChromaDeleteOptions,
+  ): Promise<void> {
+    const collection = await this.getCollection();
+    await collection.delete({
+      ids: [refDocId],
+      where: deleteOptions?.where,
+      whereDocument: deleteOptions?.whereDocument,
+    });
+  }
+
+  async query(
+    query: VectorStoreQuery,
+    options?: ChromaQueryOptions,
+  ): Promise<VectorStoreQueryResult> {
+    if (query.docIds) {
+      throw new Error("ChromaDB does not support querying by docIDs");
+    }
+    if (query.mode != VectorStoreQueryMode.DEFAULT) {
+      throw new Error("ChromaDB does not support querying by mode");
+    }
+
+    const chromaWhere: { [x: string]: string | number | boolean } = {};
+    if (query.filters) {
+      query.filters.filters.map((filter) => {
+        const filterKey = filter.key;
+        const filterValue = filter.value;
+        chromaWhere[filterKey] = filterValue;
+      });
+    }
+
+    const collection = await this.getCollection();
+    const queryResponse: QueryResponse = await collection.query({
+      queryEmbeddings: query.queryEmbedding ?? undefined,
+      queryTexts: query.queryStr ?? undefined,
+      nResults: query.similarityTopK,
+      where: Object.keys(chromaWhere).length ? chromaWhere : undefined,
+      whereDocument: options?.whereDocument,
+      //ChromaDB doesn't return the result embeddings by default so we need to include them
+      include: [
+        IncludeEnum.Distances,
+        IncludeEnum.Metadatas,
+        IncludeEnum.Documents,
+        IncludeEnum.Embeddings,
+      ],
+    });
+    const vectorStoreQueryResult: VectorStoreQueryResult = {
+      nodes: queryResponse.ids[0].map((id, index) => {
+        const text = (queryResponse.documents as string[][])[0][index];
+        const metaData = queryResponse.metadatas[0][index] ?? {};
+        const node = metadataDictToNode(metaData);
+        node.setContent(text);
+        return node;
+      }),
+      similarities: (queryResponse.distances as number[][])[0].map(
+        (distance) => 1 - distance,
+      ),
+      ids: queryResponse.ids[0],
+    };
+    return vectorStoreQueryResult;
+  }
+}
@@ -7,7 +7,9 @@ describe("SentenceSplitter", () => {
  });

  test("splits paragraphs w/o effective chunk size", () => {
-    const sentenceSplitter = new SentenceSplitter({});
+    const sentenceSplitter = new SentenceSplitter({
+      paragraphSeparator: "\n\n\n",
+    });
    // generate the same line as above but correct syntax errors
    let splits = sentenceSplitter.getParagraphSplits(
      "This is a paragraph.\n\n\nThis is another paragraph.",
@@ -0,0 +1,20 @@
+import { DocxReader } from "../../readers/DocxReader";
+
+describe("DocxReader", () => {
+  let docxReader: DocxReader;
+
+  beforeEach(() => {
+    docxReader = new DocxReader();
+  });
+
+  describe("loadData", () => {
+    it("should load data from a docx file, return an array of documents and contain text", async () => {
+      const filePath = "../../examples/data/stars.docx";
+      const docs = await docxReader.loadData(filePath);
+      const docContent = docs.map((doc) => doc.text).join("");
+
+      expect(docs).toBeInstanceOf(Array);
+      expect(docContent).toContain("Venturing into the zodiac");
+    });
+  });
+});
@@ -0,0 +1,20 @@
+import { MarkdownReader } from "../../readers/MarkdownReader";
+
+describe("MarkdownReader", () => {
+  let markdownReader: MarkdownReader;
+
+  beforeEach(() => {
+    markdownReader = new MarkdownReader();
+  });
+
+  describe("loadData", () => {
+    it("should load data from a markdown file, return an array of documents and contain text", async () => {
+      const filePath = "../../examples/data/planets.md";
+      const docs = await markdownReader.loadData(filePath);
+      const docContent = docs.map((doc) => doc.text).join("");
+
+      expect(docs).toBeInstanceOf(Array);
+      expect(docContent).toContain("Solar System");
+    });
+  });
+});
@@ -1,5 +1,7 @@
 {
  "compilerOptions": {
+    "rootDir": ".",
+    "outDir": "./lib/",
    "esModuleInterop": true,
    "forceConsistentCasingInFileNames": true,
    "isolatedModules": true,
@@ -10,8 +12,8 @@
    "strict": true,
    "lib": ["es2015", "dom"],
    "target": "ES2015",
-    "resolveJsonModule": true,
-    "typeRoots": ["./types", "./node_modules/@types"]
+    "resolveJsonModule": true
  },
+  "include": ["./src"],
  "exclude": ["node_modules"]
 }
@@ -1,5 +1,19 @@
 # create-llama

+## 0.0.14
+
+### Patch Changes
+
+- 2e6b36e: fix: re-organize file structure
+- 2b356c8: fix: relative path incorrect
+
+## 0.0.13
+
+### Patch Changes
+
+- Added PostgreSQL vector store (for Typescript and Python)
+- Improved async handling in FastAPI
+
 ## 0.0.12

 ### Patch Changes
@@ -9,8 +9,8 @@ import { makeDir } from "./helpers/make-dir";

 import fs from "fs";
 import terminalLink from "terminal-link";
-import type { InstallTemplateArgs } from "./templates";
-import { installTemplate } from "./templates";
+import type { InstallTemplateArgs } from "./helpers";
+import { installTemplate } from "./helpers";

 export type InstallAppArgs = Omit<
  InstallTemplateArgs,
@@ -94,7 +94,7 @@ export async function createApp({
    });
    // copy readme for fullstack
    await fs.promises.copyFile(
-      path.join(__dirname, "templates", "README-fullstack.md"),
+      path.join(__dirname, "..", "templates", "README-fullstack.md"),
      path.join(root, "README.md"),
    );
  } else {
@@ -8,7 +8,7 @@ import type {
  TemplateFramework,
  TemplateType,
  TemplateUI,
-} from "../templates";
+} from "../helpers";
 import { createTestDir, runApp, runCreateLlama, type AppType } from "./utils";

 const templateTypes: TemplateType[] = ["streaming", "simple"];
@@ -0,0 +1,12 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "tsBuildInfoFile": "./lib/.e2e.tsbuildinfo"
+  },
+  "include": ["./**/*.ts"],
+  "references": [
+    {
+      "path": ".."
+    }
+  ]
+}
@@ -1,14 +1,14 @@
-import { copy } from "../helpers/copy";
-import { callPackageManager } from "../helpers/install";
+import { copy } from "./copy";
+import { callPackageManager } from "./install";

 import fs from "fs/promises";
 import path from "path";
 import { cyan } from "picocolors";

-import { COMMUNITY_OWNER, COMMUNITY_REPO } from "../helpers/constant";
-import { PackageManager } from "../helpers/get-pkg-manager";
-import { downloadAndExtractRepo } from "../helpers/repo";
+import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./constant";
+import { PackageManager } from "./get-pkg-manager";
 import { installPythonTemplate } from "./python";
+import { downloadAndExtractRepo } from "./repo";
 import {
  InstallTemplateArgs,
  TemplateEngine,
@@ -71,7 +71,13 @@ const copyTestData = async (
  vectorDb?: TemplateVectorDB,
 ) => {
  if (engine === "context") {
-    const srcPath = path.join(__dirname, "components", "data");
+    const srcPath = path.join(
+      __dirname,
+      "..",
+      "templates",
+      "components",
+      "data",
+    );
    const destPath = path.join(root, "data");
    console.log(`\nCopying test data to ${cyan(destPath)}\n`);
    await copy("**", destPath, {
@@ -2,12 +2,13 @@ import fs from "fs/promises";
 import path from "path";
 import { cyan } from "picocolors";
 import { parse, stringify } from "smol-toml";
-import { copy } from "../helpers/copy";
+import { copy } from "./copy";
 import { InstallTemplateArgs, TemplateVectorDB } from "./types";

 interface Dependency {
  name: string;
-  version: string;
+  version?: string;
+  extras?: string[];
 }

 const getAdditionalDependencies = (vectorDb?: TemplateVectorDB) => {
@@ -21,12 +22,43 @@ const getAdditionalDependencies = (vectorDb?: TemplateVectorDB) => {
      });
      break;
    }
+    case "pg": {
+      dependencies.push({
+        name: "llama-index",
+        extras: ["postgres"],
+      });
+    }
  }

  return dependencies;
 };

-const addDependencies = async (
+const mergePoetryDependencies = (
+  dependencies: Dependency[],
+  existingDependencies: any,
+) => {
+  for (const dependency of dependencies) {
+    let value = existingDependencies[dependency.name] ?? {};
+
+    // default string value is equal to attribute "version"
+    if (typeof value === "string") {
+      value = { version: value };
+    }
+
+    value.version = dependency.version ?? value.version;
+    value.extras = dependency.extras ?? value.extras;
+
+    if (value.version === undefined) {
+      throw new Error(
+        `Dependency "${dependency.name}" is missing attribute "version"!`,
+      );
+    }
+
+    existingDependencies[dependency.name] = value;
+  }
+};
+
+export const addDependencies = async (
  projectDir: string,
  dependencies: Dependency[],
 ) => {
@@ -42,9 +74,7 @@ const addDependencies = async (
    // Modify toml dependencies
    const tool = fileParsed.tool as any;
    const existingDependencies = tool.poetry.dependencies as any;
-    for (const dependency of dependencies) {
-      existingDependencies[dependency.name] = dependency.version;
-    }
+    mergePoetryDependencies(dependencies, existingDependencies);

    // Write toml file
    const newFileContent = stringify(fileParsed);
@@ -71,7 +101,14 @@ export const installPythonTemplate = async ({
  "root" | "framework" | "template" | "engine" | "vectorDb"
 >) => {
  console.log("\nInitializing Python project with template:", template, "\n");
-  const templatePath = path.join(__dirname, "types", template, framework);
+  const templatePath = path.join(
+    __dirname,
+    "..",
+    "templates",
+    "types",
+    template,
+    framework,
+  );
  await copy("**", root, {
    parents: true,
    cwd: templatePath,
@@ -93,7 +130,7 @@ export const installPythonTemplate = async ({
  });

  if (engine === "context") {
-    const compPath = path.join(__dirname, "components");
+    const compPath = path.join(__dirname, "..", "templates", "components");
    const VectorDBPath = path.join(
      compPath,
      "vectordbs",
@@ -46,7 +46,14 @@ export const installTSTemplate = async ({
   * Copy the template files to the target directory.
   */
  console.log("\nInitializing project with template:", template, "\n");
-  const templatePath = path.join(__dirname, "types", template, framework);
+  const templatePath = path.join(
+    __dirname,
+    "..",
+    "templates",
+    "types",
+    template,
+    framework,
+  );
  const copySource = ["**"];
  if (!eslint) copySource.push("!eslintrc.json");

@@ -80,7 +87,7 @@ export const installTSTemplate = async ({
   * Copy the selected chat engine files to the target directory and reference it.
   */
  let relativeEngineDestPath;
-  const compPath = path.join(__dirname, "components");
+  const compPath = path.join(__dirname, "..", "templates", "components");
  if (engine && (framework === "express" || framework === "nextjs")) {
    console.log("\nUsing chat engine:", engine, "\n");

@@ -1,6 +1,6 @@
 {
  "name": "create-llama",
-  "version": "0.0.12",
+  "version": "0.0.14",
  "keywords": [
    "rag",
    "llamaindex",
@@ -17,7 +17,8 @@
    "create-llama": "./dist/index.js"
  },
  "files": [
-    "dist"
+    "./dist/index.js",
+    "./templates"
  ],
  "scripts": {
    "clean": "rimraf --glob ./dist ./templates/**/__pycache__ ./templates/**/node_modules ./templates/**/poetry.lock",
@@ -4,9 +4,9 @@ import path from "path";
 import { blue, green } from "picocolors";
 import prompts from "prompts";
 import { InstallAppArgs } from "./create-app";
+import { TemplateFramework } from "./helpers";
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant";
 import { getRepoRootFolders } from "./helpers/repo";
-import { TemplateFramework } from "./templates";

 export type QuestionArgs = Omit<InstallAppArgs, "appPath" | "packageManager">;

@@ -40,7 +40,7 @@ const getVectorDbChoices = (framework: TemplateFramework) => {
  ];

  const vectodbLang = framework === "fastapi" ? "python" : "typescript";
-  const compPath = path.join(__dirname, "components");
+  const compPath = path.join(__dirname, "..", "templates", "components");
  const vectordbPath = path.join(compPath, "vectordbs", vectodbLang);

  const availableChoices = fs
@@ -0,0 +1,5 @@
+DATA_DIR = "data"  # directory containing the documents to index
+CHUNK_SIZE = 1024
+CHUNK_OVERLAP = 20
+PGVECTOR_SCHEMA = "public"
+PGVECTOR_TABLE = "llamaindex_embedding"
@@ -0,0 +1,14 @@
+from llama_index import ServiceContext
+
+from app.context import create_base_context
+from app.engine.constants import CHUNK_SIZE, CHUNK_OVERLAP
+
+
+def create_service_context():
+    base = create_base_context()
+    return ServiceContext.from_defaults(
+        llm=base.llm,
+        embed_model=base.embed_model,
+        chunk_size=CHUNK_SIZE,
+        chunk_overlap=CHUNK_OVERLAP,
+    )
@@ -0,0 +1,38 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+import logging
+
+from app.engine.constants import DATA_DIR
+from app.engine.context import create_service_context
+from app.engine.utils import init_pg_vector_store_from_env
+
+from llama_index import (
+    SimpleDirectoryReader,
+    VectorStoreIndex,
+    StorageContext,
+)
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def generate_datasource(service_context):
+    logger.info("Creating new index")
+    # load the documents and create the index
+    documents = SimpleDirectoryReader(DATA_DIR).load_data()
+    store = init_pg_vector_store_from_env()
+    storage_context = StorageContext.from_defaults(vector_store=store)
+    VectorStoreIndex.from_documents(
+        documents,
+        service_context=service_context,
+        storage_context=storage_context,
+        show_progress=True,  # this will show you a progress bar as the embeddings are created
+    )
+    logger.info(
+        f"Successfully created embeddings in the PG vector store, schema={store.schema_name} table={store.table_name}"
+    )
+
+
+if __name__ == "__main__":
+    generate_datasource(create_service_context())
@@ -0,0 +1,16 @@
+import logging
+from llama_index import (
+    VectorStoreIndex,
+)
+from app.engine.context import create_service_context
+from app.engine.utils import init_pg_vector_store_from_env
+
+
+def get_chat_engine():
+    service_context = create_service_context()
+    logger = logging.getLogger("uvicorn")
+    logger.info("Connecting to index from PGVector...")
+    store = init_pg_vector_store_from_env()
+    index = VectorStoreIndex.from_vector_store(store, service_context)
+    logger.info("Finished connecting to index from PGVector.")
+    return index.as_chat_engine(similarity_top_k=5)
@@ -0,0 +1,27 @@
+import os
+from llama_index.vector_stores import PGVectorStore
+from urllib.parse import urlparse
+from app.engine.constants import PGVECTOR_SCHEMA, PGVECTOR_TABLE
+
+
+def init_pg_vector_store_from_env():
+    original_conn_string = os.environ.get("PG_CONNECTION_STRING")
+    if original_conn_string is None or original_conn_string == "":
+        raise ValueError("PG_CONNECTION_STRING environment variable is not set.")
+
+    # The PGVectorStore requires both two connection strings, one for psycopg2 and one for asyncpg
+    # Update the configured scheme with the psycopg2 and asyncpg schemes
+    original_scheme = urlparse(original_conn_string).scheme + "://"
+    conn_string = original_conn_string.replace(
+        original_scheme, "postgresql+psycopg2://"
+    )
+    async_conn_string = original_conn_string.replace(
+        original_scheme, "postgresql+asyncpg://"
+    )
+
+    return PGVectorStore(
+        connection_string=conn_string,
+        async_connection_string=async_conn_string,
+        schema_name=PGVECTOR_SCHEMA,
+        table_name=PGVECTOR_TABLE,
+    )
@@ -50,7 +50,7 @@ async def chat(
    ]

    # query chat engine
-    response = chat_engine.chat(lastMessage.content, messages)
+    response = await chat_engine.achat(lastMessage.content, messages)
    return _Result(
        result=_Message(role=MessageRole.ASSISTANT, content=response.response)
    )
@@ -49,11 +49,11 @@ async def chat(
    ]

    # query chat engine
-    response = chat_engine.stream_chat(lastMessage.content, messages)
+    response = await chat_engine.astream_chat(lastMessage.content, messages)

    # stream response
    async def event_generator():
-        for token in response.response_gen:
+        async for token in response.async_response_gen():
            # If client closes connection, stop sending events
            if await request.is_disconnected():
                break
@@ -1,11 +1,19 @@
 {
+  "extends": "../../tsconfig.json",
  "compilerOptions": {
    "target": "es2019",
    "moduleResolution": "node",
    "strict": true,
    "resolveJsonModule": true,
    "esModuleInterop": true,
-    "skipLibCheck": false
+    "skipLibCheck": true
  },
-  "exclude": ["templates", "dist"]
+  "include": [
+    "create-app.ts",
+    "index.ts",
+    "./helpers",
+    "questions.ts",
+    "package.json"
+  ],
+  "exclude": ["dist"]
 }
@@ -10,6 +10,7 @@ module.exports = {
          "REPLICATE_API_TOKEN",
          "ANTHROPIC_API_KEY",
          "ASSEMBLYAI_API_KEY",
+          "TOGETHER_API_KEY",

          "ASTRA_DB_APPLICATION_TOKEN",
          "ASTRA_DB_ENDPOINT",
@@ -1,22 +0,0 @@
-# Sweep AI turns bug fixes & feature requests into code changes (https://sweep.dev)
-# For details on our config file, check out our docs at https://docs.sweep.dev
-
-# If you use this be sure to frequently sync your default branch(main, master) to dev.
-branch: "main"
-# If you want to enable GitHub Actions for Sweep, set this to true.
-gha_enabled: False
-# This is the description of your project. It will be used by sweep when creating PRs. You can tell Sweep what's unique about your project, what frameworks you use, or anything else you want.
-# Here's an example: sweepai/sweep is a python project. The main api endpoints are in sweepai/api.py. Write code that adheres to PEP8.
-description: "LlamaIndexTS is a data framework in TypeScript for your LLM applications"
-
-sandbox:
-  install:
-    - npm install -g pnpm
-    - pnpm i
-    - pnpm add --save-dev prettier -w
-  check:
-    - pnpx prettier --write {file_path}
-    - pnpm eslint --fix {file_path}
-    - pnpx ts-node --type-check {file_path}
-    - pnpm test
-# Default Values: https://github.com/sweepai/sweep/blob/main/sweep.yaml
@@ -0,0 +1,32 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "outDir": "./lib",
+    "tsBuildInfoFile": "./lib/.tsbuildinfo",
+    "incremental": true,
+    "composite": true
+  },
+  "files": [],
+  "references": [
+    {
+      "path": "./apps/docs/tsconfig.json"
+    },
+    {
+      "path": "./packages/core"
+    },
+    {
+      "path": "./packages/create-llama"
+    },
+    {
+      "path": "./packages/create-llama/e2e"
+    },
+    {
+      "path": "./examples"
+    }
+  ]
+}
Author	SHA1	Message	Date
Alex Yang	b185bda5b1	RELEASING: Releasing 2 package(s) Releases: create-llama@0.0.14 llamaindex@0.0.45 [skip ci]	2024-01-14 18:14:55 -06:00
Alex Yang	d79804e271	docs: update README.md (#376 )	2024-01-14 18:10:55 -06:00
Alex Yang	2b356c8613	fix(create-llama): component choice (#377 )	2024-01-14 18:10:33 -06:00
Alex Yang	2e6b36ef4b	docs: update changelog (#374 )	2024-01-12 18:49:53 -06:00
Alex Yang	edd0f66234	feat: support Together AI (#373 )	2024-01-12 18:41:48 -06:00
Alex Yang	2da407d66c	fix: cover type check on all ts files (#372 )	2024-01-12 15:36:04 -06:00
Alex Yang	fa574f709e	chore(core): use `bunchee` to bundle (#370 )	2024-01-12 12:50:33 -06:00
Alex Yang	1e6171521b	refactor: use `crypto.randomUUID` (#371 ) Ref: https://nodejs.org/docs/latest/api/crypto.html#cryptorandomuuidoptions	2024-01-12 12:30:17 -06:00
Marcus Schiesser	3f3e4eca66	RELEASING: Releasing 1 package(s) Releases: llamaindex@0.0.44 [skip ci]	2024-01-12 14:21:11 +07:00
Thuc Pham	648482b0f1	Feat: Add support for ChromaDB (#310 ) Co-authored-by: Aarav Navani <38411399+oofmeister27@users.noreply.github.com>	2024-01-12 13:35:00 +07:00
Marcus Schiesser	bb46afe33d	RELEASING: Releasing 1 package(s) Releases: create-llama@0.0.13 [skip ci]	2024-01-11 14:30:38 +07:00
Marcus Schiesser	80f5914abf	fix: raise exception if PG_CONNECTION_STRING is empty (generated value)	2024-01-11 14:25:00 +07:00
Marcus Schiesser	1c4e7f9c3e	chore: change release process	2024-01-11 13:22:54 +07:00
Marcus Schiesser	e4ae6e9076	RELEASING: Releasing 1 package(s) Releases: llamaindex@0.0.43 [skip ci]	2024-01-11 13:19:23 +07:00
Marcus Schiesser	f93efa2ea1	fix: disabled sweep (was trying to work on a draft PR)	2024-01-11 11:02:24 +07:00
Thuc Pham	7d79365262	feat: add examples and docs for readers (#323 )	2024-01-11 11:42:13 +08:00
Huu Le (Lee)	555692207e	Feat[cl]: Add postgresql vector store for fastapi (#318 )	2024-01-11 11:03:12 +08:00
Alex Yang	fcc06b227a	fix(perf): use regex to spilt texts (#364 )	2024-01-10 17:27:50 -06:00
Thuc Pham	08a39790e4	fix: default separator not work for window os (#324 )	2024-01-10 17:07:08 +08:00
Marcus Schiesser	a8270082a0	fix: improve async handling in fastapi (#322 )	2024-01-10 11:55:13 +08:00