Update README.md

.changeset/chatty-parents-smell.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat(qdrant): Add Qdrant Vector DB

.changeset/curly-mangos-brake.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-update dependencies

.changeset/mighty-chicken-trade.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Preview: Add ingestion pipeline (incl. different strategies to handle doc store duplicates)

.changeset/nervous-numbers-swim.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add an option that allows the user to run the generated app

.changeset/popular-grapes-cheat.md

@@ -0,0 +1,16 @@
+---
+"llamaindex": patch
+---
+
+feat: use conditional exports
+
+The benefit of conditional exports is we split the llamaindex into different files. This will improve the tree shake if you are building web apps.
+
+This also requires node16 (see https://nodejs.org/api/packages.html#conditional-exports).
+
+If you are seeing typescript issue `TS2724`('llamaindex' has no exported member named XXX):
+
+1. update `moduleResolution` to `bundler` in `tsconfig.json`, more for the web applications like Next.js, and vite, but still works for ts-node or tsx.
+2. consider the ES module in your project, add `"type": "module"` into `package.json` and update `moduleResolution` to `node16` or `nodenext` in `tsconfig.json`.
+
+We still support both cjs and esm, but you should update `tsconfig.json` to make the typescript happy.

.changeset/three-hounds-count.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat(extractors): add keyword extractor and base extractor

.husky/pre-commit

@@ -1,3 +1,6 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
 pnpm format
 pnpm lint
 npx lint-staged

.husky/pre-push

@@ -1 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
 pnpm test

.npmrc

@@ -1,2 +1 @@
 auto-install-peers = true
-enable-pre-post-scripts = true

.prettierignore

@@ -1,5 +1,4 @@
 apps/docs/i18n
-apps/docs/docs/api
 pnpm-lock.yaml
 lib/
 dist/

apps/docs/CHANGELOG.md

@@ -1,7 +0,0 @@
-# docs
-
-## 0.0.1
-
-### Patch Changes
-
-- 3154f52: chore: add qdrant readme

apps/docs/docs/concepts.md

@@ -2,7 +2,7 @@
 sidebar_position: 3
 ---
 
-# Concepts
+# High-Level Concepts
 
 LlamaIndex.TS helps you build LLM-powered applications (e.g. Q&A, chatbot) over custom data.
 
@@ -18,7 +18,7 @@ LlamaIndex uses a two stage method when using an LLM with your data:
 1. **indexing stage**: preparing a knowledge base, and
 2. **querying stage**: retrieving relevant context from the knowledge to assist the LLM in responding to a question
 
-![](../_static/concepts/rag.jpg)
+![](./_static/concepts/rag.jpg)
 
 This process is also known as Retrieval Augmented Generation (RAG).
 
@@ -30,7 +30,7 @@ Let's explore each stage in detail.
 
 LlamaIndex.TS help you prepare the knowledge base with a suite of data connectors and indexes.
 
-![](../_static/concepts/indexing.jpg)
+![](./_static/concepts/indexing.jpg)
 
 [**Data Loaders**](./modules/high_level/data_loader.md):
 A data connector (i.e. `Reader`) ingest data from different data sources and data formats into a simple `Document` representation (text and simple metadata).
@@ -56,7 +56,7 @@ LlamaIndex provides composable modules that help you build and integrate RAG pip
 
 These building blocks can be customized to reflect ranking preferences, as well as composed to reason over multiple knowledge bases in a structured way.
 
-![](../_static/concepts/querying.jpg)
+![](./_static/concepts/querying.jpg)
 
 #### Building Blocks
 

apps/docs/docs/end_to_end.md

@@ -0,0 +1,49 @@
+---
+sidebar_position: 4
+---
+
+# End to End Examples
+
+We include several end-to-end examples using LlamaIndex.TS in the repository
+
+Check out the examples below or try them out and complete them in minutes with interactive Github Codespace tutorials provided by Dev-Docs [here](https://codespaces.new/team-dev-docs/lits-dev-docs-playground?devcontainer_path=.devcontainer%2Fjavascript_ltsquickstart%2Fdevcontainer.json):
+
+## [Chat Engine](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/chatEngine.ts)
+
+Read a file and chat about it with the LLM.
+
+## [Vector Index](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/vectorIndex.ts)
+
+Create a vector index and query it. The vector index will use embeddings to fetch the top k most relevant nodes. By default, the top k is 2.
+
+## [Summary Index](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/summaryIndex.ts)
+
+Create a list index and query it. This example also use the `LLMRetriever`, which will use the LLM to select the best nodes to use when generating answer.
+
+## [Save / Load an Index](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/storageContext.ts)
+
+Create and load a vector index. Persistance to disk in LlamaIndex.TS happens automatically once a storage context object is created.
+
+## [Customized Vector Index](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/vectorIndexCustomize.ts)
+
+Create a vector index and query it, while also configuring the `LLM`, the `ServiceContext`, and the `similarity_top_k`.
+
+## [OpenAI LLM](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/openai.ts)
+
+Create an OpenAI LLM and directly use it for chat.
+
+## [Llama2 DeuceLLM](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/llamadeuce.ts)
+
+Create a Llama-2 LLM and directly use it for chat.
+
+## [SubQuestionQueryEngine](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/subquestion.ts)
+
+Uses the `SubQuestionQueryEngine`, which breaks complex queries into multiple questions, and then aggreates a response across the answers to all sub-questions.
+
+## [Low Level Modules](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/lowlevel.ts)
+
+This example uses several low-level components, which removes the need for an actual query engine. These components can be used anywhere, in any application, or customized and sub-classed to meet your own needs.
+
+## [JSON Entity Extraction](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/jsonExtract.ts)
+
+Features OpenAI's chat API (using [`json_mode`](https://platform.openai.com/docs/guides/text-generation/json-mode)) to extract a JSON object from a sales call transcript.

apps/docs/docs/environments.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 2
+sidebar_position: 5
 ---
 
 # Environments

apps/docs/docs/examples/_category_.yml

@@ -1,2 +0,0 @@
-label: Examples
-position: 2

apps/docs/docs/examples/chat_engine.mdx

@@ -1,12 +0,0 @@
----
-sidebar_position: 1
----
-
-import CodeBlock from "@theme/CodeBlock";
-import CodeSource from "!raw-loader!../../../../examples/chatEngine";
-
-# Chat Engine
-
-Chat Engine is a class that allows you to create a chatbot from a retriever. It is a wrapper around a retriever that allows you to chat with it in a conversational manner.
-
-<CodeBlock language="ts">{CodeSource}</CodeBlock>

apps/docs/docs/examples/more_examples.mdx

@@ -1,7 +0,0 @@
----
-sidebar_position: 5
----
-
-# More examples
-
-You can check out more examples in the [examples](https://github.com/run-llama/LlamaIndexTS/tree/main/examples) folder of the repository.

apps/docs/docs/examples/save_load_index.mdx

@@ -1,10 +0,0 @@
----
-sidebar_position: 4
----
-
-import CodeBlock from "@theme/CodeBlock";
-import CodeSource from "!raw-loader!../../../../examples/storageContext";
-
-# Save/Load an Index
-
-<CodeBlock language="ts">{CodeSource}</CodeBlock>

apps/docs/docs/examples/summary_index.mdx

@@ -1,10 +0,0 @@
----
-sidebar_position: 3
----
-
-import CodeBlock from "@theme/CodeBlock";
-import CodeSource from "!raw-loader!../../../../examples/summaryIndex";
-
-# Summary Index
-
-<CodeBlock language="ts">{CodeSource}</CodeBlock>

apps/docs/docs/examples/vector_index.mdx

@@ -1,10 +0,0 @@
----
-sidebar_position: 2
----
-
-import CodeBlock from "@theme/CodeBlock";
-import CodeSource from "!raw-loader!../../../../examples/vectorIndex";
-
-# Vector Index
-
-<CodeBlock language="ts">{CodeSource}</CodeBlock>

apps/docs/docs/getting_started/_category_.yml

@@ -1,2 +0,0 @@
-label: Getting Started
-position: 1

apps/docs/docs/installation.mdx

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 0
+sidebar_position: 1
 ---
 
 # Installation and Setup

apps/docs/docs/introduction.md

@@ -39,7 +39,7 @@ For more complex applications, our lower-level APIs allow advanced users to cust
 
 Our documentation includes [Installation Instructions](./installation.mdx) and a [Starter Tutorial](./starter.md) to build your first application.
 
-Once you're up and running, [High-Level Concepts](./getting_started/concepts.md) has an overview of LlamaIndex's modular architecture. For more hands-on practical examples, look through our [End-to-End Tutorials](./end_to_end.md).
+Once you're up and running, [High-Level Concepts](./concepts.md) has an overview of LlamaIndex's modular architecture. For more hands-on practical examples, look through our [End-to-End Tutorials](./end_to_end.md).
 
 ## 🗺️ Ecosystem
 

apps/docs/docs/modules/high_level/_category_.yml

@@ -0,0 +1 @@
+label: High-Level Modules

apps/docs/docs/modules/high_level/documents_and_nodes.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 1
+sidebar_position: 0
 ---
 
 # Documents and Nodes

apps/docs/docs/modules/low_level/_category_.yml

@@ -0,0 +1 @@
+label: Low-Level Modules

apps/docs/docs/modules/low_level/llm.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 1
+sidebar_position: 0
 ---
 
 # LLM

apps/docs/docs/modules/observability/_category_.yml

@@ -1,2 +1 @@
 label: Observability
-position: 5

apps/docs/docs/modules/vectorStores/_category_.yml

@@ -1,2 +0,0 @@
-label: "Vector Stores"
-position: 0

apps/docs/docs/modules/vectorStores/qdrant.md

@@ -1,88 +0,0 @@
-# Qdrant Vector Store
-
-To run this example, you need to have a Qdrant instance running. You can run it with Docker:
-
-```bash
-docker pull qdrant/qdrant
-docker run -p 6333:6333 qdrant/qdrant
-```
-
-## Importing the modules
-
-```ts
-import fs from "node:fs/promises";
-import { Document, VectorStoreIndex, QdrantVectorStore } from "llamaindex";
-```
-
-## Load the documents
-
-```ts
-const path = "node_modules/llamaindex/examples/abramov.txt";
-const essay = await fs.readFile(path, "utf-8");
-```
-
-## Setup Qdrant
-
-```ts
-const vectorStore = new QdrantVectorStore({
-  url: "http://localhost:6333",
-  port: 6333,
-});
-```
-
-## Setup the index
-
-```ts
-const document = new Document({ text: essay, id_: path });
-
-const index = await VectorStoreIndex.fromDocuments([document], {
-  vectorStore,
-});
-```
-
-## Query the index
-
-```ts
-const queryEngine = index.asQueryEngine();
-
-const response = await queryEngine.query({
-  query: "What did the author do in college?",
-});
-
-// Output response
-console.log(response.toString());
-```
-
-## Full code
-
-```ts
-import fs from "node:fs/promises";
-import { Document, VectorStoreIndex, QdrantVectorStore } from "llamaindex";
-
-async function main() {
-  const path = "node_modules/llamaindex/examples/abramov.txt";
-  const essay = await fs.readFile(path, "utf-8");
-
-  const vectorStore = new QdrantVectorStore({
-    url: "http://localhost:6333",
-    port: 6333,
-  });
-
-  const document = new Document({ text: essay, id_: path });
-
-  const index = await VectorStoreIndex.fromDocuments([document], {
-    vectorStore,
-  });
-
-  const queryEngine = index.asQueryEngine();
-
-  const response = await queryEngine.query({
-    query: "What did the author do in college?",
-  });
-
-  // Output response
-  console.log(response.toString());
-}
-
-main().catch(console.error);
-```

apps/docs/docs/starter.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 1
+sidebar_position: 2
 ---
 
 # Starter Tutorial

apps/docs/i18n/ar/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ LlamaIndex.TS هو إطار بيانات لتطبيقات LLM لاستيعاب
 
 تتضمن وثائقنا [تعليمات التثبيت](./installation.mdx) و[دليل البداية](./starter.md) لبناء تطبيقك الأول.
 
-بمجرد أن تكون جاهزًا وتعمل ، يحتوي [مفاهيم عالية المستوى](./getting_started/concepts.md) على نظرة عامة على الهندسة المعمارية المتعددة المستويات لـ LlamaIndex. لمزيد من الأمثلة العملية التفصيلية ، يمكنك الاطلاع على [دروس النهاية إلى النهاية](./end_to_end.md).
+بمجرد أن تكون جاهزًا وتعمل ، يحتوي [مفاهيم عالية المستوى](./concepts.md) على نظرة عامة على الهندسة المعمارية المتعددة المستويات لـ LlamaIndex. لمزيد من الأمثلة العملية التفصيلية ، يمكنك الاطلاع على [دروس النهاية إلى النهاية](./end_to_end.md).
 
 ## 🗺️ النظام البيئي
 

apps/docs/i18n/bg/docusaurus-plugin-content-docs/current/introduction.md

@@ -43,7 +43,7 @@ LlamaIndex.TS предоставя основен набор от инструм
 
 Документацията ни включва [Инструкции за инсталиране](./installation.mdx) и [Урок за начинаещи](./starter.md), за да построите първото си приложение.
 
-След като сте готови, [Високо ниво концепции](./getting_started/concepts.md) представя общ преглед на модулната архитектура на LlamaIndex. За повече практически примери, разгледайте нашите [Уроци от начало до край](./end_to_end.md).
+След като сте готови, [Високо ниво концепции](./concepts.md) представя общ преглед на модулната архитектура на LlamaIndex. За повече практически примери, разгледайте нашите [Уроци от начало до край](./end_to_end.md).
 
 ## 🗺️ Екосистема
 

apps/docs/i18n/cat/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Per a aplicacions més complexes, les nostres API de nivell inferior permeten al
 
 La nostra documentació inclou [Instruccions d'Instal·lació](./installation.mdx) i un [Tutorial d'Inici](./starter.md) per a construir la vostra primera aplicació.
 
-Un cop tingueu tot a punt, [Conceptes de Nivell Alt](./getting_started/concepts.md) ofereix una visió general de l'arquitectura modular de LlamaIndex. Per a més exemples pràctics, consulteu els nostres [Tutorials de Principi a Fi](./end_to_end.md).
+Un cop tingueu tot a punt, [Conceptes de Nivell Alt](./concepts.md) ofereix una visió general de l'arquitectura modular de LlamaIndex. Per a més exemples pràctics, consulteu els nostres [Tutorials de Principi a Fi](./end_to_end.md).
 
 ## 🗺️ Ecosistema
 

apps/docs/i18n/cs/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Pro složitější aplikace naše API na nižší úrovni umožňuje pokročilý
 
 Naše dokumentace obsahuje [Návod k instalaci](./installation.mdx) a [Úvodní tutoriál](./starter.md) pro vytvoření vaší první aplikace.
 
-Jakmile jste připraveni, [Vysokoúrovňové koncepty](./getting_started/concepts.md) poskytují přehled o modulární architektuře LlamaIndexu. Pro více praktických příkladů se podívejte na naše [Tutoriály od začátku do konce](./end_to_end.md).
+Jakmile jste připraveni, [Vysokoúrovňové koncepty](./concepts.md) poskytují přehled o modulární architektuře LlamaIndexu. Pro více praktických příkladů se podívejte na naše [Tutoriály od začátku do konce](./end_to_end.md).
 
 ## 🗺️ Ekosystém
 

apps/docs/i18n/da/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Til mere komplekse applikationer giver vores API'er på lavere niveau avancerede
 
 Vores dokumentation inkluderer [Installationsinstruktioner](./installation.mdx) og en [Starter Tutorial](./starter.md) til at bygge din første applikation.
 
-Når du er i gang, giver [Højniveaukoncepter](./getting_started/concepts.md) et overblik over LlamaIndex's modulære arkitektur. For flere praktiske eksempler, kan du kigge igennem vores [End-to-End Tutorials](./end_to_end.md).
+Når du er i gang, giver [Højniveaukoncepter](./concepts.md) et overblik over LlamaIndex's modulære arkitektur. For flere praktiske eksempler, kan du kigge igennem vores [End-to-End Tutorials](./end_to_end.md).
 
 ## 🗺️ Økosystem
 

apps/docs/i18n/de/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Für komplexere Anwendungen ermöglichen unsere APIs auf niedrigerer Ebene fortg
 
 Unsere Dokumentation enthält [Installationsanweisungen](./installation.mdx) und ein [Einführungstutorial](./starter.md), um Ihre erste Anwendung zu erstellen.
 
-Sobald Sie bereit sind, bietet [High-Level-Konzepte](./getting_started/concepts.md) einen Überblick über die modulare Architektur von LlamaIndex. Für praktische Beispiele schauen Sie sich unsere [End-to-End-Tutorials](./end_to_end.md) an.
+Sobald Sie bereit sind, bietet [High-Level-Konzepte](./concepts.md) einen Überblick über die modulare Architektur von LlamaIndex. Für praktische Beispiele schauen Sie sich unsere [End-to-End-Tutorials](./end_to_end.md) an.
 
 ## 🗺️ Ökosystem
 

apps/docs/i18n/el/docusaurus-plugin-content-docs/current/introduction.md

@@ -43,7 +43,7 @@ slug: /
 
 Η τεκμηρίωσή μας περιλαμβάνει [Οδηγίες Εγκατάστασης](./installation.mdx) και ένα [Εισαγωγικό Εκπαιδευτικό Πρόγραμμα](./starter.md) για να δημιουργήσετε την πρώτη σας εφαρμογή.
 
-Αφού ξεκινήσετε, οι [Υψηλού Επιπέδου Έννοιες](./getting_started/concepts.md) παρέχουν μια επισκόπηση της μοντουλαρισμένης αρχιτεκτονικής του LlamaIndex. Για περισσότερα πρακτικά παραδείγματα, ρίξτε μια ματιά στα [Ολοκληρωμένα Εκπαιδευτικά Προγράμματα](./end_to_end.md).
+Αφού ξεκινήσετε, οι [Υψηλού Επιπέδου Έννοιες](./concepts.md) παρέχουν μια επισκόπηση της μοντουλαρισμένης αρχιτεκτονικής του LlamaIndex. Για περισσότερα πρακτικά παραδείγματα, ρίξτε μια ματιά στα [Ολοκληρωμένα Εκπαιδευτικά Προγράμματα](./end_to_end.md).
 
 ## 🗺️ Οικοσύστημα
 

apps/docs/i18n/es/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Para aplicaciones más complejas, nuestras API de nivel inferior permiten a los
 
 Nuestra documentación incluye [Instrucciones de instalación](./installation.mdx) y un [Tutorial de inicio](./starter.md) para construir tu primera aplicación.
 
-Una vez que estés en funcionamiento, [Conceptos de alto nivel](./getting_started/concepts.md) ofrece una visión general de la arquitectura modular de LlamaIndex. Para obtener ejemplos prácticos más detallados, consulta nuestros [Tutoriales de extremo a extremo](./end_to_end.md).
+Una vez que estés en funcionamiento, [Conceptos de alto nivel](./concepts.md) ofrece una visión general de la arquitectura modular de LlamaIndex. Para obtener ejemplos prácticos más detallados, consulta nuestros [Tutoriales de extremo a extremo](./end_to_end.md).
 
 ## 🗺️ Ecosistema
 

apps/docs/i18n/et/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Täpsemate rakenduste jaoks võimaldavad meie madalama taseme API-d edasijõudnu
 
 Meie dokumentatsioonis on [paigaldusjuhised](./installation.mdx) ja [algõpetus](./starter.md) oma esimese rakenduse loomiseks.
 
-Kui olete valmis ja töötate, siis [kõrgtasemel kontseptsioonid](./getting_started/concepts.md) annavad ülevaate LlamaIndexi moodularhitektuurist. Praktiliste näidete jaoks vaadake läbi meie [otsast lõpuni õpetused](./end_to_end.md).
+Kui olete valmis ja töötate, siis [kõrgtasemel kontseptsioonid](./concepts.md) annavad ülevaate LlamaIndexi moodularhitektuurist. Praktiliste näidete jaoks vaadake läbi meie [otsast lõpuni õpetused](./end_to_end.md).
 
 ## 🗺️ Ökosüsteem
 

apps/docs/i18n/fa/docusaurus-plugin-content-docs/current/introduction.md

@@ -43,7 +43,7 @@ API سطح بالای ما به کاربران مبتدی امکان استفا
 
 مستندات ما شامل [دستورالعمل نصب](./installation.mdx) و [آموزش شروع کار](./starter.md) برای ساخت اولین برنامه شما است.
 
-با راه اندازی و اجرا شدن، [مفاهیم سطح بالا](./getting_started/concepts.md) یک نمای کلی از معماری ماژولار لاماایندکس را ارائه می دهد. برای مثال های عملی بیشتر، به [آموزش های پایان به پایان](./end_to_end.md) مراجعه کنید.
+با راه اندازی و اجرا شدن، [مفاهیم سطح بالا](./concepts.md) یک نمای کلی از معماری ماژولار لاماایندکس را ارائه می دهد. برای مثال های عملی بیشتر، به [آموزش های پایان به پایان](./end_to_end.md) مراجعه کنید.
 
 "
 

apps/docs/i18n/fi/docusaurus-plugin-content-docs/current/introduction.md

@@ -43,7 +43,7 @@ Monimutkaisempiin sovelluksiin tarjoamme matalamman tason API:t, jotka mahdollis
 
 Dokumentaatiostamme löydät [asennusohjeet](./installation.mdx) ja [aloitusopetusohjelman](./starter.md) ensimmäisen sovelluksesi rakentamiseen.
 
-Kun olet päässyt vauhtiin, [Korkean tason käsitteet](./getting_started/concepts.md) antaa yleiskuvan LlamaIndexin modulaarisesta arkkitehtuurista. Lisää käytännön esimerkkejä löydät [Päästä päähän -opetusohjelmista](./end_to_end.md).
+Kun olet päässyt vauhtiin, [Korkean tason käsitteet](./concepts.md) antaa yleiskuvan LlamaIndexin modulaarisesta arkkitehtuurista. Lisää käytännön esimerkkejä löydät [Päästä päähän -opetusohjelmista](./end_to_end.md).
 
 ## 🗺️ Ekosysteemi
 

apps/docs/i18n/he/docusaurus-plugin-content-docs/current/introduction.md

@@ -43,7 +43,7 @@ API הרמה הגבוהה שלנו מאפשר למשתמשים מתחילים ל
 
 התיעוד שלנו כולל [הוראות התקנה](./installation.mdx) ו[מדריך התחלה](./starter.md) לבניית היישום הראשון שלך.
 
-כאשר אתה מוכן ורץ, [מושגים ברמה גבוהה](./getting_started/concepts.md) מציג סקירה על ארכיטקטורה מודולרית של LlamaIndex. לדוגמאות פרקטיות יותר, עיין ב[מדריכים מתקדמים מתחילה ועד סוף](./end_to_end.md).
+כאשר אתה מוכן ורץ, [מושגים ברמה גבוהה](./concepts.md) מציג סקירה על ארכיטקטורה מודולרית של LlamaIndex. לדוגמאות פרקטיות יותר, עיין ב[מדריכים מתקדמים מתחילה ועד סוף](./end_to_end.md).
 
 "
 

apps/docs/i18n/hi/docusaurus-plugin-content-docs/current/introduction.md

@@ -43,7 +43,7 @@ LlamaIndex.TS जावास्क्रिप्ट और TypeScript के
 
 हमारी दस्तावेज़ी में [स्थापना निर्देश](./installation.mdx) और [स्टार्टर ट्यूटोरियल](./starter.md) शामिल हैं, जिनका उपयोग करके आप अपना पहला एप्लिकेशन बना सकते हैं।
 
-एक बार जब आप शुरू हो जाएं, [उच्च स्तरीय अवधारणाएँ](./getting_started/concepts.md) में LlamaIndex की मॉड्यूलर आर्किटेक्चर का अवलोकन है। अधिक हैंड्स-ऑन प्रैक्टिकल उदाहरणों के लिए, हमारे [एंड-टू-एंड ट्यूटोरियल](./end_to_end.md) को देखें।
+एक बार जब आप शुरू हो जाएं, [उच्च स्तरीय अवधारणाएँ](./concepts.md) में LlamaIndex की मॉड्यूलर आर्किटेक्चर का अवलोकन है। अधिक हैंड्स-ऑन प्रैक्टिकल उदाहरणों के लिए, हमारे [एंड-टू-एंड ट्यूटोरियल](./end_to_end.md) को देखें।
 
 "
 

apps/docs/i18n/hr/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Za složenije aplikacije, naše API-je niže razine omogućuju naprednim korisni
 
 Naša dokumentacija uključuje [Upute za instalaciju](./installation.mdx) i [Uvodni vodič](./starter.md) za izgradnju vaše prve aplikacije.
 
-Kada ste spremni za rad, [Visokorazinski koncepti](./getting_started/concepts.md) pružaju pregled modularne arhitekture LlamaIndex-a. Za praktične primjere, pogledajte naše [Vodiče od početka do kraja](./end_to_end.md).
+Kada ste spremni za rad, [Visokorazinski koncepti](./concepts.md) pružaju pregled modularne arhitekture LlamaIndex-a. Za praktične primjere, pogledajte naše [Vodiče od početka do kraja](./end_to_end.md).
 
 ## 🗺️ Ekosustav
 

apps/docs/i18n/hu/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ A komplexebb alkalmazásokhoz a mélyebb szintű API-k lehetővé teszik a halad
 
 A dokumentációnk tartalmazza a [Telepítési utasításokat](./installation.mdx) és egy [Kezdő útmutatót](./starter.md) az első alkalmazás létrehozásához.
 
-Miután elindultál, a [Magas szintű fogalmak](./getting_started/concepts.md) áttekintést ad a LlamaIndex moduláris architektúrájáról. További gyakorlati példákért tekintsd meg az [End-to-End útmutatóinkat](./end_to_end.md).
+Miután elindultál, a [Magas szintű fogalmak](./concepts.md) áttekintést ad a LlamaIndex moduláris architektúrájáról. További gyakorlati példákért tekintsd meg az [End-to-End útmutatóinkat](./end_to_end.md).
 
 ## 🗺️ Ökoszisztéma
 

apps/docs/i18n/in/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Untuk aplikasi yang lebih kompleks, API tingkat lebih rendah kami memungkinkan p
 
 Dokumentasi kami mencakup [Instruksi Instalasi](./installation.mdx) dan [Tutorial Awal](./starter.md) untuk membangun aplikasi pertama Anda.
 
-Setelah Anda mulai, [Konsep Tingkat Tinggi](./getting_started/concepts.md) memberikan gambaran tentang arsitektur modular LlamaIndex. Untuk contoh praktis yang lebih mendalam, lihat [Tutorial End-to-End](./end_to_end.md).
+Setelah Anda mulai, [Konsep Tingkat Tinggi](./concepts.md) memberikan gambaran tentang arsitektur modular LlamaIndex. Untuk contoh praktis yang lebih mendalam, lihat [Tutorial End-to-End](./end_to_end.md).
 
 ## 🗺️ Ekosistem
 

apps/docs/i18n/it/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Per applicazioni più complesse, le nostre API di livello inferiore consentono a
 
 La nostra documentazione include le [Istruzioni di installazione](./installation.mdx) e un [Tutorial introduttivo](./starter.md) per creare la tua prima applicazione.
 
-Una volta che sei pronto, i [Concetti di alto livello](./getting_started/concepts.md) offrono una panoramica dell'architettura modulare di LlamaIndex. Per ulteriori esempi pratici, consulta i nostri [Tutorial end-to-end](./end_to_end.md).
+Una volta che sei pronto, i [Concetti di alto livello](./concepts.md) offrono una panoramica dell'architettura modulare di LlamaIndex. Per ulteriori esempi pratici, consulta i nostri [Tutorial end-to-end](./end_to_end.md).
 
 ## 🗺️ Ecosistema
 

apps/docs/i18n/ja/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ LlamaIndex.TSは、JavaScriptとTypeScriptを使用してLLMアプリを構築
 
 私たちのドキュメントには、[インストール手順](./installation.mdx)と[スターターチュートリアル](./starter.md)が含まれており、最初のアプリケーションの構築をサポートします。
 
-一度準備ができたら、[ハイレベルなコンセプト](./getting_started/concepts.md)では、LlamaIndexのモジュラーアーキテクチャの概要を説明しています。より実践的な例については、[エンドツーエンドのチュートリアル](./end_to_end.md)を参照してください。
+一度準備ができたら、[ハイレベルなコンセプト](./concepts.md)では、LlamaIndexのモジュラーアーキテクチャの概要を説明しています。より実践的な例については、[エンドツーエンドのチュートリアル](./end_to_end.md)を参照してください。
 
 ## 🗺️ エコシステム
 

apps/docs/i18n/ko/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ LlamaIndex.TS는 JavaScript와 TypeScript로 LLM 앱을 개발하는 모든 사
 
 저희 문서에는 [설치 지침](./installation.mdx)과 [스타터 튜토리얼](./starter.md)이 포함되어 있어 첫 번째 애플리케이션을 빌드할 수 있습니다.
 
-한 번 시작하면, [고수준 개념](./getting_started/concepts.md)에서 LlamaIndex의 모듈식 아키텍처에 대한 개요를 확인할 수 있습니다. 더 많은 실전 예제를 원하신다면, [End-to-End 튜토리얼](./end_to_end.md)을 참조해주세요.
+한 번 시작하면, [고수준 개념](./concepts.md)에서 LlamaIndex의 모듈식 아키텍처에 대한 개요를 확인할 수 있습니다. 더 많은 실전 예제를 원하신다면, [End-to-End 튜토리얼](./end_to_end.md)을 참조해주세요.
 
 "
 

apps/docs/i18n/lt/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Sudėtingesnėms programoms mūsų žemesnio lygio API leidžia pažengusiems na
 
 Mūsų dokumentacija apima [įdiegimo instrukcijas](./installation.mdx) ir [pradžios vadovą](./starter.md), skirtą sukurti pirmąją aplikaciją.
 
-Kai jau esate paleidę, [aukšto lygio konceptai](./getting_started/concepts.md) pateikia apžvalgą apie LlamaIndex modularią architektūrą. Norėdami gauti daugiau praktinių pavyzdžių, peržiūrėkite mūsų [nuo pradžių iki pabaigos vadovus](./end_to_end.md).
+Kai jau esate paleidę, [aukšto lygio konceptai](./concepts.md) pateikia apžvalgą apie LlamaIndex modularią architektūrą. Norėdami gauti daugiau praktinių pavyzdžių, peržiūrėkite mūsų [nuo pradžių iki pabaigos vadovus](./end_to_end.md).
 
 ## 🗺️ Ekosistema
 

apps/docs/i18n/lv/docusaurus-plugin-content-docs/current/introduction.md

@@ -43,7 +43,7 @@ Lielākām un sarežģītākām lietojumprogrammām mūsu zemāka līmeņa API
 
 Mūsu dokumentācijā ir iekļautas [Instalācijas instrukcijas](./installation.mdx) un [Sākuma pamācība](./starter.md), lai izveidotu savu pirmo lietojumprogrammu.
 
-Kad esat gatavs, [Augsta līmeņa koncepti](./getting_started/concepts.md) sniedz pārskatu par LlamaIndex modulāro arhitektūru. Lai iegūtu vairāk praktisku piemēru, apskatiet mūsu [Galēji līdz galam pamācības](./end_to_end.md).
+Kad esat gatavs, [Augsta līmeņa koncepti](./concepts.md) sniedz pārskatu par LlamaIndex modulāro arhitektūru. Lai iegūtu vairāk praktisku piemēru, apskatiet mūsu [Galēji līdz galam pamācības](./end_to_end.md).
 
 ## 🗺️ Ekosistēma
 

apps/docs/i18n/nl/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Voor complexere toepassingen stellen onze API's op lager niveau gevorderde gebru
 
 Onze documentatie bevat [Installatie-instructies](./installation.mdx) en een [Starterzelfstudie](./starter.md) om uw eerste toepassing te bouwen.
 
-Zodra u aan de slag bent, geeft [Hoog-niveau Concepten](./getting_started/concepts.md) een overzicht van de modulaire architectuur van LlamaIndex. Voor meer praktische voorbeelden kunt u onze [End-to-End Tutorials](./end_to_end.md) bekijken.
+Zodra u aan de slag bent, geeft [Hoog-niveau Concepten](./concepts.md) een overzicht van de modulaire architectuur van LlamaIndex. Voor meer praktische voorbeelden kunt u onze [End-to-End Tutorials](./end_to_end.md) bekijken.
 
 ## 🗺️ Ecosysteem
 

apps/docs/i18n/no/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ For mer komplekse applikasjoner lar våre lavnivå-APIer avanserte brukere tilpa
 
 Dokumentasjonen vår inkluderer [Installasjonsinstruksjoner](./installation.mdx) og en [Starterveiledning](./starter.md) for å bygge din første applikasjon.
 
-Når du er oppe og kjører, gir [Høynivåkonsepter](./getting_started/concepts.md) en oversikt over LlamaIndex sin modulære arkitektur. For mer praktiske eksempler, kan du se gjennom våre [End-to-End veiledninger](./end_to_end.md).
+Når du er oppe og kjører, gir [Høynivåkonsepter](./concepts.md) en oversikt over LlamaIndex sin modulære arkitektur. For mer praktiske eksempler, kan du se gjennom våre [End-to-End veiledninger](./end_to_end.md).
 
 ## 🗺️ Økosystem
 

apps/docs/i18n/pl/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Dla bardziej zaawansowanych aplikacji nasze API na niższym poziomie umożliwia
 
 Nasza dokumentacja zawiera [Instrukcje instalacji](./installation.mdx) oraz [Samouczek dla początkujących](./starter.md), który pomoże Ci zbudować swoją pierwszą aplikację.
 
-Gdy już będziesz gotowy, [Wysokopoziomowe koncepcje](./getting_started/concepts.md) zawierają przegląd modułowej architektury LlamaIndex. Jeśli chcesz zobaczyć praktyczne przykłady, zapoznaj się z naszymi [Samouczkami od początku do końca](./end_to_end.md).
+Gdy już będziesz gotowy, [Wysokopoziomowe koncepcje](./concepts.md) zawierają przegląd modułowej architektury LlamaIndex. Jeśli chcesz zobaczyć praktyczne przykłady, zapoznaj się z naszymi [Samouczkami od początku do końca](./end_to_end.md).
 
 ## 🗺️ Ekosystem
 

apps/docs/i18n/pt/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Para aplicativos mais complexos, nossas APIs de nível inferior permitem que usu
 
 Nossa documentação inclui [Instruções de Instalação](./installation.mdx) e um [Tutorial Inicial](./starter.md) para construir seu primeiro aplicativo.
 
-Depois de estar pronto para começar, [Conceitos de Alto Nível](./getting_started/concepts.md) oferece uma visão geral da arquitetura modular do LlamaIndex. Para exemplos práticos mais detalhados, consulte nossos [Tutoriais de Ponta a Ponta](./end_to_end.md).
+Depois de estar pronto para começar, [Conceitos de Alto Nível](./concepts.md) oferece uma visão geral da arquitetura modular do LlamaIndex. Para exemplos práticos mais detalhados, consulte nossos [Tutoriais de Ponta a Ponta](./end_to_end.md).
 
 ## 🗺️ Ecossistema
 

apps/docs/i18n/ro/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Pentru aplicații mai complexe, API-urile noastre de nivel inferior permit utili
 
 Documentația noastră include [Instrucțiuni de instalare](./installation.mdx) și un [Tutorial de pornire](./starter.md) pentru a construi prima ta aplicație.
 
-Odată ce ai început, [Concepte de nivel înalt](./getting_started/concepts.md) oferă o prezentare generală a arhitecturii modulare a LlamaIndex. Pentru mai multe exemple practice, consultă [Tutorialele de la cap la coadă](./end_to_end.md).
+Odată ce ai început, [Concepte de nivel înalt](./concepts.md) oferă o prezentare generală a arhitecturii modulare a LlamaIndex. Pentru mai multe exemple practice, consultă [Tutorialele de la cap la coadă](./end_to_end.md).
 
 ## 🗺️ Ecosistem
 

apps/docs/i18n/ru/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ LlamaIndex.TS предоставляет основной набор инстр
 
 Наша документация включает [Инструкции по установке](./installation.mdx) и [Стартовое руководство](./starter.md) для создания вашего первого приложения.
 
-Когда вы начнете работу, [Высокоуровневые концепции](./getting_started/concepts.md) предоставляют обзор модульной архитектуры LlamaIndex. Для более практических примеров руководство [Полный цикл руководств](./end_to_end.md) будет полезно.
+Когда вы начнете работу, [Высокоуровневые концепции](./concepts.md) предоставляют обзор модульной архитектуры LlamaIndex. Для более практических примеров руководство [Полный цикл руководств](./end_to_end.md) будет полезно.
 
 ## 🗺️ Экосистема
 

apps/docs/i18n/se/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Za složenije aplikacije, naše API-je na nižem nivou omogućavaju naprednim ko
 
 Naša dokumentacija uključuje [Uputstva za instalaciju](./installation.mdx) i [Uvodni tutorijal](./starter.md) za izgradnju vaše prve aplikacije.
 
-Kada ste spremni za rad, [Koncepti na visokom nivou](./getting_started/concepts.md) pružaju pregled modularne arhitekture LlamaIndex-a. Za praktične primere, pogledajte naše [Vodiče od početka do kraja](./end_to_end.md).
+Kada ste spremni za rad, [Koncepti na visokom nivou](./concepts.md) pružaju pregled modularne arhitekture LlamaIndex-a. Za praktične primere, pogledajte naše [Vodiče od početka do kraja](./end_to_end.md).
 
 ## 🗺️ Ekosistem
 

apps/docs/i18n/sk/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Za bolj kompleksne aplikacije naša nizkonivojska API omogoča naprednim uporabn
 
 Naša dokumentacija vključuje [Navodila za namestitev](./installation.mdx) in [Vodič za začetek](./starter.md), ki vam pomagata zgraditi vašo prvo aplikacijo.
 
-Ko ste pripravljeni, [Visokonivojski koncepti](./getting_started/concepts.md) ponujajo pregled modularne arhitekture LlamaIndex-a. Za več praktičnih primerov si oglejte naše [Vodiče od začetka do konca](./end_to_end.md).
+Ko ste pripravljeni, [Visokonivojski koncepti](./concepts.md) ponujajo pregled modularne arhitekture LlamaIndex-a. Za več praktičnih primerov si oglejte naše [Vodiče od začetka do konca](./end_to_end.md).
 
 ## 🗺️ Ekosistem
 

apps/docs/i18n/sl/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ Pre zložitejšie aplikácie naša nižšia úroveň API umožňuje pokročilým
 
 Naša dokumentácia obsahuje [Inštalačné pokyny](./installation.mdx) a [Úvodný tutoriál](./starter.md) pre vytvorenie vašej prvej aplikácie.
 
-Keď už máte všetko pripravené, [Vysokoúrovňové koncepty](./getting_started/concepts.md) poskytujú prehľad o modulárnej architektúre LlamaIndexu. Pre viac praktických príkladov si prečítajte naše [Tutoriály od začiatku do konca](./end_to_end.md).
+Keď už máte všetko pripravené, [Vysokoúrovňové koncepty](./concepts.md) poskytujú prehľad o modulárnej architektúre LlamaIndexu. Pre viac praktických príkladov si prečítajte naše [Tutoriály od začiatku do konca](./end_to_end.md).
 
 ## 🗺️ Ekosystém
 

apps/docs/i18n/sv/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ För mer komplexa applikationer tillåter våra lägre nivå-API:er avancerade a
 
 Vår dokumentation inkluderar [Installationsinstruktioner](./installation.mdx) och en [Starterhandledning](./starter.md) för att bygga din första applikation.
 
-När du är igång, ger [Högnivåkoncept](./getting_started/concepts.md) en översikt över LlamaIndex modulära arkitektur. För mer praktiska exempel, titta igenom våra [Steg-för-steg handledningar](./end_to_end.md).
+När du är igång, ger [Högnivåkoncept](./concepts.md) en översikt över LlamaIndex modulära arkitektur. För mer praktiska exempel, titta igenom våra [Steg-för-steg handledningar](./end_to_end.md).
 
 ## 🗺️ Ekosystem
 

apps/docs/i18n/th/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ API ระดับสูงของเราช่วยให้ผู้ใ
 
 เอกสารของเราประกอบด้วย[คำแนะนำการติดตั้ง](./installation.mdx)และ[บทแนะนำเบื้องต้น](./starter.md)เพื่อสร้างแอปพลิเคชันครั้งแรกของคุณ
 
-เมื่อคุณเริ่มใช้งานแล้ว [แนวคิดระดับสูง](./getting_started/concepts.md) มีภาพรวมของสถาปัตยกรรมแบบโมดูลของ LlamaIndex สำหรับตัวอย่างที่เป็นปฏิบัติจริงมากขึ้น โปรดดูที่ [บทแนะนำจบสู่จบ](./end_to_end.md) เพื่อตัวอย่างที่ใช้งานได้จริง
+เมื่อคุณเริ่มใช้งานแล้ว [แนวคิดระดับสูง](./concepts.md) มีภาพรวมของสถาปัตยกรรมแบบโมดูลของ LlamaIndex สำหรับตัวอย่างที่เป็นปฏิบัติจริงมากขึ้น โปรดดูที่ [บทแนะนำจบสู่จบ](./end_to_end.md) เพื่อตัวอย่างที่ใช้งานได้จริง
 
 "
 

apps/docs/i18n/tr/docusaurus-plugin-content-docs/current/introduction.md

@@ -43,7 +43,7 @@ Daha karmaşık uygulamalar için, düşük seviyeli API'larımız, gelişmiş k
 
 Dökümantasyonumuz, [Kurulum Talimatları](./installation.mdx) ve ilk uygulamanızı oluşturmanız için bir [Başlangıç Kılavuzu](./starter.md) içerir.
 
-Çalışmaya başladıktan sonra, [Yüksek Düzeyli Kavramlar](./getting_started/concepts.md) LlamaIndex'in modüler mimarisinin bir genel bakışını sunar. Daha fazla pratik örnek için [Uçtan Uca Öğreticilerimize](./end_to_end.md) göz atabilirsiniz.
+Çalışmaya başladıktan sonra, [Yüksek Düzeyli Kavramlar](./concepts.md) LlamaIndex'in modüler mimarisinin bir genel bakışını sunar. Daha fazla pratik örnek için [Uçtan Uca Öğreticilerimize](./end_to_end.md) göz atabilirsiniz.
 
 ## 🗺️ Ekosistem
 

apps/docs/i18n/uk/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ LlamaIndex.TS надає основний набір інструментів,
 
 Наша документація містить [Інструкції з встановлення](./installation.mdx) та [Посібник для початківців](./starter.md) для створення вашої першої програми.
 
-Після того, як ви розпочнете роботу, [Високорівневі концепції](./getting_started/concepts.md) містить огляд модульної архітектури LlamaIndex. Для більш практичних прикладів роботи, перегляньте наші [Посібники з кінця в кінець](./end_to_end.md).
+Після того, як ви розпочнете роботу, [Високорівневі концепції](./concepts.md) містить огляд модульної архітектури LlamaIndex. Для більш практичних прикладів роботи, перегляньте наші [Посібники з кінця в кінець](./end_to_end.md).
 
 ## 🗺️ Екосистема
 

apps/docs/i18n/vi/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ API cấp cao của chúng tôi cho phép người dùng mới bắt đầu sử
 
 Tài liệu của chúng tôi bao gồm [Hướng dẫn cài đặt](./installation.mdx) và [Hướng dẫn bắt đầu](./starter.md) để xây dựng ứng dụng đầu tiên của bạn.
 
-Khi bạn đã sẵn sàng, [Khái niệm cấp cao](./getting_started/concepts.md) cung cấp một cái nhìn tổng quan về kiến trúc mô-đun của LlamaIndex. Để có thêm ví dụ thực tế, hãy xem qua [Hướng dẫn từ đầu đến cuối](./end_to_end.md).
+Khi bạn đã sẵn sàng, [Khái niệm cấp cao](./concepts.md) cung cấp một cái nhìn tổng quan về kiến trúc mô-đun của LlamaIndex. Để có thêm ví dụ thực tế, hãy xem qua [Hướng dẫn từ đầu đến cuối](./end_to_end.md).
 
 ## 🗺️ Hệ sinh thái
 

apps/docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/introduction.md

@@ -39,7 +39,7 @@ LlamaIndex.TS 提供了一套核心工具，对于任何使用JavaScript和TypeS
 
 我们的文档包括[安装说明](./installation.mdx)和一个[入门教程](./starter.md)，帮助你构建第一个应用程序。
 
-一旦你开始运行，[高级概念](./getting_started/concepts.md)有一个LlamaIndex模块化架构的概览。更多实践例子，请浏览我们的[端到端教程](./end_to_end.md)。
+一旦你开始运行，[高级概念](./concepts.md)有一个LlamaIndex模块化架构的概览。更多实践例子，请浏览我们的[端到端教程](./end_to_end.md)。
 
 ## 🗺️ 生态系统
 

apps/docs/i18n/zh_tw/docusaurus-plugin-content-docs/current/introduction.md

@@ -41,7 +41,7 @@ LlamaIndex.TS 提供了一組核心工具，對於使用 JavaScript 和 TypeScri
 
 我們的文檔包括[安裝說明](./installation.mdx)和[入門教程](./starter.md)，以構建您的第一個應用程序。
 
-一旦您開始運行，[高級概念](./getting_started/concepts.md)提供了 LlamaIndex 模塊化架構的概述。如果需要更多實際的操作示例，請查看我們的[端到端教程](./end_to_end.md)。
+一旦您開始運行，[高級概念](./concepts.md)提供了 LlamaIndex 模塊化架構的概述。如果需要更多實際的操作示例，請查看我們的[端到端教程](./end_to_end.md)。
 
 ## 🗺️ 生態系統
 

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.1",
+  "version": "0.0.0",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
@@ -15,8 +15,8 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "^3.1.1",
-    "@docusaurus/remark-plugin-npm2yarn": "^3.1.1",
+    "@docusaurus/core": "^3.1.0",
+    "@docusaurus/remark-plugin-npm2yarn": "^3.1.0",
     "@mdx-js/react": "^3.0.0",
     "clsx": "^2.1.0",
     "postcss": "^8.4.33",
@@ -27,11 +27,11 @@
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "3.1.0",
-    "@docusaurus/preset-classic": "^3.1.1",
-    "@docusaurus/theme-classic": "^3.1.1",
-    "@docusaurus/types": "^3.1.1",
+    "@docusaurus/preset-classic": "^3.1.0",
+    "@docusaurus/theme-classic": "^3.1.0",
+    "@docusaurus/types": "^3.1.0",
     "@tsconfig/docusaurus": "^2.0.2",
-    "@types/node": "^18.19.10",
+    "@types/node": "^18.19.6",
     "docusaurus-plugin-typedoc": "^0.22.0",
     "typedoc": "^0.25.7",
     "typedoc-plugin-markdown": "^3.17.1",

examples/jupyter/.gitignore

@@ -1 +0,0 @@
-.ipynb_checkpoints/

examples/jupyter/README.md

@@ -1,31 +0,0 @@
-# Jupyter examples
-
-## Preparation
-
-1. Install Deno, e.g. on macOS:
-
-```
-brew install deno
-```
-
-2. Install Jupyter
-
-```
-pip3 install jupyterlab
-```
-
-3. Install Deno kernel
-
-```
-deno jupyter --unstable --install
-```
-
-4. Run Jupyter
-
-```
-jupyter lab
-```
-
-## Run examples
-
-Then you can open in Jupyter any of the examples in this directory.

examples/jupyter/nodeparser.ipynb

@@ -1,82 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "8be89595-8885-4d5e-b1da-1df04eda5c7a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import {\n",
-    "  Document,\n",
-    "  SimpleNodeParser\n",
-    "} from \"npm:llamaindex\";"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "65de03f9-455a-4c59-9089-093cb6998af7",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[\n",
-      "  TextNode {\n",
-      "    id_: \u001b[32m\"1b2ab25e-562a-4821-bdde-860bd23121c1\"\u001b[39m,\n",
-      "    metadata: {},\n",
-      "    excludedEmbedMetadataKeys: [],\n",
-      "    excludedLlmMetadataKeys: [],\n",
-      "    relationships: {\n",
-      "      SOURCE: {\n",
-      "        nodeId: \u001b[32m\"0cb8de0e-845f-4e73-a7bd-f04426aacfed\"\u001b[39m,\n",
-      "        metadata: {},\n",
-      "        hash: \u001b[32m\"jatVVXETDFjV2fV1/fbTrdpY6ZGnSYekq9m1X/Ff1qs=\"\u001b[39m\n",
-      "      }\n",
-      "    },\n",
-      "    hash: \u001b[32m\"zVyeDsfMwWH1CqK2269o5uzGWl/DpIWO4ZcVCuyENi4=\"\u001b[39m,\n",
-      "    text: \u001b[32m\"I am 10 years old. John is 20 years old.\"\u001b[39m,\n",
-      "    metadataSeparator: \u001b[32m\"\\n\"\u001b[39m\n",
-      "  }\n",
-      "]\n"
-     ]
-    }
-   ],
-   "source": [
-    "const nodeParser = new SimpleNodeParser();\n",
-    "const nodes = nodeParser.getNodesFromDocuments([\n",
-    "    new Document({ text: \"I am 10 years old. John is 20 years old.\" }),\n",
-    "]);\n",
-    "\n",
-    "console.log(nodes);"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "fc0ec12f-2062-47af-916d-7c77ca39433a",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Deno",
-   "language": "typescript",
-   "name": "deno"
-  },
-  "language_info": {
-   "file_extension": ".ts",
-   "mimetype": "text/x.typescript",
-   "name": "typescript",
-   "nb_converter": "script",
-   "pygments_lexer": "typescript",
-   "version": "5.3.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

examples/jupyter/vectorIndex.ipynb

@@ -1,83 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "8be89595-8885-4d5e-b1da-1df04eda5c7a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import {\n",
-    "  Document,\n",
-    "  VectorStoreIndex\n",
-    "} from \"npm:llamaindex\";"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "65de03f9-455a-4c59-9089-093cb6998af7",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "In college, the author studied subjects such as linear algebra and physics, but did not find them particularly interesting. They also slacked off and skipped lectures, leading to gaps in their knowledge. They had a negative experience with their English classes and became resentful and suspicious of higher education. They eventually dropped out of college and did not return until five years later to pick up their papers.\n"
-     ]
-    }
-   ],
-   "source": [
-    "// Create Document object with essay\n",
-    "const resp = await fetch('https://raw.githubusercontent.com/run-llama/LlamaIndexTS/main/packages/core/examples/abramov.txt');\n",
-    "const text = await resp.text();\n",
-    "const document = new Document({ text });\n",
-    "\n",
-    "// Split text and create embeddings. Store them in a VectorStoreIndex\n",
-    "const index = await VectorStoreIndex.fromDocuments([document]);\n",
-    "\n",
-    "// Query the index\n",
-    "const queryEngine = index.asQueryEngine();\n",
-    "const response = await queryEngine.query({\n",
-    "    query: \"What did the author do in college?\",\n",
-    "});\n",
-    "\n",
-    "// Output response\n",
-    "console.log(response.toString());"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "fc0ec12f-2062-47af-916d-7c77ca39433a",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "68bdd292-5cf7-46e0-8646-51be1f070ad6",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Deno",
-   "language": "typescript",
-   "name": "deno"
-  },
-  "language_info": {
-   "file_extension": ".ts",
-   "mimetype": "text/x.typescript",
-   "name": "typescript",
-   "nb_converter": "script",
-   "pygments_lexer": "typescript",
-   "version": "5.3.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

examples/package.json

@@ -2,18 +2,18 @@
   "name": "examples",
   "private": true,
   "dependencies": {
-    "@datastax/astra-db-ts": "^0.1.4",
+    "@datastax/astra-db-ts": "^0.1.2",
     "@notionhq/client": "^2.2.14",
-    "@pinecone-database/pinecone": "^1.1.3",
-    "chromadb": "^1.8.1",
+    "@pinecone-database/pinecone": "^1.1.2",
+    "chromadb": "^1.7.3",
     "commander": "^11.1.0",
-    "dotenv": "^16.4.1",
-    "llamaindex": "workspace:^",
+    "dotenv": "^16.3.1",
+    "llamaindex": "latest",
     "mongodb": "^6.2.0"
   },
   "devDependencies": {
-    "@types/node": "^18.19.10",
-    "ts-node": "^10.9.2"
+    "@types/node": "^18.18.6",
+    "ts-node": "^10.9.1"
   },
   "scripts": {
     "lint": "eslint ."

examples/together-ai.ts

@@ -23,7 +23,6 @@ const together = new TogetherLLM({
   for await (const message of generator) {
     process.stdout.write(message.delta);
   }
-  console.log();
   const embedding = new TogetherEmbedding();
   const vector = await embedding.getTextEmbedding("Hello world!");
   console.log("vector:", vector);

examples/vectorIndexEmbed3.ts

@@ -1,41 +0,0 @@
-import fs from "node:fs/promises";
-
-import {
-  Document,
-  OpenAIEmbedding,
-  VectorStoreIndex,
-  serviceContextFromDefaults,
-} from "llamaindex";
-
-async function main() {
-  // Load essay from abramov.txt in Node
-  const path = "node_modules/llamaindex/examples/abramov.txt";
-
-  const essay = await fs.readFile(path, "utf-8");
-
-  // Create Document object with essay
-  const document = new Document({ text: essay, id_: path });
-
-  // Create service context and specify text-embedding-3-large
-  const embedModel = new OpenAIEmbedding({
-    model: "text-embedding-3-large",
-    dimensions: 1024,
-  });
-  const serviceContext = serviceContextFromDefaults({ embedModel });
-
-  // Split text and create embeddings. Store them in a VectorStoreIndex
-  const index = await VectorStoreIndex.fromDocuments([document], {
-    serviceContext,
-  });
-
-  // Query the index
-  const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query({
-    query: "What did the author do in college?",
-  });
-
-  // Output response
-  console.log(response.toString());
-}
-
-main().catch(console.error);

package.json

@@ -8,30 +8,29 @@
     "format": "prettier --ignore-unknown --cache --check .",
     "format:write": "prettier --ignore-unknown --write .",
     "lint": "turbo run lint",
-    "prepare": "husky",
+    "prepare": "husky install",
     "test": "turbo run test",
     "type-check": "tsc -b --diagnostics",
     "release": "pnpm run build:release && changeset publish",
-    "new-llamaindex": "pnpm run build:release && changeset version --ignore create-llama",
-    "new-create-llama": "pnpm run build:release && changeset version --ignore llamaindex",
-    "new-snapshots": "pnpm run build:release && changeset version --snapshot"
+    "new-version": "pnpm run build:release && changeset version",
+    "new-snapshot": "pnpm run build:release && changeset version --snapshot"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",
-    "@turbo/gen": "^1.11.3",
+    "@turbo/gen": "^1.11.2",
     "@types/jest": "^29.5.11",
     "eslint": "^8.56.0",
     "eslint-config-custom": "workspace:*",
-    "husky": "^9.0.6",
+    "husky": "^8.0.3",
     "jest": "^29.7.0",
     "lint-staged": "^15.2.0",
     "prettier": "^3.2.4",
     "prettier-plugin-organize-imports": "^3.2.4",
-    "ts-jest": "^29.1.2",
-    "turbo": "^1.11.3",
+    "ts-jest": "^29.1.1",
+    "turbo": "^1.11.2",
     "typescript": "^5.3.3"
   },
-  "packageManager": "pnpm@8.14.3+sha256.2d0363bb6c314daa67087ef07743eea1ba2e2d360c835e8fec6b5575e4ed9484",
+  "packageManager": "pnpm@8.10.5+sha256.a4bd9bb7b48214bbfcd95f264bd75bb70d100e5d4b58808f5cd6ab40c6ac21c5",
   "pnpm": {
     "overrides": {
       "trim": "1.0.1",

packages/core/CHANGELOG.md

@@ -1,56 +1,5 @@
 # llamaindex
 
-## 0.1.2
-
-- e4b807a: fix: invalid package.json
-
-## 0.1.1
-
-No changes for this release.
-
-## 0.1.0
-
-### Minor Changes
-
-- 3154f52: chore: add qdrant readme
-
-### Patch Changes
-
-- bb66cb7: add new OpenAI embeddings (with dimension reduction support)
-
-## 0.0.51
-
-### Patch Changes
-
-- fda8024: revert: export conditions not working with moduleResolution `node`
-
-## 0.0.50
-
-### Patch Changes
-
-- 8a729cd: fix bugs in Together.AI integration (thanks @Nutlope for reporting)
-
-## 0.0.49
-
-### Patch Changes
-
-- eee3922: feat(qdrant): Add Qdrant Vector DB
-- e2790da: Preview: Add ingestion pipeline (incl. different strategies to handle doc store duplicates)
-- bff40f2: feat: use conditional exports
-
-  The benefit of conditional exports is we split the llamaindex into different files. This will improve the tree shake if you are building web apps.
-
-  This also requires node16 (see https://nodejs.org/api/packages.html#conditional-exports).
-
-  If you are seeing typescript issue `TS2724`('llamaindex' has no exported member named XXX):
-
-  1. update `moduleResolution` to `bundler` in `tsconfig.json`, more for the web applications like Next.js, and vite, but still works for ts-node or tsx.
-  2. consider the ES module in your project, add `"type": "module"` into `package.json` and update `moduleResolution` to `node16` or `nodenext` in `tsconfig.json`.
-
-  We still support both cjs and esm, but you should update `tsconfig.json` to make the typescript happy.
-
-- 2d8845b: feat(extractors): add keyword extractor and base extractor
-
 ## 0.0.48
 
 ### Patch Changes

packages/core/README.md

@@ -0,0 +1,130 @@
+# LlamaIndex.TS
+
+[![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
+
+LlamaIndex is a data framework for your LLM application.
+
+Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.
+
+Documentation: https://ts.llamaindex.ai/
+
+## What is LlamaIndex.TS?
+
+LlamaIndex.TS aims to be a lightweight, easy to use set of libraries to help you integrate large language models into your applications with your own data.
+
+## Getting started with an example:
+
+LlamaIndex.TS requires Node v18 or higher. You can download it from https://nodejs.org or use https://nvm.sh (our preferred option).
+
+In a new folder:
+
+```bash
+export OPENAI_API_KEY="sk-......" # Replace with your key from https://platform.openai.com/account/api-keys
+pnpm init
+pnpm install typescript
+pnpm exec tsc --init # if needed
+pnpm install llamaindex
+pnpm install @types/node
+```
+
+Create the file example.ts
+
+```ts
+// example.ts
+import fs from "fs/promises";
+import { Document, VectorStoreIndex } from "llamaindex";
+
+async function main() {
+  // Load essay from abramov.txt in Node
+  const essay = await fs.readFile(
+    "node_modules/llamaindex/examples/abramov.txt",
+    "utf-8",
+  );
+
+  // Create Document object with essay
+  const document = new Document({ text: essay });
+
+  // Split text and create embeddings. Store them in a VectorStoreIndex
+  const index = await VectorStoreIndex.fromDocuments([document]);
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query(
+    "What did the author do in college?",
+  );
+
+  // Output response
+  console.log(response.toString());
+}
+
+main();
+```
+
+Then you can run it using
+
+```bash
+pnpx ts-node example.ts
+```
+
+## Playground
+
+Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground
+
+## Core concepts for getting started:
+
+- [Document](/packages/core/src/Node.ts): A document represents a text file, PDF file or other contiguous piece of data.
+
+- [Node](/packages/core/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
+
+- [Embedding](/packages/core/src/Embedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton.
+
+- [Indices](/packages/core/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
+
+- [QueryEngine](/packages/core/src/QueryEngine.ts): Query engines are what generate the query you put in and give you back the result. Query engines generally combine a pre-built prompt with selected Nodes from your Index to give the LLM the context it needs to answer your query.
+
+- [ChatEngine](/packages/core/src/ChatEngine.ts): A ChatEngine helps you build a chatbot that will interact with your Indices.
+
+- [SimplePrompt](/packages/core/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
+
+## Note: NextJS:
+
+If you're using NextJS App Router, you'll need to use the NodeJS runtime (default) and add the following config to your next.config.js to have it use imports/exports in the same way Node does.
+
+```js
+export const runtime = "nodejs"; // default
+```
+
+```js
+// next.config.js
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  webpack: (config) => {
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      sharp$: false,
+      "onnxruntime-node$": false,
+    };
+    return config;
+  },
+};
+
+module.exports = nextConfig;
+```
+
+## Supported LLMs:
+
+- OpenAI GPT-3.5-turbo and GPT-4
+- Anthropic Claude Instant and Claude 2
+- Llama2 Chat LLMs (70B, 13B, and 7B parameters)
+- MistralAI Chat LLMs
+
+## Contributing:
+
+We are in the very early days of LlamaIndex.TS. If you’re interested in hacking on it with us check out our [contributing guide](/CONTRIBUTING.md)
+
+## Bugs? Questions?
+
+Please join our Discord! https://discord.com/invite/eN6D2HQ4aX

packages/core/package.json

@@ -1,18 +1,17 @@
 {
   "name": "llamaindex",
-  "private": true,
-  "version": "0.1.2",
+  "version": "0.0.48",
   "license": "MIT",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.12.4",
-    "@datastax/astra-db-ts": "^0.1.4",
-    "@mistralai/mistralai": "^0.0.10",
+    "@anthropic-ai/sdk": "^0.9.1",
+    "@datastax/astra-db-ts": "^0.1.2",
+    "@mistralai/mistralai": "^0.0.7",
     "@notionhq/client": "^2.2.14",
-    "@pinecone-database/pinecone": "^1.1.3",
+    "@pinecone-database/pinecone": "^1.1.2",
     "@qdrant/js-client-rest": "^1.7.0",
-    "@xenova/transformers": "^2.14.1",
-    "assemblyai": "^4.2.1",
-    "chromadb": "~1.7.3",
+    "@xenova/transformers": "^2.10.0",
+    "assemblyai": "^4.0.0",
+    "chromadb": "^1.7.3",
     "file-type": "^18.7.0",
     "js-tiktoken": "^1.0.8",
     "lodash": "^4.17.21",
@@ -20,28 +19,26 @@
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.3.0",
     "notion-md-crawler": "^0.0.2",
-    "openai": "^4.26.0",
+    "openai": "^4.20.1",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
     "pdfjs-dist": "4.0.269",
     "pg": "^8.11.3",
-    "pgvector": "^0.1.7",
+    "pgvector": "^0.1.5",
     "portkey-ai": "^0.1.16",
     "rake-modified": "^1.0.8",
-    "replicate": "^0.25.2",
-    "string-strip-html": "^13.4.5",
+    "replicate": "^0.21.1",
+    "string-strip-html": "^13.4.3",
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@types/edit-json-file": "^1.7.3",
     "@types/jest": "^29.5.11",
     "@types/lodash": "^4.14.202",
-    "@types/node": "^18.19.10",
+    "@types/node": "^18.19.6",
     "@types/papaparse": "^5.3.14",
-    "@types/pg": "^8.11.0",
-    "bunchee": "^4.4.2",
-    "edit-json-file": "^1.8.0",
+    "@types/pg": "^8.10.9",
+    "bunchee": "^4.4.1",
     "madge": "^6.1.0",
     "typescript": "^5.3.3"
   },
@@ -53,19 +50,154 @@
   "exports": {
     ".": {
       "types": "./dist/index.d.mts",
+      "edge-light": "./dist/index.edge-light.mjs",
       "import": "./dist/index.mjs",
-      "edge-light": "./dist/index.mjs",
       "require": "./dist/index.js"
     },
     "./env": {
       "types": "./dist/env.d.mts",
+      "edge-light": "./dist/env.edge-light.mjs",
       "import": "./dist/env.mjs",
-      "edge-light": "./dist/env.mjs",
       "require": "./dist/env.js"
+    },
+    "./storage/FileSystem": {
+      "types": "./dist/storage/FileSystem.d.mts",
+      "edge-light": "./dist/storage/FileSystem.edge-light.mjs",
+      "import": "./dist/storage/FileSystem.mjs",
+      "require": "./dist/storage/FileSystem.js"
+    },
+    "./ChatHistory": {
+      "types": "./dist/ChatHistory.d.mts",
+      "import": "./dist/ChatHistory.mjs",
+      "require": "./dist/ChatHistory.js"
+    },
+    "./constants": {
+      "types": "./dist/constants.d.mts",
+      "import": "./dist/constants.mjs",
+      "require": "./dist/constants.js"
+    },
+    "./GlobalsHelper": {
+      "types": "./dist/GlobalsHelper.d.mts",
+      "import": "./dist/GlobalsHelper.mjs",
+      "require": "./dist/GlobalsHelper.js"
+    },
+    "./Node": {
+      "types": "./dist/Node.d.mts",
+      "import": "./dist/Node.mjs",
+      "require": "./dist/Node.js"
+    },
+    "./OutputParser": {
+      "types": "./dist/OutputParser.d.mts",
+      "import": "./dist/OutputParser.mjs",
+      "require": "./dist/OutputParser.js"
+    },
+    "./Prompt": {
+      "types": "./dist/Prompt.d.mts",
+      "import": "./dist/Prompt.mjs",
+      "require": "./dist/Prompt.js"
+    },
+    "./PromptHelper": {
+      "types": "./dist/PromptHelper.d.mts",
+      "import": "./dist/PromptHelper.mjs",
+      "require": "./dist/PromptHelper.js"
+    },
+    "./QueryEngine": {
+      "types": "./dist/QueryEngine.d.mts",
+      "import": "./dist/QueryEngine.mjs",
+      "require": "./dist/QueryEngine.js"
+    },
+    "./QuestionGenerator": {
+      "types": "./dist/QuestionGenerator.d.mts",
+      "import": "./dist/QuestionGenerator.mjs",
+      "require": "./dist/QuestionGenerator.js"
+    },
+    "./Response": {
+      "types": "./dist/Response.d.mts",
+      "import": "./dist/Response.mjs",
+      "require": "./dist/Response.js"
+    },
+    "./Retriever": {
+      "types": "./dist/Retriever.d.mts",
+      "import": "./dist/Retriever.mjs",
+      "require": "./dist/Retriever.js"
+    },
+    "./ServiceContext": {
+      "types": "./dist/ServiceContext.d.mts",
+      "import": "./dist/ServiceContext.mjs",
+      "require": "./dist/ServiceContext.js"
+    },
+    "./TextSplitter": {
+      "types": "./dist/TextSplitter.d.mts",
+      "import": "./dist/TextSplitter.mjs",
+      "require": "./dist/TextSplitter.js"
+    },
+    "./Tool": {
+      "types": "./dist/Tool.d.mts",
+      "import": "./dist/Tool.mjs",
+      "require": "./dist/Tool.js"
+    },
+    "./readers/AssemblyAI": {
+      "types": "./dist/readers/AssemblyAI.d.mts",
+      "import": "./dist/readers/AssemblyAI.mjs",
+      "require": "./dist/readers/AssemblyAI.js"
+    },
+    "./readers/base": {
+      "types": "./dist/readers/base.d.mts",
+      "import": "./dist/readers/base.mjs",
+      "require": "./dist/readers/base.js"
+    },
+    "./readers/CSVReader": {
+      "types": "./dist/readers/CSVReader.d.mts",
+      "import": "./dist/readers/CSVReader.mjs",
+      "require": "./dist/readers/CSVReader.js"
+    },
+    "./readers/DocxReader": {
+      "types": "./dist/readers/DocxReader.d.mts",
+      "import": "./dist/readers/DocxReader.mjs",
+      "require": "./dist/readers/DocxReader.js"
+    },
+    "./readers/HTMLReader": {
+      "types": "./dist/readers/HTMLReader.d.mts",
+      "import": "./dist/readers/HTMLReader.mjs",
+      "require": "./dist/readers/HTMLReader.js"
+    },
+    "./readers/ImageReader": {
+      "types": "./dist/readers/ImageReader.d.mts",
+      "import": "./dist/readers/ImageReader.mjs",
+      "require": "./dist/readers/ImageReader.js"
+    },
+    "./readers/MarkdownReader": {
+      "types": "./dist/readers/MarkdownReader.d.mts",
+      "import": "./dist/readers/MarkdownReader.mjs",
+      "require": "./dist/readers/MarkdownReader.js"
+    },
+    "./readers/NotionReader": {
+      "types": "./dist/readers/NotionReader.d.mts",
+      "import": "./dist/readers/NotionReader.mjs",
+      "require": "./dist/readers/NotionReader.js"
+    },
+    "./readers/PDFReader": {
+      "types": "./dist/readers/PDFReader.d.mts",
+      "import": "./dist/readers/PDFReader.mjs",
+      "require": "./dist/readers/PDFReader.js"
+    },
+    "./readers/SimpleDirectoryReader": {
+      "types": "./dist/readers/SimpleDirectoryReader.d.mts",
+      "import": "./dist/readers/SimpleDirectoryReader.mjs",
+      "require": "./dist/readers/SimpleDirectoryReader.js"
+    },
+    "./readers/SimpleMongoReader": {
+      "types": "./dist/readers/SimpleMongoReader.d.mts",
+      "import": "./dist/readers/SimpleMongoReader.mjs",
+      "require": "./dist/readers/SimpleMongoReader.js"
     }
   },
   "files": [
-    "**"
+    "dist",
+    "examples",
+    "src",
+    "types",
+    "CHANGELOG.md"
   ],
   "repository": {
     "type": "git",
@@ -76,10 +208,6 @@
     "lint": "eslint .",
     "test": "jest",
     "build": "bunchee",
-    "postbuild": "pnpm run copy && pnpm run modify-package-json",
-    "copy": "cp -r package.json CHANGELOG.md ../../README.md ../../LICENSE examples src dist/",
-    "modify-package-json": "node ./scripts/modify-package-json.mjs",
-    "prepublish": "pnpm run modify-package-json && echo \"please cd ./dist and run pnpm publish\" && exit 1",
     "dev": "bunchee -w",
     "circular-check": "madge --circular ./src/*.ts"
   }

packages/core/scripts/modify-package-json.mjs

@@ -1,25 +0,0 @@
-#!/usr/bin/env node
-/**
- * This script is used to modify the package.json file in the dist folder
- * so that it can be published to npm.
- */
-import editJsonFile from "edit-json-file";
-import fs from "node:fs/promises";
-
-{
-  await fs.copyFile("./package.json", "./dist/package.json");
-  const file = editJsonFile("./dist/package.json");
-
-  file.unset("scripts");
-  file.unset("private");
-  await new Promise((resolve) => file.save(resolve));
-}
-{
-  const packageJson = await fs.readFile("./dist/package.json", "utf8");
-  const modifiedPackageJson = packageJson.replaceAll("./dist/", "./");
-  await fs.writeFile(
-    "./dist/package.json",
-    JSON.stringify(JSON.parse(modifiedPackageJson), null, 2),
-    "utf8",
-  );
-}

packages/core/src/constants.ts

@@ -6,4 +6,6 @@ export const DEFAULT_CHUNK_OVERLAP = 20;
 export const DEFAULT_CHUNK_OVERLAP_RATIO = 0.1;
 export const DEFAULT_SIMILARITY_TOP_K = 2;
 
+// NOTE: for text-embedding-ada-002
+export const DEFAULT_EMBEDDING_DIM = 1536;
 export const DEFAULT_PADDING = 5;

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -9,55 +9,28 @@ import {
 import { OpenAISession, getOpenAISession } from "../llm/openai";
 import { BaseEmbedding } from "./types";
 
-export const ALL_OPENAI_EMBEDDING_MODELS = {
-  "text-embedding-ada-002": {
-    dimensions: 1536,
-    maxTokens: 8191,
-  },
-  "text-embedding-3-small": {
-    dimensions: 1536,
-    dimensionOptions: [512, 1536],
-    maxTokens: 8191,
-  },
-  "text-embedding-3-large": {
-    dimensions: 3072,
-    dimensionOptions: [256, 1024, 3072],
-    maxTokens: 8191,
-  },
-};
+export enum OpenAIEmbeddingModelType {
+  TEXT_EMBED_ADA_002 = "text-embedding-ada-002",
+}
 
 export class OpenAIEmbedding extends BaseEmbedding {
-  /** embeddding model. defaults to "text-embedding-ada-002" */
-  model: string;
-  /** number of dimensions of the resulting vector, for models that support choosing fewer dimensions. undefined will default to model default */
-  dimensions: number | undefined;
+  model: OpenAIEmbeddingModelType | string;
 
   // OpenAI session params
-
-  /** api key */
   apiKey?: string = undefined;
-  /** maximum number of retries, default 10 */
   maxRetries: number;
-  /** timeout in ms, default 60 seconds  */
   timeout?: number;
-  /** other session options for OpenAI */
   additionalSessionOptions?: Omit<
     Partial<OpenAIClientOptions>,
     "apiKey" | "maxRetries" | "timeout"
   >;
 
-  /** session object */
   session: OpenAISession;
 
-  /**
-   * OpenAI Embedding
-   * @param init - initial parameters
-   */
   constructor(init?: Partial<OpenAIEmbedding> & { azure?: AzureOpenAIConfig }) {
     super();
 
-    this.model = init?.model ?? "text-embedding-ada-002";
-    this.dimensions = init?.dimensions; // if no dimensions provided, will be undefined/not sent to OpenAI
+    this.model = OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002;
 
     this.maxRetries = init?.maxRetries ?? 10;
     this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
@@ -103,7 +76,6 @@ export class OpenAIEmbedding extends BaseEmbedding {
   private async getOpenAIEmbedding(input: string) {
     const { data } = await this.session.openai.embeddings.create({
       model: this.model,
-      dimensions: this.dimensions, // only sent to OpenAI if set by user
       input,
     });
 

packages/core/src/embeddings/together.ts

@@ -1,26 +1,16 @@
 import { OpenAIEmbedding } from "./OpenAIEmbedding";
 
 export class TogetherEmbedding extends OpenAIEmbedding {
+  override model: string;
   constructor(init?: Partial<OpenAIEmbedding>) {
-    const {
-      apiKey = process.env.TOGETHER_API_KEY,
-      additionalSessionOptions = {},
-      model = "togethercomputer/m2-bert-80M-32k-retrieval",
-      ...rest
-    } = init ?? {};
-
-    if (!apiKey) {
-      throw new Error("Set Together Key in TOGETHER_API_KEY env variable"); // Tell user to set correct env variable, and not OPENAI_API_KEY
-    }
-
-    additionalSessionOptions.baseURL =
-      additionalSessionOptions.baseURL ?? "https://api.together.xyz/v1";
-
     super({
-      apiKey,
-      additionalSessionOptions,
-      model,
-      ...rest,
+      apiKey: process.env.TOGETHER_API_KEY,
+      ...init,
+      additionalSessionOptions: {
+        ...init?.additionalSessionOptions,
+        baseURL: "https://api.together.xyz/v1",
+      },
     });
+    this.model = init?.model ?? "togethercomputer/m2-bert-80M-32k-retrieval";
   }
 }

packages/core/src/extractors/index.ts

@@ -4,4 +4,3 @@ export {
   SummaryExtractor,
   TitleExtractor,
 } from "./MetadataExtractors";
-export { BaseExtractor } from "./types";

packages/core/src/extractors/types.ts

@@ -46,10 +46,7 @@ export abstract class BaseExtractor implements TransformComponent {
     let curMetadataList = await this.extract(newNodes);
 
     for (let idx in newNodes) {
-      newNodes[idx].metadata = {
-        ...newNodes[idx].metadata,
-        ...curMetadataList[idx],
-      };
+      newNodes[idx].metadata = curMetadataList[idx];
     }
 
     for (let idx in newNodes) {

packages/core/src/llm/LLM.ts

@@ -41,21 +41,14 @@ import {
 export const GPT4_MODELS = {
   "gpt-4": { contextWindow: 8192 },
   "gpt-4-32k": { contextWindow: 32768 },
-  "gpt-4-32k-0613": { contextWindow: 32768 },
-  "gpt-4-turbo-preview": { contextWindow: 128000 },
   "gpt-4-1106-preview": { contextWindow: 128000 },
-  "gpt-4-0125-preview": { contextWindow: 128000 },
-  "gpt-4-vision-preview": { contextWindow: 128000 },
+  "gpt-4-vision-preview": { contextWindow: 8192 },
 };
 
-// NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
 export const GPT35_MODELS = {
   "gpt-3.5-turbo": { contextWindow: 4096 },
-  "gpt-3.5-turbo-0613": { contextWindow: 4096 },
   "gpt-3.5-turbo-16k": { contextWindow: 16384 },
-  "gpt-3.5-turbo-16k-0613": { contextWindow: 16384 },
   "gpt-3.5-turbo-1106": { contextWindow: 16384 },
-  "gpt-3.5-turbo-0125": { contextWindow: 16384 },
 };
 
 /**

packages/core/src/llm/azure.ts

@@ -17,14 +17,6 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
   },
   "gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
   "gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
-  "gpt-4-vision-preview": {
-    contextWindow: 128000,
-    openAIModel: "gpt-4-vision-preview",
-  },
-  "gpt-4-1106-preview": {
-    contextWindow: 128000,
-    openAIModel: "gpt-4-1106-preview",
-  },
 };
 
 const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
@@ -33,29 +25,13 @@ const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
     openAIModel: "text-embedding-ada-002",
     maxTokens: 8191,
   },
-  "text-embedding-3-small": {
-    dimensions: 1536,
-    dimensionOptions: [512, 1536],
-    openAIModel: "text-embedding-3-small",
-    maxTokens: 8191,
-  },
-  "text-embedding-3-large": {
-    dimensions: 3072,
-    dimensionOptions: [256, 1024, 3072],
-    openAIModel: "text-embedding-3-large",
-    maxTokens: 8191,
-  },
 };
 
 const ALL_AZURE_API_VERSIONS = [
   "2022-12-01",
   "2023-05-15",
-  "2023-03-15-preview", // retiring 2024-04-02
-  "2023-06-01-preview", // retiring 2024-04-02
-  "2023-07-01-preview", // retiring 2024-04-02
-  "2023-08-01-preview", // retiring 2024-04-02
-  "2023-09-01-preview",
-  "2023-12-01-preview",
+  "2023-06-01-preview",
+  "2023-07-01-preview",
 ];
 
 const DEFAULT_API_VERSION = "2023-05-15";

packages/core/src/llm/together.ts

@@ -2,25 +2,13 @@ import { OpenAI } from "./LLM";
 
 export class TogetherLLM extends OpenAI {
   constructor(init?: Partial<OpenAI>) {
-    const {
-      apiKey = process.env.TOGETHER_API_KEY,
-      additionalSessionOptions = {},
-      model = "togethercomputer/llama-2-7b-chat",
-      ...rest
-    } = init ?? {};
-
-    if (!apiKey) {
-      throw new Error("Set Together Key in TOGETHER_API_KEY env variable"); // Tell user to set correct env variable, and not OPENAI_API_KEY
-    }
-
-    additionalSessionOptions.baseURL =
-      additionalSessionOptions.baseURL ?? "https://api.together.xyz/v1";
-
     super({
-      apiKey,
-      additionalSessionOptions,
-      model,
-      ...rest,
+      ...init,
+      apiKey: process.env.TOGETHER_API_KEY,
+      additionalSessionOptions: {
+        ...init?.additionalSessionOptions,
+        baseURL: "https://api.together.xyz/v1",
+      },
     });
   }
 }

packages/core/src/storage/vectorStore/PGVectorStore.ts

@@ -20,7 +20,6 @@ export class PGVectorStore implements VectorStore {
   private schemaName: string = PGVECTOR_SCHEMA;
   private tableName: string = PGVECTOR_TABLE;
   private connectionString: string | undefined = undefined;
-  private dimensions: number = 1536;
 
   private db?: pg.Client;
 
@@ -39,18 +38,15 @@ export class PGVectorStore implements VectorStore {
    * @param {string} config.schemaName - The name of the schema (optional). Defaults to PGVECTOR_SCHEMA.
    * @param {string} config.tableName - The name of the table (optional). Defaults to PGVECTOR_TABLE.
    * @param {string} config.connectionString - The connection string (optional).
-   * @param {number} config.dimensions - The dimensions of the embedding model.
    */
   constructor(config?: {
     schemaName?: string;
     tableName?: string;
     connectionString?: string;
-    dimensions?: number;
   }) {
     this.schemaName = config?.schemaName ?? PGVECTOR_SCHEMA;
     this.tableName = config?.tableName ?? PGVECTOR_TABLE;
     this.connectionString = config?.connectionString;
-    this.dimensions = config?.dimensions ?? 1536;
   }
 
   /**
@@ -112,7 +108,7 @@ export class PGVectorStore implements VectorStore {
       collection VARCHAR,
       document TEXT,
       metadata JSONB DEFAULT '{}',
-      embeddings VECTOR(${this.dimensions})
+      embeddings VECTOR(1536)
     )`;
     await db.query(tbl);