Forward backend options during streaming speech generation (#1389) (#1390)

Switch SQLite ANN connections when load() is given a new path

.coveragerc

@@ -2,6 +2,7 @@
 source = src/python
 concurrency = multiprocessing,thread
 disable_warnings = no-data-collected
+omit = **/__main__.py
 
 [combine]
 disable_warnings = no-data-collected

.github/workflows/build.yml

@@ -10,44 +10,67 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
 
-    timeout-minutes: 60
+    timeout-minutes: 75
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install Python
+        uses: actions/setup-python@v6
         with:
-          python-version: 3.7
-      - uses: actions/setup-java@v1
+          python-version: "3.11"
+
+      - name: Install Java
+        uses: actions/setup-java@v5
         with:
-          java-version: 1.8
+          distribution: "zulu"
+          java-version: 21
 
       - name: Install dependencies - Linux
-        run: sudo apt-get install libsndfile1
+        run: |
+          sudo apt-get update
+          sudo apt-get install libportaudio2 libsndfile1 libegl1 libgles2
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
         if: matrix.os == 'ubuntu-latest'
 
-      - name: Install dependencies - MacOS
+      - name: Install dependencies - macOS
         run: |
-          brew install libomp
-          pip install annoy==1.16.3
+          echo "PYTORCH_MPS_DISABLE=1" >> $GITHUB_ENV
+          echo "LLAMA_NO_METAL=1" >> $GITHUB_ENV
+          echo "TIKA_STARTUP_SLEEP=30" >> $GITHUB_ENV
+          echo "TIKA_STARTUP_MAX_RETRY=10" >> $GITHUB_ENV
+          brew install portaudio
+          pip install torch==2.12.1 torchvision==0.27.1
         if: matrix.os == 'macos-latest'
 
       - name: Install dependencies - Windows
-        run: choco install wget
+        run: |
+          "PYTHONIOENCODING=utf-8" >> $env:GITHUB_ENV
+          choco install wget
         if: matrix.os == 'windows-latest'
 
       - name: Build
         run: |
-          pip install -U pip
-          pip install -U wheel coverage coveralls
-          pip install .[all]
-          python -c "import nltk; nltk.download('punkt')"
+          pip install -U wheel
+          pip install .[all,dev]
+          pip cache purge
+          python -c "import nltk; nltk.download(['punkt', 'punkt_tab', 'averaged_perceptron_tagger_eng'])"
           python --version
           make data coverage
+        env:
+          HF_HUB_ETAG_TIMEOUT: 100
+          HF_HUB_DOWNLOAD_TIMEOUT: 100
+          HF_XET_CHUNK_CACHE_SIZE_BYTES: 0
+          DOCLING_DEVICE: "cpu"
+          DOCLING_INFERENCE_COMPILE_TORCH_MODELS: false
 
-      - uses: pre-commit/action@v2.0.0
+      - uses: pre-commit/action@v3.0.1
         if: matrix.os == 'ubuntu-latest'
 
       - name: Test Coverage
-        run: coveralls --service=github
+        run: |
+          coverage report --fail-under=100
+          coveralls --service=github
         if: matrix.os == 'ubuntu-latest'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yml

@@ -7,12 +7,19 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install Python
+        uses: actions/setup-python@v6
         with:
-          python-version: 3.7
-      - run: |
+          python-version: "3.11"
+
+      - name: Install txtai
+        run: |
           pip install -U pip wheel
-          pip install mkdocs-material mkdocstrings .[all]
-      - run: mkdocs gh-deploy --force
+          pip install .[all,dev]
+
+      - name: Deploy documentation
+        run: mkdocs gh-deploy --force
  

.github/workflows/minimal.yml

@@ -0,0 +1,22 @@
+# GitHub Actions minimal build
+name: minimal
+
+on: ["push", "pull_request"]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Minimal install
+        run: |
+          pip install -U pip wheel
+          MINIMAL=1 pip install .
+          python -c "import txtai"

.gitignore

@@ -7,3 +7,4 @@ __pycache__/
 .coverage
 .coverage.*
 *.pyc
+.vscode/

.pre-commit-config.yaml

@@ -1,13 +1,14 @@
 repos:
   - repo: https://github.com/pycqa/pylint
-    rev: v2.12.1
+    rev: v3.3.1
     hooks:
     - id: pylint
       args:
       - -d import-error
       - -d duplicate-code
+      - -d too-many-positional-arguments
   - repo: https://github.com/ambv/black
-    rev: 21.11b1
+    rev: 24.10.0
     hooks:
     - id: black
       language_version: python3

.pylintrc

@@ -12,9 +12,8 @@ max-locals=40
 max-returns=10
 max-attributes=20
 min-public-methods=0
+max-public-methods=80
 
 [FORMAT]
 max-line-length=150
-
-[MESSAGES CONTROL]
-disable=R0201
+max-module-lines=1500

CITATION.cff

@@ -0,0 +1,11 @@
+cff-version: 1.2.0
+date-released: 2020-08-11
+message: "If you use this software, please cite it as below."
+title: "txtai: the all-in-one AI framework"
+abstract: "txtai is an all-in-one open-source AI framework for semantic search, LLM orchestration and language model workflows"
+url: "https://github.com/neuml/txtai"
+authors:
+- family-names: "Mezzetti"
+  given-names: "David"
+  affiliation: NeuML
+license: Apache-2.0

Makefile

@@ -14,10 +14,16 @@ export TOKENIZERS_PARALLELISM := false
 # Default python executable if not provided
 PYTHON ?= python
 
+# Check for wget
+WGET := $(shell wget --version 2> /dev/null)
+ifndef WGET
+    $(error "Required binary `wget` not found, please install wget OS package")
+endif
+
 # Download test data
 data:
 	mkdir -p /tmp/txtai
-	wget -N https://github.com/neuml/txtai/releases/download/v3.5.0/tests.tar.gz -P /tmp
+	wget -N https://github.com/neuml/txtai/releases/download/v6.2.0/tests.tar.gz -P /tmp
 	tar -xvzf /tmp/tests.tar.gz -C /tmp
 
 # Unit tests
@@ -26,5 +32,27 @@ test:
 
 # Run tests while calculating code coverage
 coverage:
-	coverage run -m unittest discover -v -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testagent -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testann -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testapi -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testapp -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testarchive -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testcloud -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testconsole -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testdatabase -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testembeddings -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testgraph -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testlibrary -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testmodels -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testoptional -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testpipeline.testaudio -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testpipeline.testdata -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testpipeline.testimage -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testpipeline.testllm -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testpipeline.testtext -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testpipeline.testtrain -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testscoring -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testserialize -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testvectors -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testworkflow -s ${TEST_DIR}
 	coverage combine

README.md

@@ -2,85 +2,182 @@
     <img src="https://raw.githubusercontent.com/neuml/txtai/master/logo.png"/>
 </p>
 
-<h3 align="center">
-    <p>Build AI-powered semantic search applications</p>
-</h3>
+<p align="center">
+    <b>All-in-one AI framework</b>
+</p>
 
 <p align="center">
     <a href="https://github.com/neuml/txtai/releases">
         <img src="https://img.shields.io/github/release/neuml/txtai.svg?style=flat&color=success" alt="Version"/>
     </a>
-    <a href="https://github.com/neuml/txtai/releases">
-        <img src="https://img.shields.io/github/release-date/neuml/txtai.svg?style=flat&color=blue" alt="GitHub Release Date"/>
+    <a href="https://github.com/neuml/txtai">
+        <img src="https://img.shields.io/github/last-commit/neuml/txtai.svg?style=flat&color=blue" alt="GitHub last commit"/>
     </a>
     <a href="https://github.com/neuml/txtai/issues">
         <img src="https://img.shields.io/github/issues/neuml/txtai.svg?style=flat&color=success" alt="GitHub issues"/>
     </a>
-    <a href="https://github.com/neuml/txtai">
-        <img src="https://img.shields.io/github/last-commit/neuml/txtai.svg?style=flat&color=blue" alt="GitHub last commit"/>
+    <a href="https://join.slack.com/t/txtai/shared_invite/zt-37c1zfijp-Y57wMty6YOx_hyIHEQvQJA">
+        <img src="https://img.shields.io/badge/slack-join-blue?style=flat&logo=slack&logocolor=white" alt="Join Slack"/>
     </a>
     <a href="https://github.com/neuml/txtai/actions?query=workflow%3Abuild">
         <img src="https://github.com/neuml/txtai/workflows/build/badge.svg" alt="Build Status"/>
     </a>
     <a href="https://coveralls.io/github/neuml/txtai?branch=master">
-        <img src="https://img.shields.io/coveralls/github/neuml/txtai" alt="Coverage Status">
+        <img src="https://img.shields.io/coverallsCoverage/github/neuml/txtai" alt="Coverage Status">
     </a>
 </p>
 
--------------------------------------------------------------------------------------------------------------------------------------------------------
+txtai is an all-in-one AI framework for semantic search, LLM orchestration and language model workflows.
 
-txtai executes machine-learning workflows to transform data and build AI-powered semantic search applications.
+![architecture](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/architecture.png#gh-light-mode-only)
+![architecture](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/architecture-dark.png#gh-dark-mode-only)
 
-![demo](https://raw.githubusercontent.com/neuml/txtai/master/demo.gif)
+The key component of txtai is an embeddings database, which is a union of vector indexes (sparse and dense), graph networks and relational databases.
 
-Traditional search systems use keywords to find data. Semantic search applications have an understanding of natural language and identify results that have the same meaning, not necessarily the same keywords.
+This foundation enables vector search and/or serves as a powerful knowledge source for large language model (LLM) applications.
 
-Backed by state-of-the-art machine learning models, data is transformed into vector representations for search (also known as embeddings). Innovation is happening at a rapid pace, models can understand concepts in documents, audio, images and more.
+Build autonomous agents, retrieval augmented generation (RAG) processes, multi-model workflows and more.
 
 Summary of txtai features:
 
-- 🔎 Large-scale similarity search with multiple index backends ([Faiss](https://github.com/facebookresearch/faiss), [Annoy](https://github.com/spotify/annoy), [Hnswlib](https://github.com/nmslib/hnswlib))
-- 📄 Create embeddings for text snippets, documents, audio, images and video. Supports transformers and word vectors.
-- 💡 Machine-learning pipelines to run extractive question-answering, zero-shot labeling, transcription, translation, summarization and text extraction
-- ↪️️ Workflows that join pipelines together to aggregate business logic. txtai processes can be microservices or full-fledged indexing workflows.
-- 🔗 API bindings for [JavaScript](https://github.com/neuml/txtai.js), [Java](https://github.com/neuml/txtai.java), [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go)
-- ☁️ Cloud-native architecture that scales out with container orchestration systems (e.g. Kubernetes)
+- 🔎 Vector search with SQL, object storage, topic modeling, graph analysis and multimodal indexing
+- 📄 Create embeddings for text, documents, audio, images and video
+- 💡 Pipelines powered by language models that run LLM prompts, question-answering, labeling, transcription, translation, summarization and more
+- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be simple microservices or multi-model workflows.
+- 🤖 Agents that intelligently connect embeddings, pipelines, workflows and other agents together to autonomously solve complex problems
+- ⚙️ Web and Model Context Protocol (MCP) APIs. Bindings available for [JavaScript](https://github.com/neuml/txtai.js), [Java](https://github.com/neuml/txtai.java), [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go).
+- 🔋 Batteries included with defaults to get up and running fast
+- ☁️ Run local or scale out with container orchestration
 
-Applications range from similarity search to complex NLP-driven data extractions to generate structured databases. The following applications are powered by txtai.
+txtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
 
-![apps](https://raw.githubusercontent.com/neuml/txtai/master/apps.jpg)
-
-| Application  | Description  |
-|:----------|:-------------|
-| [paperai](https://github.com/neuml/paperai) | AI-powered literature discovery and review engine for medical/scientific papers |
-| [tldrstory](https://github.com/neuml/tldrstory) | AI-powered understanding of headlines and story text |
-| [neuspo](https://neuspo.com) | Fact-driven, real-time sports event and news site |
-| [codequestion](https://github.com/neuml/codequestion) | Ask coding questions directly from the terminal |
-
-txtai is built with Python 3.7+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi)
+> [!NOTE]
+>
+> [NeuML](https://neuml.com) is the company behind txtai and we provide AI consulting services around our stack. [Schedule a meeting](https://cal.com/neuml/intro) or [send a message](mailto:[email protected]) to learn more.
+>
+> We're also building an easy and secure way to run hosted txtai applications with [txtai.cloud](https://txtai.cloud).
 
 ## Why txtai?
 
 ![why](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/why.png#gh-light-mode-only)
 ![why](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/why-dark.png#gh-dark-mode-only)
 
-In addition to traditional search systems, a growing number of semantic search solutions are available, so why txtai?
+New vector databases, LLM frameworks and everything in between are sprouting up daily. Why build with txtai?
 
-- `pip install txtai` is all you need
+- Up and running in minutes with [pip](https://neuml.github.io/txtai/install/) or [Docker](https://neuml.github.io/txtai/cloud/)
 ```python
 # Get started in a couple lines
-from txtai.embeddings import Embeddings
+import txtai
 
-embeddings = Embeddings({"path": "sentence-transformers/all-MiniLM-L6-v2"})
-embeddings.index([(0, "Correct", None), (1, "Not what we hoped", None)])
+embeddings = txtai.Embeddings()
+embeddings.index(["Correct", "Not what we hoped"])
 embeddings.search("positive", 1)
-#[(0, 0.2986203730106354)]
+#[(0, 0.29862046241760254)]
 ```
-- Works well with both small and big data - scale up as needed
-- Rich data processing framework (pipelines and workflows) to pre and post process data
-- Work in your programming language of choice via the API
-- Modular with low footprint - install additional dependencies when you need them
-- [Learn by example](#examples) - notebooks cover all available functionality
+- Built-in API makes it easy to develop applications using your programming language of choice
+```yaml
+# app.yml
+embeddings:
+    path: sentence-transformers/all-MiniLM-L6-v2
+```
+```bash
+CONFIG=app.yml uvicorn "txtai.api:app"
+curl -X GET "http://localhost:8000/search?query=positive"
+```
+- Run local - no need to ship data off to disparate remote services
+- Work with micromodels all the way up to large language models (LLMs)
+- Low footprint - install additional dependencies and scale up when needed
+- [Learn by example](https://neuml.github.io/txtai/examples) - notebooks cover all available functionality
+
+## Use Cases
+
+The following sections introduce common txtai use cases. A comprehensive set of over 70 [example notebooks and applications](https://neuml.github.io/txtai/examples) are also available.
+
+### Semantic Search
+
+Build semantic/similarity/vector/neural search applications.
+
+![demo](https://raw.githubusercontent.com/neuml/txtai/master/demo.gif)
+
+Traditional search systems use keywords to find data. Semantic search has an understanding of natural language and identifies results that have the same meaning, not necessarily the same keywords.
+
+![search](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/search.png#gh-light-mode-only)
+![search](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/search-dark.png#gh-dark-mode-only)
+
+Get started with the following examples.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Introducing txtai](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=SIezMnVdmMs) | Overview of the functionality provided by txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |
+| [Similarity search with images](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) | Embed images and text into the same space for search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |
+| [Build a QA database](https://github.com/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) | Question matching with semantic search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) |
+| [Semantic Graphs](https://github.com/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) | Explore topics, data connectivity and run network analysis| [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) |
+
+### LLM Orchestration
+
+Autonomous agents, retrieval augmented generation (RAG), chat with your data, pipelines and workflows that interface with large language models (LLMs).
+
+![llm](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/llm.png)
+
+See below to learn more.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
+| [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
+| [Build knowledge graphs with LLMs](https://github.com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) | Build knowledge graphs with LLM-driven entity extraction | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |
+| [Parsing the stars with txtai](https://github.com/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) | Explore an astronomical knowledge graph of known stars, planets, galaxies | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) |
+
+#### Agents
+
+Agents connect embeddings, pipelines, workflows and other agents together to autonomously solve complex problems.
+
+![agent](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/agent.png)
+
+txtai agents are built on top of the [smolagents](https://github.com/huggingface/smolagents) framework. This supports all LLMs txtai supports (Hugging Face, llama.cpp, OpenAI / Claude / AWS Bedrock via LiteLLM). Agent prompting with [`agents.md`](https://github.com/agentsmd/agents.md) and [`skill.md`](https://agentskills.io/specification) are also supported.
+
+Check out this [Agent Quickstart Example](https://github.com/neuml/txtai/blob/master/examples/agent_quickstart.py). Additional examples are listed below.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
+| [Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) | Exploring how to improve social media engagement with AI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |
+
+#### Retrieval augmented generation
+
+Retrieval augmented generation (RAG) reduces the risk of LLM hallucinations by constraining the output with a knowledge base as context. RAG is commonly used to "chat with your data".
+
+![rag](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/rag.png#gh-light-mode-only)
+![rag](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/rag-dark.png#gh-dark-mode-only)
+
+Check out this [RAG Quickstart Example](https://github.com/neuml/txtai/blob/master/examples/rag_quickstart.py). Additional examples are listed below.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=t_OeAc8NVfQ) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [RAG is more than Vector Search](https://github.com/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) | Context retrieval via Web, SQL and other sources | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) |
+| [GraphRAG with Wikipedia and GPT OSS](https://github.com/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) | Deep graph search powered RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) |
+| [Speech to Speech RAG](https://github.com/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) [▶️](https://www.youtube.com/watch?v=tH8QWwkVMKA) | Full cycle speech to speech workflow with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) |
+
+### Language Model Workflows
+
+Language model workflows, also known as semantic workflows, connect language models together to build intelligent applications.
+
+![flows](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/flows.png#gh-light-mode-only)
+![flows](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/flows-dark.png#gh-dark-mode-only)
+
+While LLMs are powerful, there are plenty of smaller, more specialized models that work better and faster for specific tasks. This includes models for extractive question-answering, automatic summarization, text-to-speech, transcription and translation.
+
+Check out this [Workflow Quickstart Example](https://github.com/neuml/txtai/blob/master/examples/workflow_quickstart.py). Additional examples are listed below.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) [▶️](https://www.youtube.com/watch?v=UBMPDCn1gEU) | Simple yet powerful constructs to efficiently process data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
+| [Building abstractive text summaries](https://github.com/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) | Run abstractive text summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) |
+| [Transcribe audio to text](https://github.com/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) | Convert audio files to text | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) |
+| [Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) | Streamline machine translation and language detection | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |
 
 ## Installation
 
@@ -89,105 +186,73 @@ embeddings.search("positive", 1)
 
 The easiest way to install is via pip and PyPI
 
-    pip install txtai
+```
+pip install txtai
+```
 
-Python 3.7+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
+Python 3.11+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
 
-See the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering
-[installing from source](https://neuml.github.io/txtai/install/#install-from-source),
-[environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites) and
-[optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies).
+See the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies), [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites), [installing from source](https://neuml.github.io/txtai/install/#install-from-source), [conda support](https://neuml.github.io/txtai/install/#conda), [lightweight minimal installation](https://neuml.github.io/txtai/install/#minimal-install) and how to [run with containers](https://neuml.github.io/txtai/cloud).
 
-## Examples
+## Model guide
 
-![examples](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/examples.png#gh-light-mode-only)
-![examples](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/examples-dark.png#gh-dark-mode-only)
+![models](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/models.png)
 
-The examples directory has a series of notebooks and applications giving an overview of txtai. See the sections below.
+See the table below for the current recommended models. These models all allow commercial use and offer a blend of speed and performance.
 
-### Semantic Search
+| Component                                                                     | Model(s)                                                                 |
+| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------ |
+| [Embeddings](https://neuml.github.io/txtai/embeddings)                        | [all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) | 
+| [Image Captions](https://neuml.github.io/txtai/pipeline/image/caption)        | [BLIP](https://hf.co/Salesforce/blip-image-captioning-base)              |
+| [Labels - Zero Shot](https://neuml.github.io/txtai/pipeline/text/labels)      | [DeBERTa v3 Zeroshot](https://hf.co/MoritzLaurer/deberta-v3-base-zeroshot-v2.0-c) |
+| [Labels - Fixed](https://neuml.github.io/txtai/pipeline/text/labels)          | Fine-tune with [training pipeline](https://neuml.github.io/txtai/pipeline/train/trainer)          |
+| [Large Language Model (LLM)](https://neuml.github.io/txtai/pipeline/text/llm) | [Gemma 4 31B](https://hf.co/google/gemma-4-31B) |
+| [Summarization](https://neuml.github.io/txtai/pipeline/text/summary)          | [DistilBART](https://hf.co/sshleifer/distilbart-cnn-12-6)                |
+| [Text-to-Speech](https://neuml.github.io/txtai/pipeline/audio/texttospeech)   | [ESPnet JETS](https://hf.co/NeuML/ljspeech-jets-onnx)                    |
+| [Transcription](https://neuml.github.io/txtai/pipeline/audio/transcription)   | [Whisper](https://hf.co/openai/whisper-base)                             | 
+| [Translation](https://neuml.github.io/txtai/pipeline/text/translation)        | [OPUS Model Series](https://hf.co/Helsinki-NLP)                          |
 
-Build semantic/similarity/vector/neural search applications.
+Models can be loaded as either a path from the Hugging Face Hub or a local directory. Model paths are optional, defaults are loaded when not specified. For tasks with no recommended model, txtai uses the default models as shown in the Hugging Face Tasks guide.
 
-| Notebook  | Description  |       |
-|:----------|:-------------|------:|
-| [Introducing txtai](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) | Overview of the functionality provided by txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |
-| [Build an Embeddings index with Hugging Face Datasets](https://github.com/neuml/txtai/blob/master/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb) | Index and search Hugging Face Datasets | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb) |
-| [Build an Embeddings index from a data source](https://github.com/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb)  | Index and search a data source with word embeddings | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb) |
-| [Add semantic search to Elasticsearch](https://github.com/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb)  | Add semantic search to existing search systems | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb) |
-| [Similarity search with images](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) | Embed images and text into the same space for search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |
-| [Distributed embeddings cluster](https://github.com/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) | Distribute an embeddings index across multiple data nodes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) |
-| [What's new in txtai 4.0](https://github.com/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) | Content storage, SQL, object storage, reindex and compressed indexes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) |
+See the following links to learn more.
 
-### Pipelines
+- [Hugging Face Tasks](https://hf.co/tasks)
+- [Hugging Face Model Hub](https://hf.co/models)
+- [Embeddings Leaderboard](https://hf.co/spaces/mteb/leaderboard)
+- [LLM Leaderboard](https://hf.co/spaces/lmarena-ai/arena-leaderboard)
 
-Transform data with NLP-backed pipelines.
+## Powered by txtai
 
-| Notebook  | Description  |       |
-|:----------|:-------------|------:|
-| [Extractive QA with txtai](https://github.com/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) | Introduction to extractive question-answering with txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) |
-| [Extractive QA with Elasticsearch](https://github.com/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) | Run extractive question-answering queries with Elasticsearch | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) |
-| [Extractive QA to build structured data](https://github.com/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) | Build structured datasets using extractive question-answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) |
-| [Apply labels with zero shot classification](https://github.com/neuml/txtai/blob/master/examples/07_Apply_labels_with_zero_shot_classification.ipynb) | Use zero shot learning for labeling, classification and topic modeling | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/07_Apply_labels_with_zero_shot_classification.ipynb) |
-| [Building abstractive text summaries](https://github.com/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) | Run abstractive text summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) |
-| [Extract text from documents](https://github.com/neuml/txtai/blob/master/examples/10_Extract_text_from_documents.ipynb) | Extract text from PDF, Office, HTML and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/10_Extract_text_from_documents.ipynb) |
-| [Transcribe audio to text](https://github.com/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) | Convert audio files to text | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) |
-| [Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) | Streamline machine translation and language detection | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |
-| [Generate image captions and detect objects](https://github.com/neuml/txtai/blob/master/examples/25_Generate_image_captions_and_detect_objects.ipynb) | Captions and object detection for images | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/25_Generate_image_captions_and_detect_objects.ipynb) |
-| [API Gallery](https://github.com/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) | Using txtai in JavaScript, Java, Rust and Go | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) |
+The following applications are powered by txtai.
 
-### Workflows
+![apps](https://raw.githubusercontent.com/neuml/txtai/master/apps.jpg)
 
-Efficiently process data at scale.
+| Application  | Description  |
+|:------------ |:-------------|
+| [rag](https://github.com/neuml/rag) | Retrieval Augmented Generation (RAG) application |
+| [ncoder](https://github.com/neuml/ncoder) | Open-Source AI coding agent |
+| [paperai](https://github.com/neuml/paperai) | AI for medical and scientific papers |
+| [annotateai](https://github.com/neuml/annotateai) | Automatically annotate papers with LLMs |
 
-| Notebook  | Description  |       |
-|:----------|:-------------|------:|
-| [Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) | Simple yet powerful constructs to efficiently process data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
-| [Transform tabular data with composable workflows](https://github.com/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) | Transform, index and search tabular data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) |
-| [Tensor workflows](https://github.com/neuml/txtai/blob/master/examples/23_Tensor_workflows.ipynb) | Performant processing of large tensor arrays | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/23_Tensor_workflows.ipynb) |
-| [Entity extraction workflows](https://github.com/neuml/txtai/blob/master/examples/26_Entity_extraction_workflows.ipynb) | Identify entity/label combinations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/26_Entity_extraction_workflows.ipynb) |
-| [Workflow Scheduling](https://github.com/neuml/txtai/blob/master/examples/27_Workflow_scheduling.ipynb) | Schedule workflows with cron expressions | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/27_Workflow_scheduling.ipynb) |
-| [Push notifications with workflows](https://github.com/neuml/txtai/blob/master/examples/28_Push_notifications_with_workflows.ipynb) | Generate and push notifications with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/28_Push_notifications_with_workflows.ipynb) |
+In addition to this list, there are also many other [open-source projects](https://github.com/neuml/txtai/network/dependents), [published research](https://scholar.google.com/scholar?q=txtai&hl=en&as_ylo=2022) and closed proprietary/commercial projects that have built on txtai in production.
 
-### Model Training
+## Further Reading
 
-Train NLP models.
+![further](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/further.png#gh-light-mode-only)
+![further](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/further-ghdark.png#gh-dark-mode-only)
 
-| Notebook  | Description  |       |
-|:----------|:-------------|------:|
-| [Train a text labeler](https://github.com/neuml/txtai/blob/master/examples/16_Train_a_text_labeler.ipynb) | Build text sequence classification models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/16_Train_a_text_labeler.ipynb) |
-| [Train without labels](https://github.com/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) | Use zero-shot classifiers to train new models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) |
-| [Train a QA model](https://github.com/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) | Build and fine-tune question-answering models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) |
-| [Export and run models with ONNX](https://github.com/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) | Export models with ONNX, run natively in JavaScript, Java and Rust | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) |
-| [Export and run other machine learning models](https://github.com/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) | Export and run models from scikit-learn, PyTorch and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) |
+- [Introducing txtai, the all-in-one AI framework](https://medium.com/neuml/introducing-txtai-the-all-in-one-ai-framework-0660ecfc39d7)
+- [txtai: An All-in-One AI Framework for Semantic Search and LLM Workflows](https://github.com/neuml/papers/blob/master/txtai/txtai.pdf)
+- [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
+- [What's new in txtai 9.0](https://medium.com/neuml/whats-new-in-txtai-9-0-d522bb150afa) | [8.0](https://medium.com/neuml/whats-new-in-txtai-8-0-2d7d0ab4506b) | [7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
+- [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9) | [rag](https://medium.com/neuml/getting-started-with-rag-9a0cca75f748)
+- [Running txtai at scale](https://medium.com/neuml/running-at-scale-with-txtai-71196cdd99f9)
+- [Vector search & RAG Landscape: A review with txtai](https://medium.com/neuml/vector-search-rag-landscape-a-review-with-txtai-a7f37ad0e187)
 
-### Applications
+## Documentation
 
-Series of example applications with txtai. Links to hosted versions on [Hugging Face Spaces](https://hf.co/spaces) also provided.
+[Full documentation on txtai](https://neuml.github.io/txtai) including configuration settings for embeddings, pipelines, workflows, API and a FAQ with common questions/issues is available.
 
-| Application  | Description  |       |
-|:-------------|:-------------|------:|
-| [Basic similarity search](https://github.com/neuml/txtai/blob/master/examples/similarity.py) | Basic similarity search example. Data from the original txtai demo. |[🤗](https://hf.co/spaces/NeuML/similarity)|
-| [Book search](https://github.com/neuml/txtai/blob/master/examples/books.py) | Book similarity search application. Index book descriptions and query using natural language statements. |*Local run only*|
-| [Image search](https://github.com/neuml/txtai/blob/master/examples/images.py) | Image similarity search application. Index a directory of images and run searches to identify images similar to the input query. |[🤗](https://hf.co/spaces/NeuML/imagesearch)|
-| [Summarize an article](https://github.com/neuml/txtai/blob/master/examples/article.py) | Summarize an article. Workflow that extracts text from a webpage and builds a summary. |[🤗](https://hf.co/spaces/NeuML/articlesummary)|
-| [Wiki search](https://github.com/neuml/txtai/blob/master/examples/wiki.py) | Wikipedia search application. Queries Wikipedia API and summarizes the top result. |[🤗](https://hf.co/spaces/NeuML/wikisummary)|
-| [Workflow builder](https://github.com/neuml/txtai/blob/master/examples/workflows.py) | Build and execute txtai workflows. Connect summarization, text extraction, transcription, translation and similarity search pipelines together to run unified workflows. |[🤗](https://hf.co/spaces/NeuML/txtai)|
-
-### Documentation
-
-[Full documentation on txtai](https://neuml.github.io/txtai) including configuration settings for pipelines, workflows, indexing and the API.
-
-### Further Reading
-
-![further](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/further.png)
-
-- [Introducing txtai, AI-powered semantic search built on Transformers](https://towardsdatascience.com/introducing-txtai-an-ai-powered-search-engine-built-on-transformers-37674be252ec)
-- [Run machine-learning workflows to transform data and build AI-powered semantic search applications with txtai](https://towardsdatascience.com/run-machine-learning-workflows-to-transform-data-and-build-ai-powered-text-indices-with-txtai-43d769b566a7)
-- [Semantic search on the cheap](https://towardsdatascience.com/semantic-search-on-the-cheap-55940c0fcdab)
-- [What's new in txtai 4.0](https://towardsdatascience.com/whats-new-in-txtai-4-0-bbc3a65c3d1c)
-- [Tutorial series on dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
-
-### Contributing
+## Contributing
 
 For those who would like to contribute to txtai, please see [this guide](https://github.com/neuml/.github/blob/master/CONTRIBUTING.md).

README_ko.md

@@ -1,155 +0,0 @@
-<p align="center">
-    <img src="https://raw.githubusercontent.com/neuml/txtai/master/logo.png"/>
-</p>
-
-<h3 align="center">
-    <p>Build AI-powered semantic search applications</p>
-</h3>
-
-<p align="center">
-    <a href="https://github.com/neuml/txtai/releases">
-        <img src="https://img.shields.io/github/release/neuml/txtai.svg?style=flat&color=success" alt="Version"/>
-    </a>
-    <a href="https://github.com/neuml/txtai/releases">
-        <img src="https://img.shields.io/github/release-date/neuml/txtai.svg?style=flat&color=blue" alt="GitHub Release Date"/>
-    </a>
-    <a href="https://github.com/neuml/txtai/issues">
-        <img src="https://img.shields.io/github/issues/neuml/txtai.svg?style=flat&color=success" alt="GitHub issues"/>
-    </a>
-    <a href="https://github.com/neuml/txtai">
-        <img src="https://img.shields.io/github/last-commit/neuml/txtai.svg?style=flat&color=blue" alt="GitHub last commit"/>
-    </a>
-    <a href="https://github.com/neuml/txtai/actions?query=workflow%3Abuild">
-        <img src="https://github.com/neuml/txtai/workflows/build/badge.svg" alt="Build Status"/>
-    </a>
-    <a href="https://coveralls.io/github/neuml/txtai?branch=master">
-        <img src="https://img.shields.io/coveralls/github/neuml/txtai" alt="Coverage Status">
-    </a>
-</p>
-
--------------------------------------------------------------------------------------------------------------------------------------------------------
-
-txtai는 데이터를 변환하고 AI를 기반으로 한 의미론적 검색 애플리케이션을 만들기 위해 머신러닝 워크플로우를 실행합니다.
-
-![demo](https://raw.githubusercontent.com/neuml/txtai/master/demo.gif)
-
-기존에 사용하던 검색 시스템은 데이터를 찾기 위해서 키워드를 사용합니다. 의미론적 검색 애플리케이션은 자연어를 이해하고, 단지 동일한 키워드를 가진게 아닌 동일한 의미를 갖는 결과를 식별합니다.
-
-최첨단 머신 러닝 모델의 지원에 의해, 데이터는 검색을 위한 벡터 표현(임베딩(embeddings)이라고 알려진)으로 변환됩니다. 혁신은 빠른 속도로 일어나고 있으며, 이 모델은 문서, 오디오, 이미지 등의 개념을 이해할 수 있습니다.
-
-txtai 특징 요약:
-
-- 🔎 다수의 index backends를 이용해 대규모 유사성 검색 ([Faiss](https://github.com/facebookresearch/faiss), [Annoy](https://github.com/spotify/annoy), [Hnswlib](https://github.com/nmslib/hnswlib))
-- 📄 텍스트 조각, 문서, 오디오, 이미지 및 비디오에 대한 embeddings를 만듭니다. 변환기와 단어 벡터를 지원합니다.
-- 💡 머신 러닝 파이프라인은 회신, zero-shot 라벨링, 표기, 번역, 요약 및 텍스트 추출과 같은 질문을 추출하기 위해 사용됩니다.
-- ↪️️ 워크플로우는 파이프라인을 연결하여 비즈니스 로직을 합칩니다. txtai 프로세스는 마이크로 서비스 또는 완전한 인덱싱 워크플로우일 수 있습니다.
-- 🔗 [JavaScript](https://github.com/neuml/txtai.js), [Java](https://github.com/neuml/txtai.java), [Rust](https://github.com/neuml/txtai.rs)와 [Go](https://github.com/neuml/txtai.go)용 API 바인딩
-- ☁️ 컨테이너 배포 관리 시스템 (예: Kubernetes)으로 확장되는 Cloud-native 구조
-
-응용 프로그램은 복잡한 NLP기반 데이터 추출을 위한 유사성 검색에서 구조화된 데이터베이스를 생성하기까지의 범위입니다. 다음의 애플리케이션은 txtai를 기반으로 구동됩니다.
-
-| 응용 프로그램  | 설명  |
-|:----------|:-------------|
-| [paperai](https://github.com/neuml/paperai) | 의료/과학 논문을 위한 AI기반 문헌 발견 및 검토 엔진 |
-| [tldrstory](https://github.com/neuml/tldrstory) | 헤드라인과 본문에 대한 AI기반 이해 |
-| [neuspo](https://neuspo.com) | 사실 기반의 실시간 스포츠 이벤트 및 뉴스 사이트 |
-| [codequestion](https://github.com/neuml/codequestion) | 터미널에서 직접 코딩에 관한 질문을 합니다. |
-
-txtai는 Python 3.7+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) 그리고 [FastAPI](https://github.com/tiangolo/fastapi) 로 만들어졌습니다.
-
-## 왜 txtai인가요?
-
-기존 검색 시스템 외에도 점점 더 많은 의미론적 검색 솔루션을 사용할 수 있는데, 왜 txtai를 사용해야 하죠?
-
-- `pip install txtai` 만 있으면 시작할 수 있습니다.
-- 소규모 데이터와 빅 데이터에서 모두 잘 작동하며, 프로토타입을 몇 줄의 코드로 제작할 수 있고, 필요에 따라 확장할 수 있습니다.
-- 사전 및 사후 처리 데이터를 위한 풍부한 데이터 프로세싱 프레임워크(파이프라인과 워크플로우)
--	API를 통해 선택한 프로그래밍 언어로 작업할 수 있습니다.
--	설치 공간이 적고, 대부분의 종속성이 선택적이며, 필요할 때만 모듈을 요구할 수 있습니다. 
--	예시로 알아봅시다. 설명서는 모든 기능을 다룹니다.
-
-## 설치
-
-가장 쉬운 설치 방법은 via pip 과 PyPI입니다.
-
-    pip install txtai
-
-Python 3.7+ 이 지원됩니다.. 파이썬 [가상 환경](https://docs.python.org/3/library/venv.html) 을 사용하는 걸 추천드립니다.
-
-[소스 설치](https://neuml.github.io/txtai/install/#install-from-source),
-[환경별 필수 구성 요소](https://neuml.github.io/txtai/install/#environment-specific-prerequisites) 및
-[선택적 종속성](https://neuml.github.io/txtai/install/#optional-dependencies) 에 대한 자세한 정보는 [설치 지침](https://neuml.github.io/txtai/install) 을 참조하십시오.
-
-## 예시
-
-examples 디렉토리에는 txtai의 개요를 제공하는 일련의 설명서(문서) 및 애플리케이션이 있습니다. 아래 섹션을 참조하십시오.
-
-### 의미론적 검색
-
-의미론적/유사성/벡터 검색 애플리케이션을 제작합니다. 
-
-| 문서  | 설명  |       |
-|:----------|:-------------|------:|
-| [txtai 소개](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) | txtai에서 제공하는 기능 개요 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |
-| [Hugging Face 데이터 셋으로 임베딩 인덱스 구축](https://github.com/neuml/txtai/blob/master/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb) | Hugging Face 데이터 셋 색인 및 검색 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb) |
-| [데이터 소스로 임베딩 인덱스 구축](https://github.com/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb)  | 단어 임베딩을 이용한 데이터 소스 색인 및 검색 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb) |
-| [Elasticsearch에 의미론적 검색 추가](https://github.com/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb)  | 존재하는 검색 시스템에 의미론적 검색 추가 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb) |
-| [API 갤러리](https://github.com/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) | JavaScript, Java, Rust와 Go에서 txtai 사용 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) |
-| [이미지를 이용한 유사성 검색](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) | 검색을 위해 포함된 이미지와 텍스트를 동일한 공간에 삽입 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |
-| [배포된 임베딩 클러스터](https://github.com/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) | 여러 데이터 노드에 임베딩 인덱스 배포 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) |
-
-### 파이프라인 및 워크플로우
-
-NLP 지원 데이터를 파이프라인과 워크플로우로 변환합니다.
-
-| 문서  | 설명  |       |
-|:----------|:-------------|------:|
-| [txtai에 관한 질의응답](https://github.com/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) | txtai에 관한 질의응답 소개 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) |
-| [Elasticsearch에 관한 질의응답](https://github.com/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) | Elasticsearch에 관한 질의응답을 실행 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) |
-| [구조화된 데이터 구축에 관한 질의응답](https://github.com/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) | 질의응답을 통해 구조화된 데이터 셋을 구축 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) |
-| [zero-shot 분류로 레이블을 적용](https://github.com/neuml/txtai/blob/master/examples/07_Apply_labels_with_zero_shot_classification.ipynb) | 라벨링, 분류 및 주제 모델링을 위해 zero-shot 학습을 사용 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/07_Apply_labels_with_zero_shot_classification.ipynb) |
-| [추상적인 텍스트 요약 작성](https://github.com/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) | 추상적인 텍스트 요약을 실행 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) |
-| [문서로부터 텍스트를 추출](https://github.com/neuml/txtai/blob/master/examples/10_Extract_text_from_documents.ipynb) | PDF, Office, HTML 등으로부터 텍스트를 추출 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/10_Extract_text_from_documents.ipynb) |
-| [오디오를 텍스트로 변환](https://github.com/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) | 오디오 파일을 텍스트로 변환 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) |
-| [언어 간에 텍스트를 번역](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) | 기계 번역 및 언어 감지를 간소화 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |
-| [파이프라인과 워크플로우를 실행](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) | 효율적으로 데이터를 처리하기 위한 간단하면서도 강력한 구조 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
-| [구성 가능한 워크플로우로 테이블 형식 데이터 변환](https://github.com/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) | 워크플로우로 테이블 형식 데이터를 변환, 색인 생성및 검색 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) |
-
-### 모델 훈련
-
-NLP 모델을 훈련한다.
-
-| 문서  | 설명  |       |
-|:----------|:-------------|------:|
-| [텍스트 레이블러를 훈련](https://github.com/neuml/txtai/blob/master/examples/16_Train_a_text_labeler.ipynb) | 텍스트 시퀀스 분류 모델을 구축 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/16_Train_a_text_labeler.ipynb) |
-| [레이블을 제외하고 훈련](https://github.com/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) | 새로운 모델을 훈련하기위해 zero-shot 분류기를 사용 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) |
-| [QA 모델을 훈련](https://github.com/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) | 질의응답 모델 구축 및 미세 조정 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) |
-| [ONNX로 모델 내보내기 및 실행](https://github.com/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) | ONNX로 모델을 내보내고, JavaScript, Java와 Rust에서 기본적으로 실행 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) |
-| [다른 머신 러닝 모델들을 내보내기 및 실행](https://github.com/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) | scikit-learn, PyTorch 등에서 모델을 내보내기 및 실행 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) |
-
-### 응용 프로그램
-
-txtai를 이용한 일련의 애플리케이션 예시입니다. [Hugging Face Spaces](https://hf.co/spaces)에서 호스팅된 버전에 대한 링크도 제공됩니다.
-
-| 응용 프로그램  | 설명  |       |
-|:-------------|:-------------|------:|
-| [Basic similarity search](https://github.com/neuml/txtai/blob/master/examples/similarity.py) | 기본 유사성 검색 예시입니다. 원래 txtai 데모에서 사용되었습니다. |[🤗](https://hf.co/spaces/NeuML/similarity)|
-| [Book search](https://github.com/neuml/txtai/blob/master/examples/books.py) | 도서 유사성 검색 애플리케이션입니다. 자연어 문장을 사용하여 책 설명 및 요청을 색인화합니다. |*오직 로컬에서만 실행*|
-| [Image search](https://github.com/neuml/txtai/blob/master/examples/images.py) | 이미지 유사성 검색 애플리케이션입니다. 이미지 디렉토리를 인덱싱하고 검색을 실행하여 입력한 요청과 유사한 이미지를 식별합니다. |[🤗](https://hf.co/spaces/NeuML/imagesearch)|
-| [Wiki search](https://github.com/neuml/txtai/blob/master/examples/wiki.py) | 위키백과를 검색하는 애플리케이션입니다. 위키백과 API를 요청하고 상위 결과를 요약합니다. |[🤗](https://hf.co/spaces/NeuML/wikisummary)|
-| [Workflow builder](https://github.com/neuml/txtai/blob/master/examples/workflows.py) | txtai 워크플로우를 구축하고 실행합니다. 요약, 텍스트 추출, 사본(transcription), 번역 및 유사성 검색 파이프라인을 함께 연결하여 통합 워크플로우를 실행합니다. |[🤗](https://hf.co/spaces/NeuML/txtai)|
-
-### 문서
-
-파이프라인, 워크플로우, 인덱싱 및 API에 대한 구성 설정을 포함한 [txtai 전체 설명서](https://neuml.github.io/txtai)입니다.
-
-### 추가적인 읽기
-
-- [Introducing txtai, AI-powered semantic search built on Transformers](https://towardsdatascience.com/introducing-txtai-an-ai-powered-search-engine-built-on-transformers-37674be252ec)
-- [Run machine-learning workflows to transform data and build AI-powered semantic search applications with txtai](https://towardsdatascience.com/run-machine-learning-workflows-to-transform-data-and-build-ai-powered-text-indices-with-txtai-43d769b566a7)
-- [Semantic search on the cheap](https://towardsdatascience.com/semantic-search-on-the-cheap-55940c0fcdab)
-- [Tutorial series on dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
-
-### 기여하기
-
-txtai에 기여하고 싶은 사람은 [해당 가이드](https://github.com/neuml/.github/blob/master/CONTRIBUTING.md)를 참조하십시오.

docker/aws/workflow.py

@@ -8,6 +8,7 @@ from txtai.api import API
 
 APP = None
 
+
 # pylint: disable=W0603,W0613
 def handler(event, context):
     """

docker/base/Dockerfile

@@ -1,11 +1,14 @@
 # Set base image
-ARG BASE_IMAGE=ubuntu:20.04
+ARG BASE_IMAGE=python:3.11-slim
 FROM $BASE_IMAGE
 
 # Install GPU-enabled version of PyTorch if set
 ARG GPU
 
-# Set Python version (i.e. 3, 3.7, 3.8)
+# Target CPU architecture
+ARG TARGETARCH
+
+# Set Python version (i.e. 3, 3.11)
 ARG PYTHON_VERSION=3
 
 # List of txtai components to install
@@ -18,18 +21,18 @@ ENV LANG=C.UTF-8
 RUN \
     # Install required packages
     apt-get update && \
-    apt-get -y --no-install-recommends install libgomp1 libsndfile1 gcc g++ python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-pip && \
+    apt-get -y --no-install-recommends install libgomp1 libportaudio2 libsndfile1 libegl1 libgles2 libvulkan1 git gcc g++ python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-pip && \
     rm -rf /var/lib/apt/lists && \
     \
     # Install txtai project and dependencies
     ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python && \
     python -m pip install --no-cache-dir -U pip wheel setuptools && \
-    if [ -z ${GPU} ]; then pip install --no-cache-dir pip install torch==1.10.2+cpu torchvision==0.11.3+cpu -f https://download.pytorch.org/whl/torch_stable.html; fi && \
+    if [ -z ${GPU} ]; then pip install --no-cache-dir torch==2.14.1+cpu torchvision==0.29.1+cpu -f https://download.pytorch.org/whl/torch -f https://download.pytorch.org/whl/torchvision; fi && \
     python -m pip install --no-cache-dir txtai${COMPONENTS} && \
-    python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download('punkt')" && \
+    python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download(['punkt', 'punkt_tab', 'averaged_perceptron_tagger_eng'])" && \
     \
     # Cleanup build packages
-    apt-get -y purge gcc g++ python${PYTHON_VERSION}-dev && apt-get -y autoremove
+    apt-get -y purge git gcc g++ python${PYTHON_VERSION}-dev && apt-get -y autoremove
 
 # Set default working directory
 WORKDIR /app

docker/minimal/Dockerfile

@@ -0,0 +1,20 @@
+# Set base image
+ARG BASE_IMAGE=python:3.11-slim
+FROM $BASE_IMAGE
+
+# Target CPU architecture
+ARG TARGETARCH
+
+# Set Python version (i.e. 3, 3.11)
+ARG PYTHON_VERSION=3
+
+# Locale environment variables
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN \
+    # Install minimal txtai project and dependencies
+    MINIMAL=1 python -m pip install --no-cache-dir txtai_minimal
+
+# Set default working directory
+WORKDIR /app

docs/agent/configuration.md

@@ -0,0 +1,141 @@
+# Configuration
+
+An agent takes two main arguments, an LLM and a list of tools.
+
+The txtai agent framework is built with [smolagents](https://github.com/huggingface/smolagents). Additional options can be passed in the `Agent` constructor.
+
+```python
+from datetime import datetime
+
+from txtai import Agent
+
+wikipedia = {
+    "name": "wikipedia",
+    "description": "Searches a Wikipedia database",
+    "provider": "huggingface-hub",
+    "container": "neuml/txtai-wikipedia"
+}
+
+arxiv = {
+    "name": "arxiv",
+    "description": "Searches a database of scientific papers",
+    "provider": "huggingface-hub",
+    "container": "neuml/txtai-arxiv"
+}
+
+def today() -> str:
+    """
+    Gets the current date and time
+
+    Returns:
+        current date and time
+    """
+
+    return datetime.today().isoformat()
+
+agent = Agent(
+    model="Qwen/Qwen3-4B-Instruct-2507",
+    tools=[today, wikipedia, arxiv, "websearch"],
+)
+```
+
+## model
+
+```yaml
+model: string|llm instance
+```
+
+LLM model path or LLM pipeline instance. The `llm` parameter is also supported for backwards compatibility.
+
+See the [LLM pipeline](../../pipeline/text/llm) for more information.
+
+## tools
+
+```yaml
+tools: list
+```
+
+List of tools to supply to the agent. Supports the following configurations.
+
+### function
+
+A function tool takes the following dictionary fields.
+
+| Field       | Description              |
+|:------------|:-------------------------|
+| name        | name of the tool         |
+| description | tool description         |
+| target      | target method / callable |
+
+A function or callable method can also be directly supplied in the `tools` list. In this case, the fields are inferred from the method documentation.
+
+### embeddings
+
+Embeddings indexes have built-in support. Provide the following dictionary configuration to add an embeddings index as a tool.
+
+| Field       | Description                                |
+|:------------|:-------------------------------------------|
+| name        | embeddings index name                      |
+| description | embeddings index description               | 
+| **kwargs    | Parameters to pass to [embeddings.load](../../embeddings/methods/#txtai.embeddings.Embeddings.load) |
+
+### tool
+
+The following shortcut strings load tools directly. Passing a Tool instance is also supported.
+
+| Tool        | Description                                               |
+|:------------|:----------------------------------------------------------|
+| bash        | Runs a shell command through subprocess                   |
+| defaults    | Loads all of these tools as the default toolkit           |
+| edit        | Edits a file in place and returns a diff                  |
+| glob        | Finds matching file patterns in a directory               |
+| grep        | Finds matching file content in a directory                |
+| http.*      | HTTP Path to a [Model Context Protocol (MCP)](https://modelcontextprotocol.io/docs/getting-started/intro) server |
+| python      | Runs a Python action                                      |
+| read        | Reads file or url content, supports text extraction       |
+| todowrite   | Generates a task list to organize complex tasks           |
+| websearch   | Runs a websearch using the built-in websearch tool        |
+| webview     | Extracts content from a web page. Alias for `read` tool   |
+| write       | Writes content to file                                    |
+| *.md        | Loads a [`skill.md`](https://agentskills.io/specification) file |
+
+## instructions
+
+```yaml
+instructions: string|path
+```
+
+Supports loading an `agents.md` file. Can be provided directly as a string or as a path to a file.
+
+[Read more about agents.md here](https://github.com/agentsmd/agents.md)
+
+## template
+
+```yaml
+template: string
+```
+
+Customize the prompt template used by this agent. Supports Jinja templates. Uses a default template when this parameter is not provided.
+Must include `{{ text }}` and `{{ memory }}` placeholders.
+
+## memory
+
+```yaml
+memory: int
+```
+
+Keeps a rolling window of `memory` inputs and outputs. These are added to future prompts and serve as "agent memory".
+
+Supports storing memory by `session` to enable multiple conversation threads. Defaults to shared memory when not set. See the [method documentation](../methods#txtai.agent.base.Agent.__call__) for more information.
+
+## method
+
+```yaml
+method: code|tool
+```
+
+Sets the agent method. Supports either a `code` or `tool` (default) calling agent. A code agent generates Python code and executes that. A tool calling agent generates JSON blocks and calls the agents within those blocks.
+
+Additional options can be directly passed. See [CodeAgent](https://huggingface.co/docs/smolagents/main/en/reference/agents#smolagents.CodeAgent) or [ToolCallingAgent](https://huggingface.co/docs/smolagents/main/en/reference/agents#smolagents.ToolCallingAgent) for a list of parameters.
+
+[Read more here](https://huggingface.co/docs/smolagents/main/en/guided_tour).

docs/agent/index.md

@@ -0,0 +1,156 @@
+# Agent
+
+![agent](../images/agent.png)
+
+An agent automatically creates workflows to answer multi-faceted user requests. Agents iteratively prompt and/or interface with tools to
+step through a process and ultimately come to an answer for a request.
+
+Agent prompting with [`agents.md`](https://github.com/agentsmd/agents.md) and [`skill.md`](https://agentskills.io/specification) are also supported. [Read the configuration](./configuration/#tool) for more on how to setup those up.
+
+Agents excel at complex tasks where multiple tools and/or methods are required. They incorporate a level of randomness similar to different
+people working on the same task. When the request is simple and/or there is a rule-based process, other methods such as RAG and Workflows
+should be explored.
+
+The following code snippet defines a basic agent.
+
+```python
+from datetime import datetime
+
+from txtai import Agent
+
+wikipedia = {
+    "name": "wikipedia",
+    "description": "Searches a Wikipedia database",
+    "provider": "huggingface-hub",
+    "container": "neuml/txtai-wikipedia"
+}
+
+arxiv = {
+    "name": "arxiv",
+    "description": "Searches a database of scientific papers",
+    "provider": "huggingface-hub",
+    "container": "neuml/txtai-arxiv"
+}
+
+def today() -> str:
+    """
+    Gets the current date and time
+
+    Returns:
+        current date and time
+    """
+
+    return datetime.today().isoformat()
+
+agent = Agent(
+    model="Qwen/Qwen3-4B-Instruct-2507",
+    tools=[today, wikipedia, arxiv, "websearch"],
+    max_steps=10,
+)
+```
+
+The agent above has access to two embeddings databases (Wikipedia and ArXiv) and the web. Given the user's input request, the agent decides the best tool to solve the task.
+
+## Example
+
+The first example will solve a problem with multiple data points. See below.
+
+```python
+agent("Which city has the highest population, Boston or New York?")
+```
+
+This requires looking up the population of each city before knowing how to answer the question. Multiple search requests are run to generate a final answer.
+
+## Agentic RAG
+
+Standard retrieval augmented generation (RAG) runs a single vector search to obtain a context and builds a prompt with the context + input question. Agentic RAG is a more complex process that goes through multiple iterations. It can also utilize multiple databases to come to a final conclusion.
+
+The example below aggregates information from multiple sources and builds a report on a topic.
+
+```python
+researcher = """
+You're an expert researcher looking to write a paper on {topic}.
+Search for websites, scientific papers and Wikipedia related to the topic.
+Write a report with summaries and references (with hyperlinks).
+Write the text as Markdown.
+"""
+
+agent(researcher.format(topic="alien life"))
+```
+
+## Agent Teams
+
+Agents can also be tools. This enables the concept of building "Agent Teams" to solve problems. The previous example can be rewritten as a list of agents.
+
+```python
+from txtai import Agent, LLM
+
+llm = LLM("Qwen/Qwen3-4B-Instruct-2507")
+
+websearcher = Agent(
+    model=llm,
+    tools=["websearch"],
+)
+
+wikiman = Agent(
+    model=llm,
+    tools=[{
+        "name": "wikipedia",
+        "description": "Searches a Wikipedia database",
+        "provider": "huggingface-hub",
+        "container": "neuml/txtai-wikipedia"
+    }],
+)
+
+researcher = Agent(
+    model=llm,
+    tools=[{
+        "name": "arxiv",
+        "description": "Searches a database of scientific papers",
+        "provider": "huggingface-hub",
+        "container": "neuml/txtai-arxiv"
+    }],
+)
+
+agent = Agent(
+    model=llm,
+    tools=[{
+        "name": "websearcher",
+        "description": "I run web searches, there is no answer a web search can't solve!",
+        "target": websearcher
+    }, {
+        "name": "wikiman",
+        "description": "Wikipedia has all the answers, I search Wikipedia and answer questions",
+        "target": wikiman
+    }, {
+        "name": "researcher",
+        "description": "I'm a science guy. I search arXiv to get all my answers.",
+        "target": researcher
+    }],
+    max_steps=10
+)
+```
+
+This provides another level of intelligence to the process. Instead of just a single tool execution, each agent-tool combination has it's own reasoning engine.
+
+```python
+agent("""
+Research fundamental concepts about Signal Processing and build a comprehensive report.
+Write the output in Markdown.
+""")
+```
+
+# More examples
+
+Check out this [Agent Quickstart Example](https://github.com/neuml/txtai/blob/master/examples/agent_quickstart.py). Additional examples are listed below.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [What's new in txtai 8.0](https://github.com/neuml/txtai/blob/master/examples/67_Whats_new_in_txtai_8_0.ipynb) | Agents with txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/67_Whats_new_in_txtai_8_0.ipynb) |
+| [Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) | Explore a rich dataset with Graph Analysis and Agents | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |
+| [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
+| [Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) | Exploring how to improve social media engagement with AI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |
+| [Parsing the stars with txtai](https://github.com/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) | Explore an astronomical knowledge graph of known stars, planets, galaxies | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) |
+| [Agentic College Search](https://github.com/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) | Identify list of strong engineering colleges | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |

docs/agent/methods.md

@@ -0,0 +1,4 @@
+# Methods
+
+## ::: txtai.agent.base.Agent.__init__
+## ::: txtai.agent.base.Agent.__call__

docs/api/configuration.md

@@ -30,13 +30,38 @@ Path to save and load the embeddings index. Each API instance can only access a
 
 ### writable
 ```yaml
-path: boolean
+writable: boolean
 ```
 
-Determines if the input embeddings index is writable (true) or read-only (false). This allows serving a read-only index.
+Sets this embeddings index as writable (true) or read-only (false). This allows serving a read-only index. Defaults to read-only.
+
+### reindex
+```yaml
+reindex: boolean
+```
+
+Enables re-indexing for this embeddings index (defaults to disabled).
+
+Re-indexing accepts new configuration and should only be enabled when the configuration comes from trusted sources.
 
 ### cloud
-[Cloud storage settings](../../embeddings/configuration#cloud) can be set under a `cloud` top level configuration group.
+[Cloud storage settings](../../embeddings/configuration/cloud) can be set under a `cloud` top level configuration group.
+
+## Agent
+
+Agents are defined under a top level `agent` key. Each key under the `agent` key is the name of the agent. Constructor parameters can be passed under this key.
+
+The following example defines an agent.
+
+```yaml
+agent:
+    researcher:
+        tools:
+            - websearch
+
+llm:
+    path: Qwen/Qwen3-4B-Instruct-2507
+```
 
 ## Pipeline
 

docs/api/customization.md

@@ -0,0 +1,25 @@
+# Customization
+
+The txtai API has a number of features out of the box that are designed to help get started quickly. API services can also be augmented with custom code and functionality. The two main ways to do this are with extensions and dependencies.
+
+Extensions add a custom endpoint. Dependencies add middleware that executes with each request. See the sections below for more.
+
+## Extensions
+
+While the API is extremely flexible and complex logic can be executed through YAML-driven workflows, some may prefer to create an endpoint in Python. API extensions define custom Python endpoints that interact with txtai applications. 
+
+See the link below for a detailed example.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Custom API Endpoints](https://github.com/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) | Extend the API with custom endpoints | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) |
+
+## Dependencies
+
+txtai has a default API token authorization method that works well in many cases. Dependencies can also add custom logic with each request. This could be an additional authorization step and/or an authentication method. 
+
+See the link below for a detailed example.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [API Authorization and Authentication](https://github.com/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) | Add authorization, authentication and middleware dependencies to the API | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) |

docs/api/index.md

@@ -54,8 +54,7 @@ workflow:
     sumfrench:
         tasks:
             - action: textractor
-              task: storage
-              ids: false
+              task: url
             - action: summary
             - action: translation
               args: ["fr"]
@@ -74,7 +73,7 @@ Assuming this YAML content is stored in a file named config.yml, the following c
 CONFIG=config.yml uvicorn "txtai.api:app"
 ```
 
-uvicorn is a full-featured production ready server with support for SSL and more. See the [uvicorn deployment guide](https://www.uvicorn.org/deployment/) for details.
+Uvicorn is a full-featured production-ready server. See the [Uvicorn deployment guide](https://www.uvicorn.org/deployment/) for more on configuration options.
 
 ## Connect to API
 
@@ -91,16 +90,15 @@ curl \
 
 ## Local instance
 
-A local API instance can be instantiated. In this case, the API runs internally, without any network connections, providing the same consolidated functionality. This enables running txtai in Python with configuration.
+A local instance can be instantiated. In this case, a txtai application runs internally, without any network connections, providing the same consolidated functionality. This enables running txtai in Python with configuration.
 
 The configuration above can be run in Python with:
 
 ```python
-from txtai.api import API
+from txtai import Application
 
-app = API(config.yml)
-
-# Run action
+# Load and run workflow
+app = Application(config.yml)
 app.workflow("sumfrench", ["https://github.com/neuml/txtai"])
 ```
 
@@ -116,13 +114,21 @@ The API can be containerized and run. This will bring up an API instance without
 
 The following programming languages have bindings with the txtai API:
 
+- [Python](https://github.com/neuml/txtai.py)
 - [JavaScript](https://github.com/neuml/txtai.js)
 - [Java](https://github.com/neuml/txtai.java)
 - [Rust](https://github.com/neuml/txtai.rs)
 - [Go](https://github.com/neuml/txtai.go)
 
-See the link below for a detailed example covering how to use the API.
+The API also supports hosting [OpenAI-compatible](./openai) and [Model Context Protocol (MCP)](./mcp) endpoints.
+
+See the links below for detailed examples covering the API.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
 | [API Gallery](https://github.com/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) | Using txtai in JavaScript, Java, Rust and Go | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) |
+| [Distributed embeddings cluster](https://github.com/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) | Distribute an embeddings index across multiple data nodes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) |
+| [Embeddings in the Cloud](https://github.com/neuml/txtai/blob/master/examples/43_Embeddings_in_the_Cloud.ipynb) | Load and use an embeddings index from the Hugging Face Hub | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/43_Embeddings_in_the_Cloud.ipynb) |
+| [Custom API Endpoints](https://github.com/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) | Extend the API with custom endpoints | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) |
+| [API Authorization and Authentication](https://github.com/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) | Add authorization, authentication and middleware dependencies to the API | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) |
+| [OpenAI Compatible API](https://github.com/neuml/txtai/blob/master/examples/74_OpenAI_Compatible_API.ipynb) | Connect to txtai with a standard OpenAI client library | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/74_OpenAI_Compatible_API.ipynb) |

docs/api/mcp.md

@@ -0,0 +1,34 @@
+# Model Context Protocol
+
+The [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction) is an open standard that enables developers to build secure, two-way connections between their data sources and AI-powered tools.
+
+The API can be configured to handle MCP requests. All enabled endpoints set in the API configuration are automatically added as MCP tools.
+
+```yaml
+mcp: boolean|dict
+```
+
+When `mcp` is a boolean, default arguments are used. When `mcp` is a dictionary it supports the following options.
+
+```yaml
+mcp:
+  clientargs: http client options (dict)
+  mcpargs: FastApiMCP options (dict)
+```
+
+See the following links for details on the options.
+
+- [HTTP Client Arguments](https://www.python-httpx.org/api/#asyncclient)
+- [FastApiMCP Arguments](https://github.com/tadata-org/fastapi_mcp/blob/main/fastapi_mcp/server.py#L22)
+
+Once this configuration option is added, a new route is added to the application `/mcp`. 
+
+The [Model Context Protocol Inspector tool](https://www.npmjs.com/package/@modelcontextprotocol/inspector) is a quick way to explore how the MCP tools are exported through this interface.
+
+Run the following and go to the local URL specified.
+
+```
+npx @modelcontextprotocol/inspector node build/index.js
+```
+
+Enter `http://localhost:8000/mcp` to see the full list of tools available.

docs/api/methods.md

@@ -1,12 +1,15 @@
 # Methods
 
 ::: txtai.api.API
-    selection:
+    options:
+        inherited_members: true
         filters:
             - "!__del__"
             - "!flows"
             - "!function"
+            - "!indexes"
             - "!limit"
             - "!pipes"
             - "!read"
             - "!resolve"
+            - "!weights"

docs/api/openai.md

@@ -0,0 +1,13 @@
+# OpenAI-compatible API
+
+The API can be configured to serve an OpenAI-compatible API as shown below.
+
+```yaml
+openai: True
+```
+
+See the link below for a detailed example.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [OpenAI Compatible API](https://github.com/neuml/txtai/blob/master/examples/74_OpenAI_Compatible_API.ipynb) | Connect to txtai with a standard OpenAI client library | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/74_OpenAI_Compatible_API.ipynb) |

docs/api/security.md

@@ -0,0 +1,74 @@
+# Security
+
+The default implementation of an API service runs via HTTP and is fully open. If the service is being run as a prototype on an internal network, that may be fine. In most scenarios, the connection should at least be encrypted. Authorization is another built-in feature that requires a valid API token with each request. See below for more.
+
+## HTTPS
+
+The default API service command starts a Uvicorn server as a HTTP service on port 8000. To run a HTTPS service, consider the following options.
+
+- [TLS Proxy Server](https://fastapi.tiangolo.com/deployment/https/). *Recommended choice*. With this configuration, the txtai API service runs as a HTTP service only accessible on the localhost/local network. The proxy server handles all encryption and redirects requests to local services. See this [example configuration](https://www.uvicorn.dev/deployment/#running-behind-nginx) for more.
+
+- [Uvicorn SSL Certificate](https://www.uvicorn.dev/deployment/). Another option is setting the SSL certificate on the Uvicorn service. This works in simple situations but gets complex when hosting multiple txtai or other related services.
+
+## Authorization
+
+Authorization requires a valid API token with each API request. This token is sent as a HTTP `Authorization` header. 
+
+*Server*
+```bash
+CONFIG=config.yml TOKEN=<sha256 encoded token> uvicorn "txtai.api:app"
+```
+
+*Client*
+```bash
+curl \
+  -X POST "http://localhost:8000/workflow" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <token>" \ 
+  -d '{"name":"sumfrench", "elements": ["https://github.com/neuml/txtai"]}'
+```
+
+It's important to note that HTTPS **must** be enabled using one of the methods mentioned above. Otherwise, tokens will be exchanged as clear text. 
+
+Authentication and Authorization can be fully customized. See the [dependencies](../customization#dependencies) section for more.
+
+## Safe Open
+
+Many components in txtai work with local files and have no restrictions on what files and urls can be accessed. The [textractor pipeline](../../pipeline/data/textractor) and [retrieve](../../workflow/task/retrieve) tasks both have a built-in option called `safeopen` which is enabled by default when running the API and applications. It enforces the following rules:
+
+- Local files must be in the safeopen directory (defaults to temp dir)
+- URLs must be public URLs (also has checks for redirects and DNS rebind attempts)
+
+The following workflow only allows accessing files stored in a specific directory along with public urls.
+
+```yaml
+workflow:
+  textract:
+    tasks:
+      - action: textractor
+```
+
+A number of of pipelines support reading local files. For these it's recommended to wrap those pipelines as workflows with a `retrieve` task. Note that pipelines can be created in-line and don't need to be exposed via the API.
+
+```yaml
+workflow:
+  caption:
+    tasks:
+      - task: retrieve
+        action: caption
+```
+
+Additionally, internal pipelines can be created but not exposed via the API.
+
+```yaml
+routes:
+  caption: False
+
+caption:
+
+workflow:
+  caption:
+    tasks:
+      - task: retrieve
+        action: caption
+```

docs/cloud.md

@@ -3,20 +3,21 @@
 ![cloud](images/cloud.png#only-light)
 ![cloud](images/cloud-dark.png#only-dark)
 
-Scalable cloud-native applications can be built with txtai. The following runtimes are supported.
+Scalable cloud-native applications can be built with txtai. The following cloud runtimes are supported.
 
-- Container Orchestration (i.e. Kubernetes)
+- Container Orchestration Systems (i.e. Kubernetes)
 - Docker Engine
-- Serverless
+- Serverless Compute
+- txtai.cloud (planned for future)
 
-Images for txtai are available on Docker Hub for [CPU](https://hub.docker.com/r/neuml/txtai-cpu) and [GPU](https://hub.docker.com/r/neuml/txtai-gpu) installs. The CPU install is recommended when GPUs aren't available given the image is half the size.
+Images for txtai are available on Docker Hub for [CPU](https://hub.docker.com/r/neuml/txtai-cpu) and [GPU](https://hub.docker.com/r/neuml/txtai-gpu) installs. The CPU install is recommended when GPUs aren't available given the image is significantly smaller.
 
 The base txtai images have no models installed and models will be downloaded each time the container starts. Caching the models is recommended as that will significantly reduce container start times. This can be done a couple different ways.
 
-- Create a container with the [models cached](#cache-models-in-container-images)
+- Create a container with the [models cached](#container-image-model-caching)
 - Set the transformers cache environment variable and mount that volume when starting the image
     ```bash
-    docker run -v <local dir>:/models -e TRANSFORMERS_CACHE=/models --rm --it <docker image>
+    docker run -v <local dir>:/models -e TRANSFORMERS_CACHE=/models --rm -it <docker image>
     ```
 
 ## Build txtai images
@@ -29,8 +30,8 @@ Examples build commands below.
 # Get Dockerfile
 wget https://raw.githubusercontent.com/neuml/txtai/master/docker/base/Dockerfile
 
-# Build Ubuntu 18.04 image running Python 3.7
-docker build -t txtai --build-arg BASE_IMAGE=ubuntu:18.04 --build-arg PYTHON_VERSION=3.7 .
+# Build Ubuntu 22.04 image running Python 3.11
+docker build -t txtai --build-arg BASE_IMAGE=ubuntu:22.04 --build-arg PYTHON_VERSION=3.11 .
 
 # Build image with GPU support
 docker build -t txtai --build-arg GPU=1 .
@@ -44,9 +45,9 @@ docker build -t txtai --build-arg COMPONENTS= .
 As mentioned previously, model caching is recommended to reduce container start times. The following commands demonstrate this. In all cases, it is assumed a config.yml file is present in the local directory with the desired configuration set.
 
 ### API
-This section builds a Docker image that caches models and starts an API service. The config.yml file should be configured with the desired components to expose via the API.
+This section builds an image that caches models and starts an API service. The config.yml file should be configured with the desired components to expose via the API.
 
-The following is a sample config.yml file that creates an Embeddings API service
+The following is a sample config.yml file that creates an Embeddings API service.
 
 ```yaml
 # config.yml
@@ -57,7 +58,7 @@ embeddings:
   content: true
 ```
 
-The next section builds the Docker image and starts an instance.
+The next section builds the image and starts an instance.
 
 ```bash
 # Get Dockerfile
@@ -85,6 +86,9 @@ docker build -t txtai-service .
 
 # GPU build
 docker build -t txtai-service --build-arg BASE_IMAGE=neuml/txtai-gpu .
+
+# Run
+docker run --rm -it txtai-service
 ```
 
 ### Workflow
@@ -99,15 +103,20 @@ docker build -t txtai-workflow .
 
 # GPU build
 docker build -t txtai-workflow --build-arg BASE_IMAGE=neuml/txtai-gpu .
+
+# Run
+docker run --rm -it txtai-workflow <workflow name> <workflow parameters>
 ```
 
 ## Serverless Compute
 
 One of the most powerful features of txtai is building YAML-configured applications with the "build once, run anywhere" approach. API instances and workflows can run locally, on a server, on a cluster or serverless.
 
-Serverless instances of txtai are supported with frameworks such as [AWS SAM](https://github.com/aws/serverless-application-model) and [Serverless](https://github.com/serverless/serverless).
+Serverless instances of txtai are supported on frameworks such as [AWS Lambda](https://aws.amazon.com/lambda/), [Google Cloud Functions](https://cloud.google.com/functions), [Azure Cloud Functions](https://azure.microsoft.com/en-us/services/functions/) and [Kubernetes](https://kubernetes.io/) with [Knative](https://knative.dev/docs/).
 
-The following steps shows a basic example of how to spin up a serverless API instance with AWS SAM.
+### AWS Lambda
+
+The following steps show a basic example of how to build a serverless API instance with [AWS SAM](https://github.com/aws/serverless-application-model).
 
 - Create config.yml and template.yml
 
@@ -161,3 +170,13 @@ curl http://localhost:8080/count
 ```
 
 If successful, a local API instance is now running in a "serverless" fashion. This configuration can be deployed to AWS using SAM. [See this link for more information.](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-cli-command-reference-sam-deploy.html)
+
+### Kubernetes with Knative
+
+txtai scales with container orchestration systems. This can be self-hosted or with a cloud provider such as [Amazon Elastic Kubernetes Service](https://aws.amazon.com/eks/), [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine) and [Azure Kubernetes Service](https://azure.microsoft.com/en-us/services/kubernetes-service/). There are also other smaller providers with a managed Kubernetes offering.
+
+A full example covering how to build a serverless txtai application on Kubernetes with Knative [can be found here](https://medium.com/neuml/serverless-vector-search-with-txtai-96f6163ab972).
+
+## txtai.cloud
+
+[txtai.cloud](https://txtai.cloud) is a planned effort that will offer an easy and secure way to run hosted txtai applications.

docs/embeddings/configuration.md

@@ -1,197 +0,0 @@
-# Configuration
-
-## Embeddings
-This following describes available embeddings configuration. These parameters are set via the [Embeddings constructor](../methods#txtai.embeddings.base.Embeddings.__init__).
-
-### method
-```yaml
-method: transformers|sentence-transformers|words|external
-```
-
-Sentence embeddings method to use. Options listed below.
-
-#### transformers
-
-Builds sentence embeddings using a transformers model. While this can be any transformers model, it works best with
-[models trained](https://huggingface.co/models?pipeline_tag=sentence-similarity) to build sentence embeddings.
-
-#### sentence-transformers
-
-Same as transformers but loads models with the sentence-transformers library.
-
-#### words
-
-Builds sentence embeddings using a word embeddings model.
-
-#### external
-
-Sentence embeddings are loaded via an external model or API. Requires setting the `transform` parameter to a function that translates data into vectors.
-
-The method is inferred using the _path_, if not provided. sentence-transformers and words require the [similarity](../../install/#similarity) extras package to be installed.
-
-### path
-```yaml
-path: string
-```
-
-Sets the path for a vectors model. When using a transformers/sentence-transformers model, this can be any model on the
-[Hugging Face Model Hub](https://huggingface.co/models) or a local file path. Otherwise, it must be a local file path to a word embeddings model.
-
-### backend
-```yaml
-backend: faiss|hnsw|annoy
-```
-
-Approximate Nearest Neighbor (ANN) index backend for storing generated sentence embeddings. Defaults to Faiss. Additional backends require the
-[similarity](../../install/#similarity) extras package to be installed.
-
-Backend-specific settings are set with a corresponding configuration object having the same name as the backend (i.e. annoy, faiss, or hnsw). None of these are required and are set to defaults if omitted.
-
-#### faiss
-```yaml
-faiss:
-    components: Comma separated list of components - defaults to "Flat" for small
-                indices and "IVFx,Flat" for larger indexes where
-                x = 4 * sqrt(embeddings count)
-    nprobe: search probe setting (int) - defaults to x/16 (as defined above)
-            for larger indexes
-```
-
-See the following Faiss documentation links for more information.
-
-- [Guidelines for choosing an index](https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index)
-- [Index configuration summary](https://github.com/facebookresearch/faiss/wiki/Faiss-indexes)
-- [Index Factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory)
-- [Search Tuning](https://github.com/facebookresearch/faiss/wiki/Faster-search)
-
-#### hnsw
-```yaml
-hnsw:
-    efconstruction:  ef_construction param for init_index (int) - defaults to 200
-    m: M param for init_index (int) - defaults to 16
-    randomseed: random-seed param for init_index (init) - defaults to 100
-    efsearch: ef search param (int) - defaults to None and not set
-```
-
-See [Hnswlib documentation](https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md) for more information on these parameters.
-
-#### annoy
-```yaml
-annoy:
-    ntrees: number of trees (int) - defaults to 10
-    searchk: search_k search setting (int) - defaults to -1
-```
-
-See [Annoy documentation](https://github.com/spotify/annoy#full-python-api) for more information on these parameters. Note that annoy indexes can not be modified after creation, upserts/deletes and other modifications are not supported.
-
-### content
-```yaml
-content: string|boolean
-```
-
-Enables content storage. When true, the default content storage engine will be used. Otherwise, the string must specify the supported content storage engine to use.
-
-### quantize
-```yaml
-quantize: boolean
-```
-
-Enables quanitization of generated sentence embeddings. If the index backend supports it, sentence embeddings will be stored with 8-bit precision vs 32-bit.
-Only Faiss currently supports quantization.
-
-### Additional configuration for Transformers models
-
-#### tokenize
-```yaml
-tokenize: boolean
-```
-
-Enables string tokenization (defaults to false). This method applies tokenization rules that only work with English language text and may increase the quality of
-English language sentence embeddings in some situations.
-
-### Additional configuration for Word embedding models
-
-Word embeddings provide a good tradeoff of performance to functionality for a similarity search system. With that being said, Transformers models are making great progress in scaling performance down to smaller models and are the preferred vector backend in txtai for most cases.
-
-Word embeddings models require the [similarity](../../install/#similarity) extras package to be installed.
-
-#### storevectors
-```yaml
-storevectors: boolean
-```
-
-Enables copying of a vectors model set in path into the embeddings models output directory on save. This option enables a fully encapsulated index with no external file dependencies.
-
-#### scoring
-```yaml
-scoring: bm25|tfidf|sif
-```
-
-A scoring model builds weighted averages of word vectors for a given sentence. Supports BM25, TF-IDF and SIF (smooth inverse frequency) methods. If a scoring method is not provided, mean sentence embeddings are built.
-
-#### pca
-```yaml
-pca: int
-```
-
-Removes _n_ principal components from generated sentence embeddings. When enabled, a TruncatedSVD model is built to help with dimensionality reduction. After pooling of vectors creates a single sentence embedding, this method is applied.
-
-## Cloud
-
-This section describes parameters used to sync compressed indexes with cloud storage. These parameters are only enabled if an embeddings index is stored as compressed. They are set via the [embeddings.load](../methods/#txtai.embeddings.base.Embeddings.load) and [embeddings.save](../methods/#txtai.embeddings.base.Embeddings.save) methods.
-
-### provider
-```yaml
-provider: string
-```
-
-The cloud storage provider, see [full list of providers here](https://libcloud.readthedocs.io/en/stable/storage/supported_providers.html).
-
-### container
-```yaml
-container: string
-```
-
-Container/bucket/directory name.
-
-### key
-```yaml
-key: string
-```
-
-Provider-specific access key. Can also be set via ACCESS_KEY environment variable. Ensure the configuration file is secured if added to the file.
-
-### secret
-```yaml
-secret: string
-```
-
-Provider-specific access secret. Can also be set via ACCESS_SECRET environment variable. Ensure the configuration file is secured if added to the file.
-
-### host
-```yaml
-host: string
-```
-
-Optional server host name. Set when using a local cloud storage server.
-
-### port
-```yaml
-port: int
-```
-
-Optional server port. Set when using a local cloud storage server.
-
-### token
-```yaml
-token: string
-```
-
-Optional temporary session token
-
-### region
-```yaml
-region: string
-```
-
-Optional parameter to specify the storage region, provider-specific.

docs/embeddings/configuration/ann.md

@@ -0,0 +1,175 @@
+# ANN
+
+Approximate Nearest Neighbor (ANN) index configuration for storing vector embeddings.
+
+## backend
+```yaml
+backend: faiss|hnsw|annoy|ggml|milvus|numpy|rabitq|torch|turbovec|zvec|pgvector|sqlite|custom
+```
+
+Sets the ANN backend. Defaults to `faiss`. Additional backends are available via the [ann](../../../install/#ann) extras package. Set custom backends via setting this parameter to the fully resolvable class string.
+
+Backend-specific settings are set with a corresponding configuration object having the same name as the backend (i.e. annoy, faiss, or hnsw). These are optional and set to defaults if omitted.
+
+### faiss
+```yaml
+faiss:
+    components: comma separated list of components - defaults to "IDMap,Flat" for small
+                indices and "IVFx,Flat" for larger indexes where
+                x = min(4 * sqrt(embeddings count), embeddings count / 39)
+                automatically calculates number of IVF cells when omitted (supports "IVF,Flat")
+    nprobe: search probe setting (int) - defaults to x/16 (as defined above)
+            for larger indexes
+    nflip: same as nprobe - only used with binary hash indexes
+    quantize: store vectors with x-bit precision vs 32-bit (boolean|int)
+              true sets 8-bit precision, false disables, int sets specified
+              precision
+    mmap: load as on-disk index (boolean) - trade query response time for a
+          smaller RAM footprint, defaults to false
+    sample: percent of data to use for model training (0.0 - 1.0)
+            reduces indexing time for larger (>1M+ row) indexes, defaults to 1.0
+```
+
+Faiss supports both floating point and binary indexes. Floating point indexes are the default. Binary indexes are used when indexing scalar-quantized datasets.
+
+See the following Faiss documentation links for more information.
+
+- [Guidelines for choosing an index](https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index)
+- [Index configuration summary](https://github.com/facebookresearch/faiss/wiki/Faiss-indexes)
+- [Index Factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory)
+- [Binary Indexes](https://github.com/facebookresearch/faiss/wiki/Binary-indexes)
+- [Search Tuning](https://github.com/facebookresearch/faiss/wiki/Faster-search)
+
+Note: For macOS users, an existing bug in an upstream package restricts the number of processing threads to 1. This limitation is managed internally to prevent system crashes.
+
+### hnsw
+```yaml
+hnsw:
+    efconstruction:  ef_construction param for init_index (int) - defaults to 200
+    m: M param for init_index (int) - defaults to 16
+    randomseed: random-seed param for init_index (int) - defaults to 100
+    efsearch: ef search param (int) - defaults to None and not set
+```
+
+See [Hnswlib documentation](https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md) for more information on these parameters.
+
+### annoy
+```yaml
+annoy:
+    ntrees: number of trees (int) - defaults to 10
+    searchk: search_k search setting (int) - defaults to -1
+```
+
+See [Annoy documentation](https://github.com/spotify/annoy#full-python-api) for more information on these parameters. Note that annoy indexes can not be modified after creation, upserts/deletes and other modifications are not supported.
+
+### ggml
+```yaml
+ggml:
+    gpu: enable GPU - defaults to True
+    quantize: sets the tensor quantization - defaults to F32
+    querysize: query buffer size - defaults to 64
+```
+
+The [GGML](https://github.com/ggml-org/ggml) backend is a k-nearest neighbors backend. It stores tensors using GGML and [GGUF](https://huggingface.co/docs/hub/en/gguf). It supports GPU-enabled operations and supports quantization. GGML is the framework used by [llama.cpp](https://github.com/ggml-org/llama.cpp).
+
+[See this](https://github.com/ggml-org/ggml/blob/master/include/ggml.h#L379) for a list of quantization types.
+
+### milvus
+
+```yaml
+milvus:
+    m: number of HNSW links per element (int) - defaults to 50
+```
+
+### numpy
+
+```yaml
+numpy:
+    safetensors: stores vectors using the safetensors format
+                 defaults to NumPy array storage
+```
+
+The NumPy backend is a k-nearest neighbors backend. It's designed for simplicity and works well with smaller datasets that fit into memory.
+
+### rabitq
+
+```yaml
+rabitq:
+    mode: index mode (ivf or hnsw) - defaults to "ivf"
+    nbits: number of quantization bits per dimension (int) - defaults to 1,
+           supports 1 to 9 in both modes, ivf mode also supports 32 which
+           keeps raw vectors for reranking
+    clusters: number of IVF clusters (int) - defaults to
+              max(1, min(round(4 * sqrt(embeddings count)), embeddings count))
+    nprobe: search probe setting for ivf mode (int) - defaults to
+            max(1, round(num_clusters/16))
+    m: M param for hnsw mode (int) - defaults to 16
+    efconstruction: ef_construction param for hnsw mode (int) - defaults to 200
+    efsearch: ef search param for hnsw mode (int) - defaults to None and not set
+    randomseed: random-seed param for hnsw mode (int) - defaults to 100
+```
+
+The [rabitq](https://github.com/VectorDB-NTU/RaBitQ-Library) backend is a quantized index powered by the [RaBitQ algorithm](https://doi.org/10.1145/3725413). It supports ivf and hnsw search modes. Vectors are stored with 1-bit quantization by default. More bits increase accuracy at the cost of memory. The index is stored as a single file and no vectors are kept outside of it. With nbits set to 32, the vectors are stored in the index for reranking. The ivf mode supports upserts and deletes. Deleted rows keep their storage. Rows added after the first index are assigned to the clusters trained when the index was built, so recall can drop if the new data differs from the original data. Rebuild the index to train new clusters. Note that hnsw indexes can not be modified after creation, upserts/deletes and other modifications are not supported.
+
+### torch
+
+```yaml
+torch:
+    safetensors: stores vectors using the safetensors format - defaults
+                 to NumPy array storage if quantization is disabled
+    quantize:
+        type: quantization type (fp4, nf4, int8)
+        blocksize: quantization block size parameter
+```
+
+The Torch backend is a k-nearest neighbors backend like NumPy. It supports GPU-enabled operations. It also has support for quantization which enables fitting larger arrays into GPU memory.
+
+When quantization is enabled, vectors are _always_ stored in safetensors. _Note that macOS support for quantization is limited._
+
+### turbovec
+
+```yaml
+turbovec:
+    bitwidth: number of bits to store each vector dimension as. Supports 2, 3 or 4.
+```
+
+The [turbovec](https://github.com/RyanCodrai/turbovec) backend is a k-nearest neighbors backend powered by the [TurboQuant algorithm](https://arxiv.org/abs/2504.19874).
+
+### zvec
+
+```yaml
+zvec:
+    efconstruction: ef_construction param for HnswIndexParam (int) - defaults to 200
+    m: number of HNSW links per element (int) - defaults to 50
+    efsearch: ef search param for HnswQueryParam (int) - defaults to 300
+```
+
+The [zvec](https://github.com/alibaba/zvec) backend is an embedded, path-based vector index.
+
+### pgvector
+```yaml
+pgvector:
+    url: database url connection string, alternatively can be set via
+         ANN_URL environment variable
+    schema: database schema to store vectors - defaults to being
+            determined by the database
+    table: database table to store vectors - defaults to `vectors`
+    precision: vector float precision (half or full) - defaults to `full`
+    efconstruction:  ef_construction param (int) - defaults to 200
+    m: M param for init_index (int) - defaults to 16
+```
+
+The pgvector backend stores embeddings in a Postgres database. See the [pgvector documentation](https://github.com/pgvector/pgvector-python?tab=readme-ov-file#sqlalchemy) for more information on these parameters. See the [SQLAlchemy](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls) documentation for more information on how to construct url connection strings.
+
+### sqlite
+```yaml
+sqlite:
+    quantize: store vectors with x-bit precision vs 32-bit (boolean|int)
+              true sets 8-bit precision, false disables, int sets specified
+              precision
+    table: database table to store vectors - defaults to `vectors`
+```
+
+The SQLite backend stores embeddings in a SQLite database using [sqlite-vec](https://github.com/asg017/sqlite-vec). This backend supports 1-bit and 8-bit quantization at the storage level.
+
+See [this note](https://alexgarcia.xyz/sqlite-vec/python.html#macos-blocks-sqlite-extensions-by-default) on how to run this ANN on MacOS.

docs/embeddings/configuration/cloud.md

@@ -0,0 +1,110 @@
+# Cloud
+
+The following describes parameters used to sync indexes with cloud storage. Cloud object storage, the [Hugging Face Hub](https://huggingface.co/models) and custom providers are all supported.
+
+Parameters are set via the [embeddings.load](../../methods/#txtai.embeddings.base.Embeddings.load) and [embeddings.save](../../methods/#txtai.embeddings.base.Embeddings.save) methods.
+
+## provider
+```yaml
+provider: string
+```
+
+Cloud provider. Can be one of the following:
+
+- Cloud object storage. Set to one of these [providers](https://libcloud.readthedocs.io/en/stable/storage/supported_providers.html). Use the text shown in the `Provider Constant` column as lower case.
+
+- Hugging Face Hub. Set to `huggingface-hub`.
+
+- Custom providers. Set to the full class path of the custom provider.
+
+## container
+```yaml
+container: string
+```
+
+Container/bucket/directory/repository name. Embeddings will be stored in the container with the filename specified by the `path` configuration.
+
+## Cloud object storage configuration
+
+In addition to the above common configuration, the cloud object storage provider has the following additional configuration parameters. Note that some cloud providers do not need any of these parameters and can use implicit authentication with service accounts.
+
+See the [libcloud documentation](https://libcloud.readthedocs.io/en/stable/apidocs/libcloud.common.html#module-libcloud.common.base) for more information on these parameters.
+
+### key
+```yaml
+key: string
+```
+
+Provider-specific access key. Can also be set via `ACCESS_KEY` environment variable. Ensure the configuration file is secured if added to the file. When using implicit authentication, set this to a value such as 'using-implicit-auth'.
+
+### secret
+```yaml
+secret: string
+```
+
+Provider-specific access secret. Can also be set via `ACCESS_SECRET` environment variable. Ensure the configuration file is secured if added to the file. When using implicit authentication, this option is not required.
+
+### prefix
+```yaml
+prefix: string
+```
+
+Optional object prefix. Object storage doesn't have the concept of a directory but a prefix is similar. For example, a prefix could be `base/dir`. This helps with organizing data in an object storage bucket.
+
+More can be found at the following links.
+
+- [Organizing objects using prefixes](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html)
+- [libcloud container method documentation](https://libcloud.readthedocs.io/en/stable/storage/api.html#libcloud.storage.base.StorageDriver.iterate_container_objects)
+
+### host
+```yaml
+host: string
+```
+
+Optional server host name. Set when using a local cloud storage server.
+
+### port
+```yaml
+port: int
+```
+
+Optional server port. Set when using a local cloud storage server.
+
+### token
+```yaml
+token: string
+```
+
+Optional temporary session token
+
+### region
+```yaml
+region: string
+```
+
+Optional parameter to specify the storage region, provider-specific.
+
+## Hugging Face Hub configuration
+
+The huggingface-hub provider supports the following additional configuration parameters. More on these parameters can be found in the [Hugging Face Hub's documentation](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/overview).
+
+### revision
+```yaml
+revision: string
+```
+
+Optional Git revision id which can be a branch name, a tag, or a commit hash
+
+### cache
+```yaml
+cache: string
+```
+
+Path to the folder where cached files are stored
+
+### token
+```yaml
+token: string|boolean
+```
+
+Token to be used for the download. If set to True, the token will be read from the Hugging Face config folder.

docs/embeddings/configuration/database.md

@@ -0,0 +1,83 @@
+# Database
+
+Databases store metadata, text and binary content.
+
+## content
+```yaml
+content: boolean|sqlite|duckdb|client|url|custom
+```
+
+Enables content storage. When true, the default storage engine, `sqlite` will be used to save metadata.
+
+Client-server connections are supported with either `client` or a full connection URL. When set to `client`, the CLIENT_URL environment variable must be set to the full connection URL. See the [SQLAlchemy](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls) documentation for more information on how to construct connection strings for client-server databases.
+
+Add custom storage engines via setting this parameter to the fully resolvable class string.
+
+Content storage specific settings are set with a corresponding configuration object having the same name as the content storage engine (i.e. duckdb or sqlite). These are optional and set to defaults if omitted.
+
+### client
+```yaml
+schema:  default database schema for the session - defaults to being
+         determined by the database
+```
+
+Additional settings for client-server databases. Also supported when the `content=url`.
+
+### sqlite
+```yaml
+sqlite:
+    wal: enable write-ahead logging - allows concurrent read/write operations,
+         defaults to false
+```
+
+Additional settings for SQLite.
+
+## objects
+```yaml
+objects: boolean|image|pickle
+```
+
+Enables object storage. Supports storing binary content. Requires content storage to also be enabled.
+
+Object encoding options are:
+
+- `standard`: Default encoder when boolean set. Encodes and decodes objects as byte arrays.
+- `image`: Image encoder. Encodes and decodes objects as image objects.
+- `pickle`: Pickle encoder. Encodes and decodes objects with the pickle module. Supports arbitrary objects.
+
+## functions
+```yaml
+functions: list
+```
+
+List of functions with user-defined SQL functions. Each list element must be one of the following:
+
+- function
+- callable object
+- dict with fields for name, argcount, function and deterministic
+
+[An example can be found here](../../query#custom-sql-functions).
+
+## expressions
+```yaml
+expressions: list
+```
+
+List of expression shortcuts. Each list element must be a dict with the following fields.
+
+- `name`: name of the expression
+- `expression`: SQL expression, defaults to `name` when empty
+- `index`: if this expression should have a database index, defaults to False when not provided
+
+The expression can be a json data column, sql function or anything that can be run as a SQL snippet.
+
+## query
+```yaml
+query:
+    path: sets the path for the query model - this can be any model on the
+          Hugging Face Model Hub or a local file path.
+    prefix: text prefix to prepend to all inputs
+    maxlength: maximum generated sequence length
+```
+
+Query translation model. Translates natural language queries to txtai compatible SQL statements.

docs/embeddings/configuration/general.md

@@ -0,0 +1,83 @@
+# General
+
+General configuration options.
+
+## keyword
+```yaml
+keyword: boolean|string
+```
+
+Enables sparse keyword indexing for this embeddings.
+
+When set to a boolean, this parameter creates a BM25 index for full text search. When set to a string, it expects a [keyword method](../scoring#method).
+
+It also implicitly disables the [defaults](#defaults) setting for vector search.
+
+## sparse
+```yaml
+sparse: boolean|path
+```
+
+Enables sparse vector indexing for this embeddings.
+
+When set to `True`, this parameter creates a sparse vector index using the [default sparse index model](https://huggingface.co/prithivida/Splade_PP_en_v2). When set to a string, it expects a local or Hugging Face model path.
+
+It also implicitly disables the [defaults](#defaults) setting for vector search.
+
+## dense
+```yaml
+dense: boolean|string
+```
+
+Alias for the [vector model path](../vectors/#path). When set to `True`, the [default transformers vector model](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) is used.
+
+## hybrid
+```yaml
+hybrid: boolean
+```
+
+Enables hybrid (sparse + dense) indexing for this embeddings.
+
+When enabled, this parameter creates a BM25 index for full text search. It has no effect on the [defaults](#defaults) or [path](../vectors/#path) settings.
+
+## defaults
+```yaml
+defaults: boolean
+```
+
+Uses default vector model path when enabled (default setting is True) and `path` is not provided. See [this link](../) for an example.
+
+## indexes
+```yaml
+indexes: dict
+```
+
+Key value pairs defining subindexes for this embeddings. Each key is the index name and the value is the full configuration. This configuration can use any of the available configurations in a standard embeddings instance.
+
+## autoid
+```yaml
+format: int|uuid function
+```
+
+Sets the auto id generation method. When this is not set, an autogenerated numeric sequence is used. This also supports [UUID generation functions](https://docs.python.org/3/library/uuid.html#uuid.uuid1). For example, setting this value to `uuid4` will generate random UUIDs. Setting this to `uuid5` will generate deterministic UUIDs for each input data row.
+
+## columns
+```yaml
+columns:
+    text: name of the text column
+    object: name of the object column
+    store: limit json data fields to this list of columns
+```
+
+Sets the `text` and `object` column names. Defaults to `text` and `object` if not provided.
+
+`store` sets a list of columns to store in the JSON data field. When this isn't provided, all columns are stored (default). When `store` is set to `None`, no JSON columns are stored. This is useful is a field is only needed at indexing time but not search time.
+
+## format
+```yaml
+format: json|pickle
+```
+
+Sets the configuration storage format. Defaults to `json`.
+
+Note that `pickle` configuration is deprecated and will raise an error with the default settings. Its only kept for reading legacy indexes and requires setting the `ALLOW_PICKLE` environment variable. Only enable for local and/or trusted sources.

docs/embeddings/configuration/graph.md

@@ -0,0 +1,80 @@
+# Graph
+
+Enable graph storage via the `graph` parameter. This component requires the [graph](../../../install/#graph) extras package.
+
+When enabled, a graph network is built using the embeddings index. Graph nodes are synced with each embeddings index operation (index/upsert/delete). Graph edges are created using the embeddings index upon completion of each index/upsert/delete embeddings index call.
+
+## backend
+```yaml
+backend: networkx|rdbms|custom
+```
+
+Sets the graph backend. Defaults to `networkx`.
+
+Add custom graph storage engines via setting this parameter to the fully resolvable class string.
+
+The `rdbms` backend has the following additional settings.
+
+### rdbms
+```yaml
+url: database url connection string, alternatively can be set via the
+     GRAPH_URL environment variable
+schema: database schema to store graph - defaults to being
+        determined by the database
+nodes: table to store node data, defaults to `nodes`
+edges: table to store edge data, defaults to `edges`
+```
+
+## batchsize
+```yaml
+batchsize: int
+```
+
+Batch query size, used to query embeddings index - defaults to 256.
+
+## limit
+```yaml
+limit: int
+```
+
+Maximum number of results to return per embeddings query - defaults to 15.
+
+## minscore
+```yaml
+minscore: float
+```
+
+Minimum score required to consider embeddings query matches - defaults to 0.1.
+
+## approximate
+```yaml
+approximate: boolean
+```
+
+When true, queries only run for nodes without edges - defaults to true.
+
+## topics
+```yaml
+topics:
+    algorithm: community detection algorithm (string), options are
+               louvain (default), greedy, lpa
+    level: controls number of topics (string), options are best (default) or first
+    resolution: controls number of topics (int), larger values create more
+                topics (int), defaults to 100
+    labels: scoring index method used to build topic labels (string)
+            options are bm25 (default), tfidf, sif
+    terms: number of frequent terms to use for topic labels (int), defaults to 4
+    stopwords: optional list of stop words to exclude from topic labels
+    categories: optional list of categories used to group topics, allows
+                granular topics with broad categories grouping topics
+```
+
+Enables topic modeling. Defaults are tuned so that in most cases these values don't need to be changed (except for categories). These parameters are available for advanced use cases where one wants full control over the community detection process.
+
+## copyattributes
+```yaml
+copyattributes: boolean|list
+```
+
+Copy these attributes from input dictionaries in the `insert` method. If this is set to `True`, all attributes are copied. Otherwise, only the
+attributes specified in this list are copied to the graph as attributes.

docs/embeddings/configuration/index.md

@@ -0,0 +1,51 @@
+# Configuration
+
+The following describes available embeddings configuration. These parameters are set in the [Embeddings constructor](../methods#txtai.embeddings.base.Embeddings.__init__) via either the `config` parameter or as keyword arguments.
+
+Configuration is designed to be optional and set only when needed. Out of the box, sensible defaults are picked to get up and running fast. For example:
+
+```python
+from txtai import Embeddings
+
+embeddings = Embeddings()
+```
+
+Creates a new embeddings instance, using [all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) as the vector model, [Faiss](https://faiss.ai/) as the ANN index backend and content disabled.
+
+```python
+from txtai import Embeddings
+
+embeddings = Embeddings(content=True)
+```
+
+Is the same as above except it adds in [SQLite](https://www.sqlite.org/index.html) for content storage. 
+
+The following sections link to all the available configuration options.
+
+## [ANN](./ann)
+
+The default vector index backend is Faiss.
+
+## [Cloud](./cloud)
+
+Embeddings databases can optionally be synced with cloud storage.
+
+## [Database](./database)
+
+Content storage is disabled by default. When enabled, SQLite is the default storage engine.
+
+## [General](./general)
+
+General configuration that doesn't fit elsewhere.
+
+## [Graph](./graph)
+
+An accomplying graph index can be created with an embeddings database. This enables topic modeling, path traversal and more. [NetworkX](https://github.com/networkx/networkx) is the default graph index.
+
+## [Scoring](./scoring)
+
+Sparse keyword indexing and word vectors term weighting.
+
+## [Vectors](./vectors)
+
+Vector search is enabled by converting text and other binary data into embeddings vectors. These vectors are then stored in an ANN index. The vector model is optional and a default model is used when not provided.

docs/embeddings/configuration/scoring.md

@@ -0,0 +1,128 @@
+# Scoring
+
+Enable scoring support via the `scoring` parameter.
+
+This scoring instance can serve two purposes, depending on the settings.
+
+One use case is building sparse/keyword indexes. This occurs when the `terms` parameter is set to `True`.
+
+The other use case is with word vector term weighting. This feature has been available since the initial version but isn't quite as common anymore.
+
+The following covers the available options.
+
+## method
+```yaml
+method: bm25|tfidf|sif|pgtext|sparse|custom
+```
+
+Sets the scoring method. Add custom scoring via setting this parameter to the fully resolvable class string.
+
+### pgtext
+```yaml
+schema: database schema to store keyword index - defaults to being
+        determined by the database
+```
+
+Additional settings for Postgres full-text keyword indexes.
+
+### sparse
+```yaml
+path: sparse vector model path
+vectormethod: vector embeddings method
+vectornormalize: enable vector embeddings normalization (boolean)
+gpu: boolean|int|string|device
+normalize: enable score normalization (boolean|float|string|dict)
+batch: Sets the transform batch size
+encodebatch: Sets the encode batch size
+vectors: additional model init args
+encodeargs: additional encode() args
+backend: ivfsparse|pgsparse|zvecsparse
+```
+
+Sparse vector scoring options. The sparse scoring instance combines a sparse vector model with a sparse approximate nearest neighbor index (ANN). This method supports both vector normalization and score normalization.
+
+Vector normalization normalizes all vectors to have a magnitude of 1. By extension, all generated scores will be 0 to 1.
+
+Score normalization scales the output between 0 and 1. This setting supports:
+
+- `True` for default scale normalization
+- `float` normalize using this as the scale factor
+- `"bayes"` for Bayesian normalization using dynamic candidate score statistics
+- `{method: "bayes", alpha: 1.0, beta: null}` for Bayesian normalization with optional custom parameters
+
+#### ivfsparse
+```yaml
+ivfsparse:
+  sample: percent of data to use for model training (0.0 - 1.0)
+  nfeatures: top n features to use for model training, defaults to all features (int)
+  nlist: desired number of clusters (int)
+  nprobe: search probe setting (int)
+  minpoints: minimum number of points for a cluster (int)
+```
+
+Inverted file (IVF) index with flat vector file storage and sparse array support.
+
+#### pgsparse
+
+Sparse ANN backed by Postgres. Supports same options as the [pgvector](../ann/#pgvector) ANN.
+
+#### zvecsparse
+```yaml
+zvecsparse:
+  efconstruction: ef_construction param for HnswIndexParam (int) - defaults to 200
+  m: number of HNSW links per element (int) - defaults to 50
+  efsearch: ef search param for HnswQueryParam (int) - defaults to 300
+```
+
+Sparse ANN backed by [zvec](https://github.com/alibaba/zvec). Stores sparse vectors in an embedded, path-based vector index. Requires the [ann](../../../install/#ann) extras package.
+
+## terms
+```yaml
+terms: boolean|dict
+```
+
+Enables term frequency sparse arrays for a scoring instance. This is the backend for sparse keyword indexes.
+
+Supports a `dict` with the parameters `cachelimit` and `cutoff`.
+
+`cachelimit` is the maximum amount of resident memory in bytes to use during indexing before flushing to disk. This parameter is an `int`.
+
+`cutoff` is used during search to determine what constitutes a common term. This parameter is a `float`, i.e. 0.1 for a cutoff of 10%.
+
+When `terms` is set to `True`, default parameters are used for the `cachelimit` and `cutoff`. Normally, these defaults are sufficient.
+
+### Wildcard queries
+
+Keyword queries containing `*` expand matching terms using SQL `LIKE`. Within these
+expressions, `%` matches any sequence and `_` matches one character. To match a literal
+percent sign, underscore or backslash, use `\%`, `\_` or `\\`, respectively.
+For example, `report\_*` matches `report_2026`, while `report_*` also matches `reportX2026`.
+Use a tokenizer that preserves these characters, such as `tokenizer: {whitespace: true}`,
+or pass pre-tokenized query terms. Queries without `*` retain exact-term matching.
+
+## normalize
+```yaml
+normalize: boolean|str|dict
+```
+
+Enables normalized scoring (ranging from 0 to 1). This setting supports:
+
+- `True` for standard score normalization
+- `"bayes"` | `"bb25"` for Bayesian normalization using dynamic candidate score statistics
+- `{method: "bayes", alpha: 1.0, beta: null}` for Bayesian normalization with optional custom parameters
+
+When standard normalization is enabled, statistics from the index are used to calculate normalized scores.
+When Bayesian/BB25 normalization is enabled, it uses positive-score candidates, dynamic `beta=median(scores)`, adaptive
+`alpha_eff=alpha/std(scores)` and a sigmoid transform (likelihood-only variant with flat prior) to map scores to `[0, 1]`.
+
+Bayesian normalization references:
+
+- [https://github.com/instructkr/bb25](https://github.com/instructkr/bb25)
+- [https://github.com/cognica-io/bayesian-bm25](https://github.com/cognica-io/bayesian-bm25)
+
+## tokenizer
+```yaml
+tokenizer: dict
+```
+
+Set tokenization rules. Passes these arguments to the underlying [Tokenization pipeline](../../../pipeline/data/tokenizer#txtai.pipeline.Tokenizer.__init__).

docs/embeddings/configuration/vectors.md

@@ -0,0 +1,200 @@
+# Vectors
+
+The following covers available vector model configuration options.
+
+## path
+```yaml
+path: string
+```
+
+Sets the path for a vectors model. When using a transformers/sentence-transformers model, this can be any model on the
+[Hugging Face Hub](https://huggingface.co/models) or a local file path. Otherwise, it must be a local file path to a word embeddings model.
+
+## method
+```yaml
+method: transformers|sentence-transformers|llama.cpp|litellm|model2vec|external|
+        words
+```
+
+Embeddings method to use. If the method is not provided, it is inferred using the `path`.
+
+`sentence-transformers`, `llama.cpp`, `litellm`, `model2vec` and `words` require the [vectors](../../../install/#vectors) extras package to be installed.
+
+### transformers
+
+Builds embeddings using a transformers model. While this can be any transformers model, it works best with
+[models trained](https://huggingface.co/models?pipeline_tag=sentence-similarity) to build embeddings.
+
+`mean`, `cls`, `last`, `late` and `max` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method from `transformers` to `meanpooling`, `clspooling`, `lastpooling`, `latepooling` or `maxpooling` respectively.
+
+Setting `maxlength` to `True` enables truncating inputs to the `max_seq_length`. Setting `maxlength` to an integer will truncate inputs to that value. When omitted (default), the `maxlength` will be set to either the model or tokenizer maxlength.
+
+Supports loading ONNX models directly. [See this section for more](../../../examples/#model-training).
+
+### sentence-transformers
+
+Same as transformers but loads models with the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) library.
+
+### llama.cpp
+
+Builds embeddings using a [llama.cpp](https://github.com/abetlen/llama-cpp-python) model. Supports both local and remote GGUF paths on the HF Hub.
+
+### litellm
+
+Builds embeddings using a LiteLLM model. See the [LiteLLM documentation](https://litellm.vercel.app/docs/providers) for the options available with LiteLLM models.
+
+### model2vec
+
+Builds embeddings using a [Model2Vec](https://github.com/MinishLab/model2vec) model. Model2Vec is a knowledge-distilled version of a transformers model with static vectors.
+
+### words
+
+Builds embeddings using a word embeddings model and static vectors. While Transformers models are preferred in most cases, this method can be useful for low resource and historical languages where there isn't much linguistic data available.
+
+#### pca
+```yaml
+pca: int
+```
+
+Removes _n_ principal components from generated embeddings. When enabled, a TruncatedSVD model is built to help with dimensionality reduction. After pooling of vectors creates a single embedding, this method is applied.
+
+### external
+
+Embeddings are created via an external model or API. Requires setting the [transform](#transform) parameter to a function that translates data into embeddings.
+
+#### transform
+```yaml
+transform: function
+```
+
+When method is `external`, this function transforms input content into embeddings. The input to this function is a list of data. This method must return either a numpy array, list of numpy arrays or list of float arrays.
+
+Note that this method is deprecated and will raise an error with the default settings. Adding a custom `Vectors` implementation is a cleaner approach. The `ALLOW_RESOLVE_TRANSFORM` environment variable must be set to enable. Only enable for trusted and/or reviewed code.
+
+## gpu
+```yaml
+gpu: boolean|int|string|device
+```
+
+Set the target device. Supports true/false, device id, device string and torch device instance. This is automatically derived if omitted.
+
+The `sentence-transformers` method supports encoding with multiple GPUs. This can be enabled by setting the gpu parameter to `all`.
+
+## batch
+```yaml
+batch: int
+```
+
+Sets the transform batch size. This parameter controls how input streams are chunked and vectorized.
+
+## encodebatch
+```yaml
+encodebatch: int
+```
+
+Sets the encode batch size. This parameter controls the underlying vector model batch size. This often corresponds to a GPU batch size, which controls GPU memory usage.
+
+## dimensionality
+```yaml
+dimensionality: int
+```
+
+Enables truncation of vectors to this dimensionality. This is only useful for models trained to store more important information in earlier dimensions such as [Matryoshka Representation Learning (MRL)](https://huggingface.co/blog/matryoshka).
+
+## quantize
+```yaml
+quantize: int|boolean
+```
+
+Enables scalar vector quantization at the specified precision. Supports 1-bit through 8-bit quantization. Scalar quantization transforms continuous floating point values to discrete unsigned integers. The `faiss`, `pgvector`, `numpy` and `torch` ANN backends support storing these vectors.
+
+This parameter supports booleans for backwards compatibility. When set to true/false, this flag sets [faiss.quantize](../ann/#faiss).
+
+In addition to vector-level quantization, some ANN backends have the ability to quantize vectors at the storage layer. See the [ANN](../ann) configuration options for more.
+
+## instructions
+```yaml
+instructions:
+    query: prefix for queries
+    data: prefix for indexing
+```
+
+Instruction-based models use prefixes to modify how embeddings are computed. This is especially useful with asymmetric search, which is when the query and indexed data are of vastly different lengths. In other words, short queries with long documents.
+
+`txtai` automatically loads prompts stored in `config_sentence_transformers.json` except if this parameter is set. For some older models such as [E5-base](https://huggingface.co/intfloat/e5-base), instructions still need to be provided via this parameter.
+
+## models
+```yaml
+models: dict
+```
+
+Loads and stores vector models in this cache. This is primarily used with subindexes but can be set on any embeddings instance. This prevents the same model from being loaded multiple times when working with multiple embeddings instances.
+
+## tokenize
+```yaml
+tokenize: boolean
+```
+
+Enables string tokenization (defaults to false). This method applies tokenization rules that only work with English language text. It's not recommended for use with recent vector models.
+
+## vectors
+```yaml
+vectors: dict
+```
+
+Passes these additional parameters to the underlying vector model.
+
+### center
+```yaml
+center: boolean | dict
+```
+
+Centers normalized late-interaction token vectors and normalizes them again before MUVERA or LEMUR encoding. Centering is applied to
+both query and document vectors. `true` uses batch scope and `false` disables centering. An explicit setting always takes precedence.
+When omitted, a LEMUR artifact with a stored collection mean uses collection scope. Other LEMUR artifacts and late-interaction models
+default to batch scope only when they load more than one `torch.nn.Linear` layer; models with zero or one loaded linear layer keep their
+previous output.
+
+Dictionary configuration supports the following scopes:
+
+```yaml
+center:
+    scope: document | batch | collection
+```
+
+`document` subtracts the mean of each document or query independently. `batch` subtracts the mean of all real token rows in the current
+model batch. Explicit `collection` scope requires a caller-provided, one-dimensional mean vector through either `mean` (an array-like
+value) or `path` (a safetensors file containing a `center.mean` tensor), but not both. LEMUR artifacts can provide this mean
+automatically. Zero-padding rows are never centered and remain zero.
+
+### muvera
+```yaml
+muvera:
+    repetitions: defaults 20
+    hashes: defaults to 5
+    projection: defaults 16
+```
+
+Settings to control the size of MUVERA fixed dimensional outputs. Default is 20 * 2^5 * 16 = 10,240 dimensions.
+Set `muvera` to `false` to disable it or `true` to use the default settings. `false` only makes sense when the multi-vector output
+is consumed directly, for example through `PoolingFactory`. Embeddings vectors require `muvera` or `lemur`.
+
+### lemur
+```yaml
+lemur:
+    path: local artifact directory or Hugging Face Hub path
+```
+
+Loads a trained [LEMUR](../../../pipeline/train/lemur) fixed dimensional encoder. LEMUR artifacts are corpus-specific and must be
+created with `LemurTrainer` before configuring an embeddings index. A bare string is treated as the artifact path. Query vectors are
+summed learned features and document vectors are ordinary least squares weights over the artifact's stored token sample. Each artifact
+contains `config.json` and `model.safetensors`; newly trained artifacts also store the collection token mean as `lemur.center`. Loading that mean automatically
+selects collection centering unless `center` is explicitly configured. Artifacts without `lemur.center` retain the previous default.
+The Safetensors file contains inference state only. See the trainer page for score filtering and Faiss exact/IVF search behavior.
+
+### trust_remote_code
+```yaml
+trust_remote_code: boolean
+```
+
+Parameter for trusting the code from Hugging Face models with custom implementations.

docs/embeddings/format.md

@@ -0,0 +1,59 @@
+# Index format
+
+![format](../images/format.png#only-light)
+![format](../images/format-dark.png#only-dark)
+
+This section documents the txtai index format. Each component is designed to ensure open access to the underlying data in a programmatic and platform independent way
+
+If an underlying library has an index format, that is used. Otherwise, txtai persists content with [MessagePack](https://msgpack.org/index.html) serialization.
+
+To learn more about how these components work together, read the [Index Guide](../indexing) and [Query Guide](../query).
+
+## ANN
+
+Approximate Nearest Neighbor (ANN) index configuration for storing vector embeddings.
+
+| Component                                                     | Storage Format                                                               |
+| ------------------------------------------------------------- | ---------------------------------------------------------------------------- |
+| [Faiss](https://github.com/facebookresearch/faiss)            | Local file format provided by library                                        |
+| [Hnswlib](https://github.com/nmslib/hnswlib)                  | Local file format provided by library                                        |
+| [Annoy](https://github.com/spotify/annoy)                     | Local file format provided by library                                        |
+| [NumPy](https://github.com/numpy/numpy)                       | Local NumPy array files via np.save / np.load                                |
+| [Postgres via pgvector](https://github.com/pgvector/pgvector) | Vector tables in a Postgres database                                         |
+
+## Core
+
+Core embeddings index files.
+
+| Component                                                     | Storage Format                                                               |
+| ------------------------------------------------------------- | ---------------------------------------------------------------------------- |
+| [Configuration](https://www.json.org/)                        | Embeddings index configuration stored as JSON                                |
+| [Index Ids](https://msgpack.org/index.html)                   | Embeddings index ids serialized with MessagePack. Only enabled when when content storage (database) is disabled. |
+
+## Database
+
+Databases store metadata, text and binary content.
+
+| Component                                                     | Storage Format                                                               |
+| ------------------------------------------------------------- | ---------------------------------------------------------------------------- |
+| [SQLite](https://www.sqlite.org/)                             | Local database files with SQLite                                             |
+| [DuckDB](https://github.com/duckdb/duckdb)                    | Local database files with DuckDB                                             |
+| [Postgres](https://www.postgresql.org/)                       | Postgres relational database via [SQLAlchemy](https://github.com/sqlalchemy/sqlalchemy). Supports additional databases via this library. |
+
+## Graph
+
+Graph nodes and edges for an embeddings index
+
+| Component                                                     | Storage Format                                                                |
+| ------------------------------------------------------------- | ----------------------------------------------------------------------------- |
+| [NetworkX](https://github.com/networkx/networkx)              | Nodes and edges exported to local file serialized with MessagePack            |
+| [Postgres](https://github.com/aplbrain/grand)                 | Nodes and edges stored in a Postgres database. Supports additional databases. |
+
+## Scoring
+
+Sparse/keyword indexing
+
+| Component                                                     | Storage Format                                                                |
+| ------------------------------------------------------------- | ----------------------------------------------------------------------------- |
+| [Local index](https://www.sqlite.org/)                        | Metadata serialized with MessagePack. Terms stored in SQLite.                 |
+| [Postgres](https://www.postgresql.org/docs/current/textsearch.html) | Text indexed with Postgres Full Text Search (FTS)                             |

docs/embeddings/index.md

@@ -3,15 +3,15 @@
 ![embeddings](../images/embeddings.png#only-light)
 ![embeddings](../images/embeddings-dark.png#only-dark)
 
-Embeddings is the engine that delivers semantic search. Data is transformed into embeddings vectors where similar concepts will produce similar vectors. Indexes both large and small are built with these vectors. The indexes are used to find results that have the same meaning, not necessarily the same keywords.
+Embeddings databases are the engine that delivers semantic search. Data is transformed into embeddings vectors where similar concepts will produce similar vectors. Indexes both large and small are built with these vectors. The indexes are used to find results that have the same meaning, not necessarily the same keywords.
 
 The following code snippet shows how to build and search an embeddings index.
 
 ```python
-from txtai.embeddings import Embeddings
+from txtai import Embeddings
 
 # Create embeddings model, backed by sentence-transformers & transformers
-embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2"})
+embeddings = Embeddings(path="sentence-transformers/nli-mpnet-base-v2")
 
 data = [
   "US tops 5 million confirmed virus cases",
@@ -24,10 +24,10 @@ data = [
   "Make huge profits without work, earn up to $100,000 a day"
 ]
 
-# Create an index for the list of text
-embeddings.index([(uid, text, None) for uid, text in enumerate(data)])
+# Index the list of text
+embeddings.index(data)
 
-print("%-20s %s" % ("Query", "Best Match"))
+print(f"{'Query':20} Best Match")
 print("-" * 50)
 
 # Run an embeddings search for each query
@@ -38,28 +38,32 @@ for query in ("feel good story", "climate change", "public health story", "war",
     uid = embeddings.search(query, 1)[0][0]
 
     # Print text
-    print("%-20s %s" % (query, data[uid]))
+    print(f"{query:20} {data[uid]}")
 ```
 
 ## Build
 
-An embeddings instance can be created as follows:
+An embeddings instance is [configuration-driven](configuration) based on what is passed in the constructor. Vectors are stored with the option to also [store content](configuration/database#content). Content storage enables additional filtering and data retrieval options.
+
+The example above sets a specific embeddings vector model via the [path](configuration/vectors/#path) parameter. An embeddings instance with no configuration can also be created.
 
 ```python
-embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2"})
+embeddings = Embeddings()
 ```
 
-The example above builds a transformers based embeddings instance. In this case, when loading and searching for data, a transformers model is used to vectorize data. The embeddings instance is [configuration driven](configuration) based on what is passed in the constructor. A number of different options are supported to cover a wide variety of use cases.
+In this case, when loading and searching for data, the [default transformers vector model](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) is used to vectorize data. See the [model guide](../models) for current model recommentations.
 
 ## Index
 
-After creating a new Embeddings instance, the next step is adding data to it. 
+After creating a new embeddings instance, the next step is adding data to it.
 
 ```python
-embeddings.index([(uid, text, None) for uid, text in enumerate(data)])
+embeddings.index(rows)
 ```
 
-The index method takes an iterable collection of tuples with three values. 
+The index method takes an iterable and supports the following formats for each element.
+
+- `(id, data, tags)` - default processing format
 
 | Element     | Description                                                   |
 | ----------- | ------------------------------------------------------------- |
@@ -67,9 +71,19 @@ The index method takes an iterable collection of tuples with three values.
 | data        | input data to index, can be text, a dictionary or object      |
 | tags        | optional tags string, used to mark/label data as it's indexed |
 
-The input iterable can be a list and/or a generator. [Generators](https://wiki.python.org/moin/Generators) help with indexing very large datasets as only portions of the data is in memory at any given time.
+- `(id, data)`
 
-txtai buffers data to temporary storage along the way during indexing as embeddings vectors can be quite large (for example 768 dimensions of float32 is 768 * 4 = 3072 bytes per vector).
+  Same as above but without tags.
+
+- `data`
+
+Single element to index. In this case, unique id's will automatically be generated. Note that for generated id's, [upsert](methods/#txtai.embeddings.base.Embeddings.upsert) and [delete](methods/#txtai.embeddings.base.Embeddings.delete) calls require a separate search to get the target ids.
+
+When the data field is a dictionary, text is passed via the `text` key, binary objects via the `object` key. Note that [content](configuration/database#content) must be enabled to store metadata and [objects](configuration/database#objects) to store binary object data. The `id` and `tags` keys will be extracted, if provided.
+
+The input iterable can be a list or generator. [Generators](https://wiki.python.org/moin/Generators) help with indexing very large datasets as only portions of the data is in memory at any given time.
+
+More information on indexing can be found in the [index guide](indexing).
 
 ## Search
 
@@ -79,10 +93,29 @@ Once data is indexed, it is ready for search.
 embeddings.search(query, limit)
 ```
 
-The search method takes two parameters, the query and query limit. The results format is different based on whether [content](configuration/#content) is stored or not.
+The search method takes two parameters, the query and query limit. The results format is different based on whether [content](configuration/database#content) is stored or not.
 
-- List of `(id, score)` when content is disabled
-- List of `{**query columns}` when content is enabled
+- List of `(id, score)` when content is _not_ stored
+- List of `{**query columns}` when content is stored
+
+Both natural language and SQL queries are supported. More information can be found in the [query guide](query).
+
+## Resource management
+
+Embeddings databases are context managers. The following blocks automatically [close](methods/#txtai.embeddings.base.Embeddings.close) and free resources upon completion.
+
+```python
+# Create a new Embeddings database, index data and save
+with Embeddings() as embeddings:
+  embeddings.index(rows)
+  embeddings.save(path)
+
+# Search a saved Embeddings database
+with Embeddings().load(path) as embeddings:
+  embeddings.search(query)
+```
+
+While calling `close` isn't always necessary (resources will be garbage collected), it's best to free shared resources like database connections as soon as they aren't needed.
 
 ## More examples
 

docs/embeddings/indexing.md

@@ -0,0 +1,136 @@
+# Index guide
+
+![indexing](../images/indexing.png#only-light)
+![indexing](../images/indexing-dark.png#only-dark)
+
+This section gives an in-depth overview on how to index data with txtai. We'll cover vectorization, indexing/updating/deleting data and the various components of an embeddings database.
+
+## Vectorization
+
+The most compute intensive step in building an index is vectorization. The [path](../configuration/vectors#path) parameter sets the path to the vector model. There is logic to automatically detect the vector model [method](../configuration/vectors#method) but it can also be set directly.
+
+The [batch](../configuration/vectors#batch) and [encodebatch](../configuration/vectors#encodebatch) parameters control the vectorization process. Larger values for `batch` will pass larger batches to the vectorization method. Larger values for `encodebatch` will pass larger batches for each vector encode call. In the case of GPU vector models, larger values will consume more GPU memory.
+
+Data is buffered to temporary storage during indexing as embeddings vectors can be quite large (for example 768 dimensions of float32 is 768 * 4 = 3072 bytes per vector). Once vectorization is complete, a mmapped array is created with all vectors for [Approximate Nearest Neighbor (ANN)](../configuration/vectors#backend) indexing.
+
+The terms `ANN` and `dense vector index` are used interchangeably throughout txtai's documentation.
+
+## Setting a backend
+
+As mentioned above, computed vectors are stored in an ANN. There are various index [backends](../configuration/ann#backend) that can be configured. Faiss is the default backend.
+
+## Content storage
+
+Embeddings indexes can optionally [store content](../configuration/database#content). When this is enabled, the input content is saved in a database alongside the computed vectors. This enables filtering on additional fields and content retrieval.
+
+The columns used for text, object and JSON data storage are set via [column configuration](../configuration/general#columns).
+
+## Index vs Upsert
+
+Data is loaded into an index with either an [index](../methods#txtai.embeddings.base.Embeddings.index) or [upsert](../methods#txtai.embeddings.base.Embeddings.upsert) call.
+
+```python
+embeddings.index([(uid, text, None) for uid, text in enumerate(data)])
+embeddings.upsert([(uid, text, None) for uid, text in enumerate(data)])
+```
+
+The `index` call will build a brand new index replacing an existing one. `upsert` will insert or update records. `upsert` ops do _not_ require a full index rebuild.
+
+## Save
+
+Indexes can be stored in a directory using the [save](../methods/#txtai.embeddings.base.Embeddings.save) method.
+
+```python
+embeddings.save("/path/to/save")
+```
+
+Compressed indexes are also supported.
+
+```python
+embeddings.save("/path/to/save/index.tar.gz")
+```
+
+In addition to saving indexes locally, they can also be persisted to [cloud storage](../configuration/cloud).
+
+```python
+embeddings.save("/path/to/save/index.tar.gz", cloud={...})
+```
+
+This is especially useful when running in a serverless context or otherwise running on temporary compute. Cloud storage is only supported with compressed indexes.
+
+Embeddings indexes can be restored using the [load](../methods/#txtai.embeddings.base.Embeddings.load) method.
+
+```python
+embeddings.load("/path/to/load")
+```
+
+## Delete
+
+Content can be removed from the index with the [delete](../methods#txtai.embeddings.base.Embeddings.delete) method. This method takes a list of ids to delete.
+
+```python
+embeddings.delete(ids)
+```
+
+## Reindex
+
+When [content storage](../configuration/database#content) is enabled, [reindex](../methods#txtai.embeddings.base.Embeddings.reindex) can be called to rebuild the index with new settings. For example, the backend can be switched from faiss to hnsw or the vector model can be updated. This prevents having to go back to the original raw data. 
+
+```python
+embeddings.reindex(path="sentence-transformers/all-MiniLM-L6-v2", backend="hnsw")
+```
+
+## Graph
+
+Enabling a [graph network](../configuration/graph) adds a semantic graph at index time as data is being vectorized. Vector embeddings are used to automatically create relationships in the graph. Relationships can also be manually specified at index time.
+
+```python
+# Manual relationships by id
+embeddings.index([{"id": "0", "text": "...", "relationships": ["2"]}])
+
+# Manual relationships with additional edge attributes
+embeddings.index(["id": "0", "text": "...", "relationships": [
+    {"id": "2", "type": "MEMBER_OF"}
+]])
+```
+
+Additionally, graphs can be used for topic modeling. Dimensionality reduction with UMAP combined with HDBSCAN is a popular topic modeling method found in a number of libraries. txtai takes a different approach using community detection algorithms to build topic clusters.
+
+This approach has the advantage of only having to vectorize data once. It also has the advantage of better topic precision given there isn't a dimensionality reduction operation (UMAP). Semantic graph examples are shown below.
+
+Get a mapping of discovered topics to associated ids.
+
+```python
+embeddings.graph.topics
+```
+
+Show the most central nodes in the index.
+
+```python
+embeddings.graph.centrality()
+```
+
+Graphs are persisted alongside an embeddings index. Each save and load will also save and load the graph.
+
+## Sparse vectors
+
+Scoring instances can create a standalone [sparse keyword indexes](../configuration/general#keyword) (BM25, TF-IDF) and [sparse vector indexes](../configuration/general#sparse) (SPLADE). This enables [hybrid](../configuration/general/#hybrid) search when there is an available dense vector index.
+
+The terms `sparse vector index`, `keyword index`, `terms index` and `scoring index` are used interchangeably throughout txtai's documentation.
+
+See [this link](../../examples/#semantic-search) to learn more.
+
+## Subindexes
+
+An embeddings instance can optionally have associated [subindexes](../configuration/general/#indexes), which are also embeddings databases. This enables indexing additional fields, vector models and much more.
+
+## Word vectors
+
+When using [word vector backed models](../configuration/vectors#words) with scoring set, a separate call is required before calling `index` as follows:
+
+```python
+embeddings.score(rows)
+embeddings.index(rows)
+```
+
+Both calls are required to support generator-backed iteration of data with word vectors models.

docs/embeddings/methods.md

@@ -1,10 +1,20 @@
 # Methods
 
 ::: txtai.embeddings.Embeddings
-    selection:
+    options:
         filters:
+            - "!columns"
+            - "!createann"
+            - "!createcloud"
+            - "!createdatabase"
+            - "!creategraph"
+            - "!createids"
+            - "!createindexes"
+            - "!createscoring"
             - "!checkarchive"
             - "!configure"
-            - "!createdatabase"
+            - "!defaultallowed"
+            - "!defaults"
+            - "!initindex"
+            - "!loadquery"
             - "!loadvectors"
-            - "!normalize"

docs/embeddings/query.md

@@ -1,5 +1,8 @@
 # Query guide
 
+![query](../images/query.png#only-light)
+![query](../images/query-dark.png#only-dark)
+
 This section covers how to query data with txtai. The simplest way to search for data is building a natural language string with the desired content to find. txtai also supports querying with SQL. We'll cover both methods here.
 
 ## Natural language queries
@@ -11,11 +14,11 @@ embeddings.search("feel good story")
 embeddings.search("wildlife")
 ```
 
-The queries above search the index for similarity matches on `feel good story` and `wildlife`. If content storage is enabled, a list of `{**query columns}` is returned. Otherwise, a list of `(id, score)` tuples are returned.
+The queries above [search](../methods#txtai.embeddings.base.Embeddings.search) the index for similarity matches on `feel good story` and `wildlife`. If content storage is enabled, a list of `{**query columns}` is returned. Otherwise, a list of `(id, score)` tuples are returned.
 
 ## SQL
 
-txtai supports more complex queries with SQL. This is only supported if [content storage](../configuration/#content) is enabled. txtai has a translation layer that analyzes input SQL statements and combines similarity results with content stored in a relational database.
+txtai supports more complex queries with SQL. This is only supported if [content storage](../configuration/database#content) is enabled. txtai has a translation layer that analyzes input SQL statements and combines similarity results with content stored in a relational database.
 
 SQL queries are run through `embeddings.search` like natural language queries but the examples below only show the SQL query for conciseness.
 
@@ -29,40 +32,52 @@ The similar clause is a txtai function that enables similarity searches with SQL
 
 ```sql
 SELECT id, text, score FROM txtai WHERE similar('feel good story')
-SELECT id, text, score FROM txtai WHERE similar('feel good story')
 ```
 
-The similar clause takes two arguments:
+The similar clause takes the following arguments:
 
 ```sql
-similar("query", "number of candidates")
+similar("query", "number of candidates", "index", "weights")
 ```
 
 | Argument              | Description                            |
 | --------------------- | ---------------------------------------|
 | query                 | natural language query to run          |
 | number of candidates  | number of candidate results to return  |
+| index                 | target index name                      |
+| weights               | hybrid score weights                   |
 
-The txtai query layer has to join results from two separate components, a relational store and a similarity index. With a similar clause, a similarity search is run and those ids are fed to the underlying database query.
+The txtai query layer joins results from two separate components, a relational store and a similarity index. With a similar clause, a similarity search is run and those ids are fed to the underlying database query.
 
-The number of candidates should be larger than the desired number of results as additional filtering could occur. If this is not specified, it defaults to 10 times the number of desired results. 
+The number of candidates should be larger than the desired number of results when applying additional filter clauses. This ensures that `limit` results are still returned after applying additional filters. If the number of candidates is not specified, it is defaulted as follows:
+
+- For a single query filter clause, the default is the query limit
+- With multiple filtering clauses, the default is 10x the query limit
+
+When a query orders results by score, its `offset` adds to the query limit.
+
+The index name is only applicable when [subindexes](../configuration/general/#indexes) are enabled. This specifies the index to use for the query.
+
+Weights sets the hybrid score weights when an index has both a sparse and dense index.
 
 ### Dynamic columns
 
-Content can be indexed in multiple ways when content storage is enabled. [Remember that input documents](../#index) take the form of `(id, data, tags)` tuples. If data is a string, then content is primarily filtered with similar clauses. If data is a dictionary, then all fields in the dictionary are indexed and searchable.
+Content can be indexed in multiple ways when content storage is enabled. [Remember that input documents](../#index) take the form of `(id, data, tags)` tuples. If data is a string or binary content, it's indexed and searchable with `similar()` clauses.
+
+If data is a dictionary, then all fields in the dictionary are stored and available via SQL. The `text` field or [field specified in the index configuration](../configuration/general/#columns) is indexed and searchable with `similar()` clauses.
 
 For example:
 
 ```python
-embeddings.index([(0, {"text": "text to index", "flag": True,
-                       "entry": "2022-01-01"}, None)])
+embeddings.index([{"text": "text to index", "flag": True,
+                   "actiondate": "2022-01-01"}])
 ```
 
 With the above input data, queries can now have more complex filters.
 
 ```sql
-SELECT text, flag, entry FROM txtai WHERE similar('query') AND flag = 1
-AND entry >= '2022-01-01'
+SELECT text, flag, actiondate FROM txtai WHERE similar('query') AND flag = 1
+AND actiondate >= '2022-01-01'
 ```
 
 txtai's query layer automatically detects columns and translates queries into a format that can be understood by the underlying database.
@@ -70,16 +85,30 @@ txtai's query layer automatically detects columns and translates queries into a
 Nested dictionaries/JSON is supported and can be escaped with bracket statements.
 
 ```python
-embeddings.index([(0, {"text": "text to index",
-                       "parent": {"child element": "abc"}}, None)])
+embeddings.index([{"text": "text to index",
+                   "parent": {"child element": "abc"}}])
 ```
 
 ```sql
-SELECT text FROM txtai WHERE [parent.child element] ='abc'
+SELECT text FROM txtai WHERE [parent.child element] = 'abc'
 ```
 
 Note the bracket statement escaping the nested column with spaces in the name.
 
+### Bind parameters
+
+txtai has support for SQL bind parameters.
+
+```python
+# Query with a bind parameter for similar clause
+query = "SELECT id, text, score FROM txtai WHERE similar(:x)"
+results = embeddings.search(query, parameters={"x": "feel good story"})
+
+# Query with a bind parameter for column filter
+query = "SELECT text, flag, actiondate FROM txtai WHERE flag = :x"
+results = embeddings.search(query, parameters={"x": 1})
+```
+
 ### Aggregation queries
 
 The goal of txtai's query language is to closely support all functions in the underlying database engine. The main challenge is ensuring dynamic columns are properly escaped into the engines native query function. 
@@ -98,22 +127,161 @@ SELECT count(*), flag FROM txtai GROUP BY flag ORDER BY count(*) DESC
 txtai has support for storing and retrieving binary objects. Binary objects can be retrieved as shown in the example below.
 
 ```python
+# Create embeddings index with content and object storage enabled
+embeddings = Embeddings(content=True, objects=True)
+
 # Get an image
-request = open("demo.gif")
+request = open("demo.gif", "rb")
 
 # Insert record
-embeddings.index([("txtai", {"text": "txtai executes machine-learning workflows.",
-                             "object": request.read()}, None)])
+embeddings.index([(
+  "txtai",
+  {"text": "txtai executes machine-learning workflows.",
+   "object": request.read()}
+)])
 
 # Query txtai and get associated object
-query = "select object from txtai where similar('machine learning') limit 1"
+query = "SELECT object FROM txtai WHERE similar('machine learning') LIMIT 1"
 result = embeddings.search(query)[0]["object"]
+
+# Query binary content with a bind parameter
+query = "SELECT object FROM txtai WHERE similar(:x) LIMIT 1"
+results = embeddings.search(query, parameters={"x": request.read()})
+```
+
+## Custom SQL functions
+
+Custom, user-defined SQL functions extend selection, filtering and ordering clauses with additional logic. For example, the following snippet defines a function that translates text using a translation pipeline.
+
+```python
+# Translation pipeline
+translate = Translation()
+
+# Create embeddings index
+embeddings = Embeddings(path="sentence-transformers/nli-mpnet-base-v2",
+                        content=True,
+                        functions=[translate]})
+
+# Run a search using a custom SQL function
+embeddings.search("""
+SELECT
+  text,
+  translation(text, 'de', null) 'text (DE)',
+  translation(text, 'es', null) 'text (ES)',
+  translation(text, 'fr', null) 'text (FR)'
+FROM txtai WHERE similar('feel good story')
+LIMIT 1
+""")
+```
+
+## Expressions
+
+Expression shortcuts expand into more complex SQL snippets. This is useful for making SQL queries more concise. Indexing is also available on expressions as a performance improvement.
+
+The following example indexes a json extraction field (`filepath`) and the length of each field.
+
+```python
+# Create embeddings index
+embeddings = Embeddings(
+  path="sentence-transformers/nli-mpnet-base-v2",
+  content=True,
+  expressions=[
+    {"name": "filepath", "index": True},
+    {"name": "textlength", "expression": "length(text)", "index": True}
+  ]
+)
+
+embeddings.search("SELECT textlength, filepath FROM txtai LIMIT 1")
+```
+
+## Query translation
+
+Natural language queries with filters can be converted to txtai-compatible SQL statements with query translation. For example:
+
+```python
+embeddings.search("feel good story since yesterday")
+```
+
+can be converted to a SQL statement with a similar clause and date filter.
+
+```sql
+select id, text, score from txtai where similar('feel good story') and
+entry >= date('now', '-1 day')
+```
+
+This requires setting a [query translation model](../configuration/database#query). The default query translation model is [t5-small-txtsql](https://huggingface.co/NeuML/t5-small-txtsql) but this can easily be finetuned to handle different use cases.
+
+## Hybrid search
+
+When an embeddings database has both a sparse and dense index, both indexes will be queried and the results will be equally weighted unless otherwise specified.
+
+```python
+embeddings.search("query", weights=0.5)
+embeddings.search(
+  "SELECT id, text, score FROM txtai WHERE similar('query', 0.5)"
+)
+```
+
+## Graph search
+
+If an embeddings database has an associated graph network, graph searches can be run. The search syntax below uses [openCypher](https://github.com/opencypher/openCypher). Follow the preceding link to learn more about this syntax.
+
+Additionally, standard embeddings searches can be returned as graphs.
+
+```python
+# Find all paths between id: 0 and id: 5 between 1 and 3 hops away
+embeddings.search("""
+MATCH P=({id: 0})-[*1..3]->({id: 5})
+RETURN P
+""")
+
+# Find related nodes for query matches
+embeddings.search("""
+  MATCH P=(A)-[]->(B)
+  WHERE SIMILAR(A, "query")
+  RETURN B
+  ORDER BY A.score DESC
+  LIMIT 10
+""")
+
+# Standard embeddings search as graph
+embeddings.search("query", graph=True)
+```
+
+## Subindexes
+
+Subindexes can be queried as follows:
+
+```python
+# Build index with subindexes
+embeddings = Embeddings(
+  content=True,
+  defaults=False,
+  indexes={
+    "keyword": {
+      "keyword": True
+    },
+    "dense":{
+      "dense": True
+    }
+  }
+)
+embeddings.index(stream())
+
+# Query with index parameter
+embeddings.search("query", index="keyword")
+
+# Specify with SQL
+embeddings.search("""
+SELECT id, text, score FROM txtai
+WHERE similar('query', 'keyword')
+""")
 ```
 
 ## Combined index architecture
 
-When content storage is enabled, txtai becomes a dual storage engine. Content is stored in an underlying database (currently supports SQLite) along with an Approximate Nearest Neighbor (ANN) index. These components combine to deliver similarity search alongside traditional structured search.
+txtai has multiple storage and indexing components. Content is stored in an underlying database along with an approximate nearest neighbor (ANN) index, keyword index and graph network. These components combine to deliver similarity search alongside traditional structured search.
 
-The ANN index stores ids and vectors for each input element. When a natural language query is run, the query is translated into a vector and a similarity query finds the best matching ids. When a database is added into the mix, an additional step is applied. This step takes those ids and effectively inserts them as part of the underlying database query.
+The ANN index stores ids and vectors for each input element. When a natural language query is run, the query is translated into a vector and a similarity query finds the best matching ids. When a database is added into the mix, an additional step is executed. This step takes those ids and effectively inserts them as part of the underlying database query. The same steps apply with keyword indexes except a term frequency index is used to find the best matching ids.
 
-Dynamic columns are supported via the underlying engine. For SQLite, data is stored as JSON and dynamic columns are converted into `json_extract` clauses. This same concept can be expanded to other storage engines like PostgreSQL and could even work with NoSQL stores. 
+Dynamic columns are supported via the underlying engine. For SQLite, data is stored as JSON and dynamic columns are converted into `json_extract` clauses. Client-server databases are supported via [SQLAlchemy](https://docs.sqlalchemy.org/en/20/dialects/) and dynamic columns are supported provided the underlying engine has [JSON](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.JSON) support.

docs/examples.md

@@ -3,7 +3,7 @@
 ![examples](images/examples.png#only-light)
 ![examples](images/examples-dark.png#only-dark)
 
-The examples directory has a series of notebooks and applications giving an overview of txtai. See the sections below.
+See below for a comprehensive series of example notebooks and applications covering txtai.
 
 ## Semantic Search
 
@@ -11,17 +11,52 @@ Build semantic/similarity/vector/neural search applications.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
-| [Introducing txtai](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) | Overview of the functionality provided by txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |
+| [Introducing txtai](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=SIezMnVdmMs) | Overview of the functionality provided by txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |
 | [Build an Embeddings index with Hugging Face Datasets](https://github.com/neuml/txtai/blob/master/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb) | Index and search Hugging Face Datasets | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb) |
 | [Build an Embeddings index from a data source](https://github.com/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb)  | Index and search a data source with word embeddings | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb) |
 | [Add semantic search to Elasticsearch](https://github.com/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb)  | Add semantic search to existing search systems | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb) |
 | [Similarity search with images](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) | Embed images and text into the same space for search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |
-| [Distributed embeddings cluster](https://github.com/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) | Distribute an embeddings index across multiple data nodes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) |
-| [What's new in txtai 4.0](https://github.com/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) | Content storage, SQL, object storage, reindex and compressed indexes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) |
+| [Custom Embeddings SQL functions](https://github.com/neuml/txtai/blob/master/examples/30_Embeddings_SQL_custom_functions.ipynb) | Add user-defined functions to Embeddings SQL | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/30_Embeddings_SQL_custom_functions.ipynb) |
+| [Model explainability](https://github.com/neuml/txtai/blob/master/examples/32_Model_explainability.ipynb) | Explainability for semantic search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/32_Model_explainability.ipynb) |
+| [Query translation](https://github.com/neuml/txtai/blob/master/examples/33_Query_translation.ipynb) | Domain-specific natural language queries with query translation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/33_Query_translation.ipynb) |
+| [Build a QA database](https://github.com/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) | Question matching with semantic search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) |
+| [Semantic Graphs](https://github.com/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) | Explore topics, data connectivity and run network analysis| [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) |
+| [Topic Modeling with BM25](https://github.com/neuml/txtai/blob/master/examples/39_Classic_Topic_Modeling_with_BM25.ipynb) | Topic modeling backed by a BM25 index | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/39_Classic_Topic_Modeling_with_BM25.ipynb) |
+
+## LLM
+
+Autonomous agents, retrieval augmented generation (RAG), chat with your data, pipelines and workflows that interface with large language models (LLMs).
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Prompt-driven search with LLMs](https://github.com/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) | Embeddings-guided and Prompt-driven search with Large Language Models (LLMs) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) |
+| [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=t_OeAc8NVfQ) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
+| [Generate knowledge with Semantic Graphs and RAG](https://github.com/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) | Knowledge exploration and discovery with Semantic Graphs and RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) |
+| [Build knowledge graphs with LLMs](https://github.com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) | Build knowledge graphs with LLM-driven entity extraction | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |
+| [Advanced RAG with graph path traversal](https://github.com/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) | Graph path traversal to collect complex sets of data for advanced RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) |
+| [Advanced RAG with guided generation](https://github.com/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) | Retrieval Augmented and Guided Generation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) |
+| [RAG with llama.cpp and external API services](https://github.com/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) | RAG with additional vector and LLM frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) |
+| [How RAG with txtai works](https://github.com/neuml/txtai/blob/master/examples/63_How_RAG_with_txtai_works.ipynb) | Create RAG processes, API services and Docker instances | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/63_How_RAG_with_txtai_works.ipynb) |
+| [Speech to Speech RAG](https://github.com/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) [▶️](https://www.youtube.com/watch?v=tH8QWwkVMKA) | Full cycle speech to speech workflow with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/65_Speech_to_Speech_RAG.ipynb) |
+| [Analyzing Hugging Face Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) | Explore a rich dataset with Graph Analysis and Agents | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/68_Analyzing_Hugging_Face_Posts_with_Graphs_and_Agents.ipynb) |
+| [Granting autonomy to agents](https://github.com/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) | Agents that iteratively solve problems as they see fit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/69_Granting_autonomy_to_agents.ipynb) |
+| [Getting started with LLM APIs](https://github.com/neuml/txtai/blob/master/examples/70_Getting_started_with_LLM_APIs.ipynb) | Generate embeddings and run LLMs with OpenAI, Claude, Gemini, Bedrock and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/70_Getting_started_with_LLM_APIs.ipynb) |
+| [Analyzing LinkedIn Company Posts with Graphs and Agents](https://github.com/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) | Exploring how to improve social media engagement with AI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/71_Analyzing_LinkedIn_Company_Posts_with_Graphs_and_Agents.ipynb) |
+| [Parsing the stars with txtai](https://github.com/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) | Explore an astronomical knowledge graph of known stars, planets, galaxies | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/72_Parsing_the_stars_with_txtai.ipynb) |
+| [Chunking your data for RAG](https://github.com/neuml/txtai/blob/master/examples/73_Chunking_your_data_for_RAG.ipynb) | Extract, chunk and index content for effective retrieval | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/73_Chunking_your_data_for_RAG.ipynb) |
+| [Medical RAG Research with txtai](https://github.com/neuml/txtai/blob/master/examples/75_Medical_RAG_Research_with_txtai.ipynb) | Analyze PubMed article metadata with RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/75_Medical_RAG_Research_with_txtai.ipynb) |
+| [GraphRAG with Wikipedia and GPT OSS](https://github.com/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) | Deep graph search powered RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/77_GraphRAG_with_Wikipedia_and_GPT_OSS.ipynb) |
+| [RAG is more than Vector Search](https://github.com/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) | Context retrieval via Web, SQL and other sources | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/79_RAG_is_more_than_Vector_Search.ipynb) |
+| [OpenCode as a txtai LLM](https://github.com/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) | Integrate OpenCode with the txtai ecosystem | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/81_OpenCode_as_a_txtai_LLM.ipynb) |
+| [Agentic College Search](https://github.com/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) | Identify a list of strong engineering colleges | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/82_Agentic_College_Search.ipynb) |
+| [TxtAI got skills](https://github.com/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) | Integrate skill.md files with your agent | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/83_TxtAI_got_skills.ipynb) |
+| [Agent Tools](https://github.com/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) [▶️](https://www.youtube.com/watch?v=RDNaFXQy3GQ) | Learn about the txtai agent toolkit | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/84_Agent_Tools.ipynb) |
 
 ## Pipelines
 
-Transform data with NLP-backed pipelines.
+Transform data with language model backed pipelines.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
@@ -31,10 +66,11 @@ Transform data with NLP-backed pipelines.
 | [Apply labels with zero shot classification](https://github.com/neuml/txtai/blob/master/examples/07_Apply_labels_with_zero_shot_classification.ipynb) | Use zero shot learning for labeling, classification and topic modeling | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/07_Apply_labels_with_zero_shot_classification.ipynb) |
 | [Building abstractive text summaries](https://github.com/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) | Run abstractive text summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) |
 | [Extract text from documents](https://github.com/neuml/txtai/blob/master/examples/10_Extract_text_from_documents.ipynb) | Extract text from PDF, Office, HTML and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/10_Extract_text_from_documents.ipynb) |
+| [Text to speech generation](https://github.com/neuml/txtai/blob/master/examples/40_Text_to_Speech_Generation.ipynb) | Generate speech from text | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/40_Text_to_Speech_Generation.ipynb) |
 | [Transcribe audio to text](https://github.com/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) | Convert audio files to text | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) |
 | [Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) | Streamline machine translation and language detection | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |
 | [Generate image captions and detect objects](https://github.com/neuml/txtai/blob/master/examples/25_Generate_image_captions_and_detect_objects.ipynb) | Captions and object detection for images | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/25_Generate_image_captions_and_detect_objects.ipynb) |
-| [API Gallery](https://github.com/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) | Using txtai in JavaScript, Java, Rust and Go | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) |
+| [Near duplicate image detection](https://github.com/neuml/txtai/blob/master/examples/31_Near_duplicate_image_detection.ipynb) | Identify duplicate and near-duplicate images | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/31_Near_duplicate_image_detection.ipynb) |
 
 ## Workflows
 
@@ -42,34 +78,86 @@ Efficiently process data at scale.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
-| [Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) | Simple yet powerful constructs to efficiently process data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
+| [Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) [▶️](https://www.youtube.com/watch?v=UBMPDCn1gEU) | Simple yet powerful constructs to efficiently process data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
 | [Transform tabular data with composable workflows](https://github.com/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) | Transform, index and search tabular data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) |
 | [Tensor workflows](https://github.com/neuml/txtai/blob/master/examples/23_Tensor_workflows.ipynb) | Performant processing of large tensor arrays | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/23_Tensor_workflows.ipynb) |
 | [Entity extraction workflows](https://github.com/neuml/txtai/blob/master/examples/26_Entity_extraction_workflows.ipynb) | Identify entity/label combinations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/26_Entity_extraction_workflows.ipynb) |
 | [Workflow Scheduling](https://github.com/neuml/txtai/blob/master/examples/27_Workflow_scheduling.ipynb) | Schedule workflows with cron expressions | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/27_Workflow_scheduling.ipynb) |
 | [Push notifications with workflows](https://github.com/neuml/txtai/blob/master/examples/28_Push_notifications_with_workflows.ipynb) | Generate and push notifications with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/28_Push_notifications_with_workflows.ipynb) |
+| [Pictures are a worth a thousand words](https://github.com/neuml/txtai/blob/master/examples/35_Pictures_are_worth_a_thousand_words.ipynb) | Generate webpage summary images with DALL-E mini | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/35_Pictures_are_worth_a_thousand_words.ipynb) |
+| [Run txtai with native code](https://github.com/neuml/txtai/blob/master/examples/36_Run_txtai_in_native_code.ipynb) | Execute workflows in native code with the Python C API | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/36_Run_txtai_in_native_code.ipynb) |
+| [Generative Audio](https://github.com/neuml/txtai/blob/master/examples/66_Generative_Audio.ipynb) | Storytelling with generative audio workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/66_Generative_Audio.ipynb) |
 
 ## Model Training
 
-Train NLP models.
+Train, distill, fine-tune and export models.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
 | [Train a text labeler](https://github.com/neuml/txtai/blob/master/examples/16_Train_a_text_labeler.ipynb) | Build text sequence classification models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/16_Train_a_text_labeler.ipynb) |
 | [Train without labels](https://github.com/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) | Use zero-shot classifiers to train new models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) |
 | [Train a QA model](https://github.com/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) | Build and fine-tune question-answering models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) |
+| [Train a language model from scratch](https://github.com/neuml/txtai/blob/master/examples/41_Train_a_language_model_from_scratch.ipynb) | Build new language models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/41_Train_a_language_model_from_scratch.ipynb) |
+| [Distilling Knowledge into Tiny LLMs](https://github.com/neuml/txtai/blob/master/examples/80_Distilling_Knowledge_into_Tiny_LLMs.ipynb) [▶️](https://www.youtube.com/watch?v=Ol560ktgkf0) | Finetune tiny LLMs to enable inference using less resources | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/80_Distilling_Knowledge_into_Tiny_LLMs.ipynb) |
+| [Progressive Distillation](https://github.com/neuml/txtai/blob/master/examples/85_Progressive_Distillation.ipynb) | Incrementally compress knowledge from a series of teachers | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/85_Progressive_Distillation.ipynb) |
 | [Export and run models with ONNX](https://github.com/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) | Export models with ONNX, run natively in JavaScript, Java and Rust | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) |
 | [Export and run other machine learning models](https://github.com/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) | Export and run models from scikit-learn, PyTorch and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) |
 
+## API
+
+Run distributed txtai, integrate with the API and cloud endpoints.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [API Gallery](https://github.com/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) | Using txtai in JavaScript, Java, Rust and Go | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) |
+| [Distributed embeddings cluster](https://github.com/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) | Distribute an embeddings index across multiple data nodes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) |
+| [Embeddings in the Cloud](https://github.com/neuml/txtai/blob/master/examples/43_Embeddings_in_the_Cloud.ipynb) | Load and use an embeddings index from the Hugging Face Hub | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/43_Embeddings_in_the_Cloud.ipynb) |
+| [Custom API Endpoints](https://github.com/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) | Extend the API with custom endpoints | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) |
+| [API Authorization and Authentication](https://github.com/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) | Add authorization, authentication and middleware dependencies to the API | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) |
+| [OpenAI Compatible API](https://github.com/neuml/txtai/blob/master/examples/74_OpenAI_Compatible_API.ipynb) | Connect to txtai with a standard OpenAI client library | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/74_OpenAI_Compatible_API.ipynb) |
+
+## Architecture
+
+Project architecture, data formats, external integrations, scale to production, benchmarks, and performance.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Anatomy of a txtai index](https://github.com/neuml/txtai/blob/master/examples/29_Anatomy_of_a_txtai_index.ipynb) | Deep dive into the file formats behind a txtai embeddings index | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/29_Anatomy_of_a_txtai_index.ipynb) |
+| [Embeddings components](https://github.com/neuml/txtai/blob/master/examples/37_Embeddings_index_components.ipynb) | Composable search with vector, SQL and scoring components | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/37_Embeddings_index_components.ipynb) |
+| [Customize your own embeddings database](https://github.com/neuml/txtai/blob/master/examples/45_Customize_your_own_embeddings_database.ipynb) | Ways to combine vector indexes with relational databases | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/45_Customize_your_own_embeddings_database.ipynb) |
+| [Building an efficient sparse keyword index in Python](https://github.com/neuml/txtai/blob/master/examples/47_Building_an_efficient_sparse_keyword_index_in_Python.ipynb) | Fast and accurate sparse keyword indexing | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/47_Building_an_efficient_sparse_keyword_index_in_Python.ipynb) |
+| [Benefits of hybrid search](https://github.com/neuml/txtai/blob/master/examples/48_Benefits_of_hybrid_search.ipynb) | Improve accuracy with a combination of semantic and keyword search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/48_Benefits_of_hybrid_search.ipynb) |
+| [External database integration](https://github.com/neuml/txtai/blob/master/examples/49_External_database_integration.ipynb) | Store metadata in PostgreSQL, MariaDB, MySQL and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/49_External_database_integration.ipynb) |
+| [All about vector quantization](https://github.com/neuml/txtai/blob/master/examples/50_All_about_vector_quantization.ipynb) | Benchmarking scalar and product quantization methods | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/50_All_about_vector_quantization.ipynb) |
+| [External vectorization](https://github.com/neuml/txtai/blob/master/examples/56_External_vectorization.ipynb) | Vectorization with precomputed embeddings datasets and APIs | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/56_External_vectorization.ipynb) |
+| [Integrate txtai with Postgres](https://github.com/neuml/txtai/blob/master/examples/61_Integrate_txtai_with_Postgres.ipynb) | Persist content, vectors and graph data in Postgres | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/61_Integrate_txtai_with_Postgres.ipynb) |
+| [Embeddings index format for open data access](https://github.com/neuml/txtai/blob/master/examples/64_Embeddings_index_format_for_open_data_access.ipynb) | Platform and programming language independent data storage with txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/64_Embeddings_index_format_for_open_data_access.ipynb) |
+| [Accessing Low Level Vector APIs](https://github.com/neuml/txtai/blob/master/examples/78_Accessing_Low_Level_Vector_APIs.ipynb) | Build a vector database using txtai's low-level APIs | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/78_Accessing_Low_Level_Vector_APIs.ipynb) |
+
+## Releases
+
+New functionality added in major releases.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [What's new in txtai 4.0](https://github.com/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) | Content storage, SQL, object storage, reindex and compressed indexes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) |
+| [What's new in txtai 6.0](https://github.com/neuml/txtai/blob/master/examples/46_Whats_new_in_txtai_6_0.ipynb) | Sparse, hybrid and subindexes for embeddings, LLM improvements | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/46_Whats_new_in_txtai_6_0.ipynb) |
+| [What's new in txtai 7.0](https://github.com/neuml/txtai/blob/master/examples/59_Whats_new_in_txtai_7_0.ipynb) | Semantic graph 2.0, LoRA/QLoRA training and binary API support | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/59_Whats_new_in_txtai_7_0.ipynb) |
+| [What's new in txtai 8.0](https://github.com/neuml/txtai/blob/master/examples/67_Whats_new_in_txtai_8_0.ipynb) | Agents with txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/67_Whats_new_in_txtai_8_0.ipynb) |
+| [What's new in txtai 9.0](https://github.com/neuml/txtai/blob/master/examples/76_Whats_new_in_txtai_9_0.ipynb) | Learned sparse vectors, late interaction models and rerankers | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/76_Whats_new_in_txtai_9_0.ipynb) |
+
 ## Applications
 
-Series of example applications with txtai. Links to hosted versions on [Hugging Face Spaces](https://hf.co/spaces) also provided.
+Series of example applications with txtai. Links to hosted versions on [Hugging Face Spaces](https://hf.co/spaces) are also provided, when available.
 
 | Application  | Description  |       |
 |:-------------|:-------------|------:|
 | [Basic similarity search](https://github.com/neuml/txtai/blob/master/examples/similarity.py) | Basic similarity search example. Data from the original txtai demo. |[🤗](https://hf.co/spaces/NeuML/similarity)|
+| [Baseball stats](https://github.com/neuml/txtai/blob/master/examples/baseball.py) | Match historical baseball player stats using vector search. |[🤗](https://hf.co/spaces/NeuML/baseball)|
+| [Benchmarks](https://github.com/neuml/txtai/blob/master/examples/benchmarks.py) | Calculate performance metrics for the BEIR datasets. |*Local run only*|
 | [Book search](https://github.com/neuml/txtai/blob/master/examples/books.py) | Book similarity search application. Index book descriptions and query using natural language statements. |*Local run only*|
 | [Image search](https://github.com/neuml/txtai/blob/master/examples/images.py) | Image similarity search application. Index a directory of images and run searches to identify images similar to the input query. |[🤗](https://hf.co/spaces/NeuML/imagesearch)|
+| [Retrieval Augmented Generation](https://github.com/neuml/rag/) | RAG with txtai embeddings databases. Ask questions and get answers from LLMs bound by a context. |*Local run only*|
 | [Summarize an article](https://github.com/neuml/txtai/blob/master/examples/article.py) | Summarize an article. Workflow that extracts text from a webpage and builds a summary. |[🤗](https://hf.co/spaces/NeuML/articlesummary)|
 | [Wiki search](https://github.com/neuml/txtai/blob/master/examples/wiki.py) | Wikipedia search application. Queries Wikipedia API and summarizes the top result. |[🤗](https://hf.co/spaces/NeuML/wikisummary)|
 | [Workflow builder](https://github.com/neuml/txtai/blob/master/examples/workflows.py) | Build and execute txtai workflows. Connect summarization, text extraction, transcription, translation and similarity search pipelines together to run unified workflows. |[🤗](https://hf.co/spaces/NeuML/txtai)|

docs/faq.md

@@ -0,0 +1,151 @@
+# FAQ
+
+![faq](images/faq.png)
+
+Below is a list of frequently asked questions and common issues encountered.
+
+## Questions
+
+----------
+
+__Question__
+
+What models are recommended?
+
+__Answer__
+
+See the [model guide](../models).
+
+----------
+
+__Question__
+
+What is the best way to track the progress of an `embeddings.index` call?
+
+__Answer__
+
+Wrap the list or generator passed to the index call with tqdm. See [#478](https://github.com/neuml/txtai/issues/478) for more.
+
+----------
+
+__Question__
+
+What is the best way to analyze and debug a txtai process?
+
+__Answer__
+
+See the [observability](../observability) section for more on how this can be enabled in txtai processes.
+
+txtai also has a console application. [This article](https://medium.com/neuml/insights-from-the-txtai-console-d307c28e149e) has more details.
+
+----------
+
+__Question__
+
+How can models be externally loaded and passed to embeddings and pipelines?
+
+__Answer__
+
+Embeddings example.
+
+```python
+from transformers import AutoModel, AutoTokenizer
+from txtai import Embeddings
+
+# Load model externally
+model = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+
+# Pass to embeddings instance
+embeddings = Embeddings(path=model, tokenizer=tokenizer)
+```
+
+LLM pipeline example.
+
+```python
+import torch
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from txtai import LLM
+
+# Load Qwen3 0.6B
+path = "Qwen/Qwen3-0.6B"
+model = AutoModelForCausalLM.from_pretrained(
+  path,
+  dtype=torch.bfloat16,
+)
+tokenizer = AutoTokenizer.from_pretrained(path)
+
+llm = LLM((model, tokenizer))
+```
+
+## Common issues
+
+----------
+
+__Issue__
+
+Embeddings query errors like this:
+
+```
+SQLError: no such function: json_extract
+```
+
+__Solution__
+
+Upgrade Python version as it doesn't have SQLite support for `json_extract`
+
+----------
+
+__Issue__
+
+Segmentation faults and similar errors on macOS
+
+__Solution__
+
+Set the following environment parameters.
+
+- Disable OpenMP multithreading via `export OMP_NUM_THREADS=1`
+- Workaround `OMP: Error #15` errors via `export KMP_DUPLICATE_LIB_OK=TRUE`
+- Disable PyTorch MPS device via `export PYTORCH_MPS_DISABLE=1`
+- Disable llama.cpp metal via `export LLAMA_NO_METAL=1`
+
+For more details, refer to [this issue on GitHub](https://github.com/kyamagu/faiss-wheels/issues/73).
+
+----------
+
+__Issue__
+
+Error running SQLite ANN on macOS
+
+```
+AttributeError: 'sqlite3.Connection' object has no attribute 'enable_load_extension'
+```
+
+__Solution__
+
+See [this note](https://alexgarcia.xyz/sqlite-vec/python.html#macos-blocks-sqlite-extensions-by-default) for options on how to fix this.
+
+----------
+
+__Issue__
+
+`ContextualVersionConflict` and/or package METADATA exception while running one of the [examples](../examples) notebooks on Google Colab
+
+__Solution__
+
+Restart the kernel. See issue [#409](https://github.com/neuml/txtai/issues/409) for more on this issue. 
+
+----------
+
+__Issue__
+
+Error installing optional/extra dependencies such as `pipeline`
+
+__Solution__
+
+The default MacOS shell (zsh) and Windows PowerShell require escaping square brackets
+
+```
+pip install 'txtai[pipeline]'
+```

docs/further.md

@@ -1,9 +1,12 @@
 # Further reading
 
-![further](images/further.png)
+![further](images/further.png#only-light)
+![further](images/further-dark.png#only-dark)
 
-- [Introducing txtai, AI-powered semantic search built on Transformers](https://towardsdatascience.com/introducing-txtai-an-ai-powered-search-engine-built-on-transformers-37674be252ec)
-- [Run machine-learning workflows to transform data and build AI-powered semantic search applications with txtai](https://towardsdatascience.com/run-machine-learning-workflows-to-transform-data-and-build-ai-powered-text-indices-with-txtai-43d769b566a7)
-- [Semantic search on the cheap](https://towardsdatascience.com/semantic-search-on-the-cheap-55940c0fcdab)
-- [What's new in txtai 4.0](https://towardsdatascience.com/whats-new-in-txtai-4-0-bbc3a65c3d1c)
-- [Tutorial series on dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
+- [Introducing txtai, the all-in-one AI framework](https://medium.com/neuml/introducing-txtai-the-all-in-one-ai-framework-0660ecfc39d7)
+- [txtai: An All-in-One AI Framework for Semantic Search and LLM Workflows](https://github.com/neuml/papers/blob/master/txtai/txtai.pdf)
+- [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
+- [What's new in txtai 9.0](https://medium.com/neuml/whats-new-in-txtai-9-0-d522bb150afa) | [8.0](https://medium.com/neuml/whats-new-in-txtai-8-0-2d7d0ab4506b) | [7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
+- [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9) | [rag](https://medium.com/neuml/getting-started-with-rag-9a0cca75f748)
+- [Running txtai at scale](https://medium.com/neuml/running-at-scale-with-txtai-71196cdd99f9)
+- [Vector search & RAG Landscape: A review with txtai](https://medium.com/neuml/vector-search-rag-landscape-a-review-with-txtai-a7f37ad0e187)

docs/images/architecture.excalidraw

@@ -0,0 +1,996 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 4858,
+      "versionNonce": 83494474,
+      "isDeleted": false,
+      "id": "1iE4X4trkcbtJpCDC5ajY",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 27.525089187647836,
+      "y": -558.1200877100962,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 227.65938859429855,
+      "height": 57.830603028426594,
+      "seed": 1321467630,
+      "groupIds": [
+        "MnVd9k06uFuFRzqXmnEvR",
+        "NFKrTdLl_zy4zOvDGWVBD"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "id": "qROQdhT3FWGzgS0JOQXlg",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 5224,
+      "versionNonce": 1682430806,
+      "isDeleted": false,
+      "id": "ZiUda1q4lvcNM9G2Or_d7",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 39.103253497881326,
+      "y": -548.6024245205953,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#343a40",
+      "width": 210.71665954589844,
+      "height": 40,
+      "seed": 2050033394,
+      "groupIds": [
+        "NFKrTdLl_zy4zOvDGWVBD"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "qROQdhT3FWGzgS0JOQXlg",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": "- Semantic, Keyword, Hybrid\n- Search with SQL",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Semantic, Keyword, Hybrid\n- Search with SQL",
+      "lineHeight": 1.25,
+      "baseline": 35
+    },
+    {
+      "type": "rectangle",
+      "version": 2527,
+      "versionNonce": 2136048394,
+      "isDeleted": false,
+      "id": "i9TRMGNTF_ckjPiI7BrdL",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 358.6031681551546,
+      "y": -598.4332496538902,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 348.08489527005264,
+      "height": 200.0037909410919,
+      "seed": 1867912238,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "id": "viSfAjT-8KRTYvsrunjfn",
+          "type": "arrow"
+        },
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        },
+        {
+          "id": "sf5i6bxPYE2ykeh2wWm8m",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411996548,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "rectangle",
+      "version": 1915,
+      "versionNonce": 211681430,
+      "isDeleted": false,
+      "id": "F1r0Plm7hDaeNndIwZAuf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 368.61999967371725,
+      "y": -538.0801566026939,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 160.26930429700263,
+      "height": 62.605196991016655,
+      "seed": 436869746,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "gw7zjxlDiAjodT1x1_fv_"
+        },
+        {
+          "id": "qROQdhT3FWGzgS0JOQXlg",
+          "type": "arrow"
+        },
+        {
+          "id": "viSfAjT-8KRTYvsrunjfn",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1897,
+      "versionNonce": 347110346,
+      "isDeleted": false,
+      "id": "gw7zjxlDiAjodT1x1_fv_",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 406.8879869357439,
+      "y": -522.695916613224,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "transparent",
+      "width": 83.73332977294922,
+      "height": 31.836717012076832,
+      "seed": 1023300530,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false,
+      "fontSize": 25.469373609661467,
+      "fontFamily": 1,
+      "text": "Sparse",
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "F1r0Plm7hDaeNndIwZAuf",
+      "originalText": "Sparse",
+      "lineHeight": 1.25,
+      "baseline": 22
+    },
+    {
+      "type": "rectangle",
+      "version": 2272,
+      "versionNonce": 696090070,
+      "isDeleted": false,
+      "id": "55xWndBPWTiyUuLIS_J8b",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 369.8721036135376,
+      "y": -469.46654385239594,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 157.76509641736197,
+      "height": 57.59678123173532,
+      "seed": 1317384558,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "Qaeh-leNa7YgcPsQbfBcq"
+        },
+        {
+          "id": "qROQdhT3FWGzgS0JOQXlg",
+          "type": "arrow"
+        },
+        {
+          "id": "viSfAjT-8KRTYvsrunjfn",
+          "type": "arrow"
+        },
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 2210,
+      "versionNonce": 1434612362,
+      "isDeleted": false,
+      "id": "Qaeh-leNa7YgcPsQbfBcq",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 412.012986935744,
+      "y": -456.59111621253714,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "transparent",
+      "width": 73.48332977294922,
+      "height": 31.84592595201767,
+      "seed": 1661259762,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false,
+      "fontSize": 25.476740761614135,
+      "fontFamily": 1,
+      "text": "Dense",
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "55xWndBPWTiyUuLIS_J8b",
+      "originalText": "Dense",
+      "lineHeight": 1.25,
+      "baseline": 22
+    },
+    {
+      "type": "rectangle",
+      "version": 2356,
+      "versionNonce": 268129046,
+      "isDeleted": false,
+      "id": "KSf88Asx9gne1WsRdsx6-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 535.1498236698217,
+      "y": -469.46654385239594,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 156.51299247754164,
+      "height": 57.59678123173532,
+      "seed": 1016194418,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "7RaoYDeBRuz16CJo7jaVO"
+        },
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 2343,
+      "versionNonce": 2120716618,
+      "isDeleted": false,
+      "id": "7RaoYDeBRuz16CJo7jaVO",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 576.8063214344714,
+      "y": -456.5934785058263,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "transparent",
+      "width": 73.19999694824219,
+      "height": 31.850650538596007,
+      "seed": 1307924782,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false,
+      "fontSize": 25.480520430876805,
+      "fontFamily": 1,
+      "text": "Graph",
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "KSf88Asx9gne1WsRdsx6-",
+      "originalText": "Graph",
+      "lineHeight": 1.25,
+      "baseline": 22
+    },
+    {
+      "type": "text",
+      "version": 1738,
+      "versionNonce": 713850966,
+      "isDeleted": false,
+      "id": "gAcYCRjD6VZUsDZce-atm",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 439.98992424347625,
+      "y": -595.9290417742495,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#ffeb3b",
+      "width": 183.39999389648438,
+      "height": 43.823637893711656,
+      "seed": 1369566386,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991786,
+      "link": null,
+      "locked": false,
+      "fontSize": 35.058910314969324,
+      "fontFamily": 1,
+      "text": "Embeddings",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "Embeddings",
+      "lineHeight": 1.25,
+      "baseline": 31
+    },
+    {
+      "type": "rectangle",
+      "version": 2094,
+      "versionNonce": 704590410,
+      "isDeleted": false,
+      "id": "UEE-Z2YT9Y5YvVCJefbd0",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 533.8977197300013,
+      "y": -535.8280526628736,
+      "strokeColor": "#7950f2",
+      "backgroundColor": "#7950f2",
+      "width": 160.26930429700263,
+      "height": 60.10098911137599,
+      "seed": 1015954930,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "Y1uf_NH7fI6arCSGiICiX"
+        },
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691412007529,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 2084,
+      "versionNonce": 1292240278,
+      "isDeleted": false,
+      "id": "Y1uf_NH7fI6arCSGiICiX",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 551.0823711155632,
+      "y": -521.695916613224,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "transparent",
+      "width": 125.9000015258789,
+      "height": 31.836717012076832,
+      "seed": 867545010,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991786,
+      "link": null,
+      "locked": false,
+      "fontSize": 25.469373609661467,
+      "fontFamily": 1,
+      "text": "Database",
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "UEE-Z2YT9Y5YvVCJefbd0",
+      "originalText": "Database",
+      "lineHeight": 1.25,
+      "baseline": 22
+    },
+    {
+      "type": "rectangle",
+      "version": 4488,
+      "versionNonce": 1920419530,
+      "isDeleted": false,
+      "id": "uV8iktEBp-zvkGceO0YNj",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 28.400291679657585,
+      "y": -482.37846781152234,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 226.65938859429855,
+      "height": 53.830603028426594,
+      "seed": 511161586,
+      "groupIds": [
+        "lsnRF8t7e-8bA4k7ZMtQs",
+        "XOEGmXNmM0vVhFi_aLf0h"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "id": "viSfAjT-8KRTYvsrunjfn",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991786,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 4865,
+      "versionNonce": 144499414,
+      "isDeleted": false,
+      "id": "VU0ZFGV4tB5pR_8AMPehi",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 38.17389738827433,
+      "y": -477.32443222094344,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#343a40",
+      "width": 190.39999389648438,
+      "height": 40,
+      "seed": 660096690,
+      "groupIds": [
+        "XOEGmXNmM0vVhFi_aLf0h"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991786,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": "- Topics + Relationships\n- Multimodal Indexes",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Topics + Relationships\n- Multimodal Indexes",
+      "lineHeight": 1.25,
+      "baseline": 35
+    },
+    {
+      "type": "arrow",
+      "version": 3140,
+      "versionNonce": 1465895050,
+      "isDeleted": false,
+      "id": "qROQdhT3FWGzgS0JOQXlg",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 357.5459819116054,
+      "y": -474.380420640035,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 101.91733646013165,
+      "height": 51.182713481535075,
+      "seed": 1412373166,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691412001500,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "55xWndBPWTiyUuLIS_J8b",
+        "focus": -0.17593583244727756,
+        "gap": 13.269493633705139
+      },
+      "endBinding": {
+        "elementId": "ZiUda1q4lvcNM9G2Or_d7",
+        "focus": -0.7240167179756973,
+        "gap": 5.808732407693981
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -101.91733646013165,
+          -51.182713481535075
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 2221,
+      "versionNonce": 297446922,
+      "isDeleted": false,
+      "id": "viSfAjT-8KRTYvsrunjfn",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 355.69072075221885,
+      "y": -473.6934438187176,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 99.11528692741001,
+      "height": 28.941684185339795,
+      "seed": 769791342,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691412005709,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "55xWndBPWTiyUuLIS_J8b",
+        "focus": 1.1614221413579826,
+        "gap": 14.797915501333051
+      },
+      "endBinding": {
+        "elementId": "uV8iktEBp-zvkGceO0YNj",
+        "focus": 0.737345008582056,
+        "gap": 1.5157535508527076
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -99.11528692741001,
+          28.941684185339795
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 5939,
+      "versionNonce": 1648005846,
+      "isDeleted": false,
+      "id": "CJaOip-NXs_WGJ3YnzrzX",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.2644916835155,
+      "y": -444.3627069906215,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 71.1891690063859,
+      "height": 28.06038115974593,
+      "seed": 1923467222,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411901454,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "SKqPLoIsGXgDL5mhOTqLv",
+        "focus": -0.634641820715333,
+        "gap": 5.063962132776453
+      },
+      "endBinding": {
+        "elementId": "UEE-Z2YT9Y5YvVCJefbd0",
+        "focus": -0.06665280707924877,
+        "gap": 15.270023644140622
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": "arrow",
+      "endArrowhead": null,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -71.1891690063859,
+          -28.06038115974593
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 3429,
+      "versionNonce": 348832458,
+      "isDeleted": false,
+      "id": "7UwPwHCm_ZiMJb1o0x5A8",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 783.2131987136702,
+      "y": -574.1983152922421,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 272.0848952700526,
+      "height": 73.0037909410919,
+      "seed": 155582550,
+      "groupIds": [
+        "kgkwY28q5i7W008rJksMR",
+        "TqYTD07S3yHeAVJOG3yyi"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1591,
+      "versionNonce": 72296150,
+      "isDeleted": false,
+      "id": "fFWZLl9F8IwK5QhXlAssA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 792.6342330724652,
+      "y": -569.5895346298003,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#343a40",
+      "width": 77.1500015258789,
+      "height": 20,
+      "seed": 543556438,
+      "groupIds": [
+        "TqYTD07S3yHeAVJOG3yyi"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": " Prompt >",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": " Prompt >",
+      "lineHeight": 1.25,
+      "baseline": 15
+    },
+    {
+      "type": "text",
+      "version": 1341,
+      "versionNonce": 450634262,
+      "isDeleted": false,
+      "id": "FgFcgIY3s9iRdPKwN4t_r",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 785.1996772966362,
+      "y": -548.1590294764883,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#343a40",
+      "width": 261.5,
+      "height": 41.41205162166277,
+      "seed": 423650058,
+      "groupIds": [
+        "TqYTD07S3yHeAVJOG3yyi"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691412007529,
+      "link": null,
+      "locked": false,
+      "fontSize": 16.56482064866511,
+      "fontFamily": 1,
+      "text": "   Answer the following question\n   using the context below",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "   Answer the following question\n   using the context below",
+      "lineHeight": 1.25,
+      "baseline": 35
+    },
+    {
+      "type": "rectangle",
+      "version": 3467,
+      "versionNonce": 1420811286,
+      "isDeleted": false,
+      "id": "_8pPaWBfdkvpd8B9PQ7wI",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 784.2131987136702,
+      "y": -487.1983152922421,
+      "strokeColor": "#6741d9",
+      "backgroundColor": "#6741d9",
+      "width": 272.0848952700526,
+      "height": 73.0037909410919,
+      "seed": 2066566410,
+      "groupIds": [
+        "FtTm_MWbxe67f2n84saes",
+        "3PAnqhix8z46z6JRJjAO7"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1602,
+      "versionNonce": 1943803978,
+      "isDeleted": false,
+      "id": "apAxY8HSw8w2lfLe1pCXP",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 792.7130504007473,
+      "y": -482.33339890257247,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#6741d9",
+      "width": 75.73332977294922,
+      "height": 20,
+      "seed": 1870973578,
+      "groupIds": [
+        "3PAnqhix8z46z6JRJjAO7"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": " Search >",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": " Search >",
+      "lineHeight": 1.25,
+      "baseline": 15
+    },
+    {
+      "type": "text",
+      "version": 1069,
+      "versionNonce": 1506273622,
+      "isDeleted": false,
+      "id": "SKqPLoIsGXgDL5mhOTqLv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 785.3284538162922,
+      "y": -458.46265045437633,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#6741d9",
+      "width": 251.21665954589844,
+      "height": 41.82578128069407,
+      "seed": 1533237462,
+      "groupIds": [
+        "3PAnqhix8z46z6JRJjAO7"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false,
+      "fontSize": 16.730312512277628,
+      "fontFamily": 1,
+      "text": "   SELECT ... FROM txtai\n   WHERE SIMILAR('question')",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "   SELECT ... FROM txtai\n   WHERE SIMILAR('question')",
+      "lineHeight": 1.25,
+      "baseline": 35
+    },
+    {
+      "type": "arrow",
+      "version": 5999,
+      "versionNonce": 1002058634,
+      "isDeleted": false,
+      "id": "sf5i6bxPYE2ykeh2wWm8m",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.8522503409429,
+      "y": -537.8502934180265,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 72.32385186561339,
+      "height": 64.0647548382936,
+      "seed": 1598499018,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691412007529,
+      "link": null,
+      "locked": false,
+      "startBinding": null,
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": "arrow",
+      "endArrowhead": null,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -72.32385186561339,
+          64.0647548382936
+        ]
+      ]
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

docs/images/cloud.excalidraw

@@ -5,8 +5,8 @@
   "elements": [
     {
       "type": "rectangle",
-      "version": 3020,
-      "versionNonce": 574956379,
+      "version": 3062,
+      "versionNonce": 456539957,
       "isDeleted": false,
       "id": "Ufh5VUA3qmvJowuFyEWz4",
       "fillStyle": "hachure",
@@ -21,187 +21,119 @@
       "backgroundColor": "transparent",
       "width": 719.330217768941,
       "height": 100.49478848195537,
-      "seed": 1218477931,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
-      "boundElements": [
-        {
-          "type": "arrow",
-          "id": "QPHaiLoteZgu8lrk22yKh"
-        }
-      ],
-      "updated": 1644866249723,
-      "link": null
+      "seed": 2109972213,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688312603126,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1748,
-      "versionNonce": 1788187605,
+      "version": 1798,
+      "versionNonce": 1560857173,
       "isDeleted": false,
       "id": "NDWbJgM53uVck-LSQzD-0",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 740.6664767795138,
-      "y": 67.41041395399299,
+      "y": 67.92614920483035,
       "strokeColor": "#ffeb3b",
       "backgroundColor": "#ffeb3b",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 1934943141,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1782446165,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "type": "text",
           "id": "0VztU7lb7T9I5zoL2LjLt"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
-      "id": "0VztU7lb7T9I5zoL2LjLt",
       "type": "text",
-      "x": 745.6664767795138,
-      "y": 98.99999999999994,
-      "width": 75,
-      "height": 21,
-      "angle": 0,
-      "strokeColor": "#000000",
-      "backgroundColor": "#7950f2",
+      "version": 385,
+      "versionNonce": 1280754907,
+      "isDeleted": false,
+      "id": "0VztU7lb7T9I5zoL2LjLt",
       "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
-      "seed": 604922325,
-      "version": 333,
-      "versionNonce": 1646133243,
-      "isDeleted": false,
-      "boundElements": null,
-      "updated": 1644866249723,
+      "angle": 0,
+      "x": 745.6664767795138,
+      "y": 99.51573525083731,
+      "strokeColor": "#000000",
+      "backgroundColor": "#7950f2",
+      "width": 75,
+      "height": 21,
+      "seed": 2095457717,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688312605190,
       "link": null,
-      "text": "Container",
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
+      "text": "Container",
       "textAlign": "center",
       "verticalAlign": "middle",
-      "baseline": 15,
       "containerId": "NDWbJgM53uVck-LSQzD-0",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     },
     {
       "type": "rectangle",
-      "version": 1824,
-      "versionNonce": 1455313205,
+      "version": 1874,
+      "versionNonce": 847689653,
       "isDeleted": false,
       "id": "rr3YJsAyrrPqZbZ4qXh5x",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 1175.874140249897,
-      "y": 70.42614920483032,
+      "y": 67.92614920483035,
       "strokeColor": "#00e676",
       "backgroundColor": "#00e676",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 1158748789,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1491183381,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "id": "9NBdG6TyrBqjuDKvzAZXN",
           "type": "text"
-        },
-        {
-          "id": "9NBdG6TyrBqjuDKvzAZXN",
-          "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "9NBdG6TyrBqjuDKvzAZXN"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 404,
-      "versionNonce": 938625179,
+      "version": 456,
+      "versionNonce": 1820291451,
       "isDeleted": false,
       "id": "9NBdG6TyrBqjuDKvzAZXN",
       "fillStyle": "hachure",
@@ -211,100 +143,65 @@
       "opacity": 100,
       "angle": 0,
       "x": 1180.874140249897,
-      "y": 102.01573525083728,
+      "y": 99.51573525083731,
       "strokeColor": "#000000",
       "backgroundColor": "#7950f2",
       "width": 75,
       "height": 21,
-      "seed": 603698011,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 547748981,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1644866249723,
+      "updated": 1688312605190,
       "link": null,
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
       "text": "Container",
-      "baseline": 15,
       "textAlign": "center",
       "verticalAlign": "middle",
       "containerId": "rr3YJsAyrrPqZbZ4qXh5x",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     },
     {
       "type": "rectangle",
-      "version": 1814,
-      "versionNonce": 766010005,
+      "version": 1864,
+      "versionNonce": 1889532181,
       "isDeleted": false,
       "id": "ji9UBE590YJ5e8DKUKbPG",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 598.874140249897,
-      "y": 68.42614920483032,
+      "y": 67.92614920483035,
       "strokeColor": "#ff7043",
       "backgroundColor": "#ff7043",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 895199477,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 200844757,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "id": "2fdy7dRwF7ZVaYPrb1Q7p",
           "type": "text"
-        },
-        {
-          "id": "2fdy7dRwF7ZVaYPrb1Q7p",
-          "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "2fdy7dRwF7ZVaYPrb1Q7p"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 394,
-      "versionNonce": 695500091,
+      "version": 446,
+      "versionNonce": 673810971,
       "isDeleted": false,
       "id": "2fdy7dRwF7ZVaYPrb1Q7p",
       "fillStyle": "hachure",
@@ -314,96 +211,65 @@
       "opacity": 100,
       "angle": 0,
       "x": 603.874140249897,
-      "y": 100.01573525083728,
+      "y": 99.51573525083731,
       "strokeColor": "#000000",
       "backgroundColor": "#7950f2",
       "width": 75,
       "height": 21,
-      "seed": 507989211,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1039082293,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1644866249723,
+      "updated": 1688312605190,
       "link": null,
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
       "text": "Container",
-      "baseline": 15,
       "textAlign": "center",
       "verticalAlign": "middle",
       "containerId": "ji9UBE590YJ5e8DKUKbPG",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     },
     {
       "type": "rectangle",
-      "version": 1754,
-      "versionNonce": 873023477,
+      "version": 1805,
+      "versionNonce": 281677429,
       "isDeleted": false,
       "id": "Kod3erHdgkFtJcsgqHeQQ",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 889.874140249897,
-      "y": 68.42614920483032,
+      "y": 67.92614920483035,
       "strokeColor": "#03a9f4",
       "backgroundColor": "#03a9f4",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 1941388379,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1525609621,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "id": "vA5QfvPySVZE9WnP6d7dp",
           "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "vA5QfvPySVZE9WnP6d7dp"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 337,
-      "versionNonce": 1644633563,
+      "version": 390,
+      "versionNonce": 1682411195,
       "isDeleted": false,
       "id": "vA5QfvPySVZE9WnP6d7dp",
       "fillStyle": "hachure",
@@ -413,104 +279,65 @@
       "opacity": 100,
       "angle": 0,
       "x": 894.874140249897,
-      "y": 100.01573525083728,
+      "y": 99.51573525083731,
       "strokeColor": "#000000",
       "backgroundColor": "#7950f2",
       "width": 75,
       "height": 21,
-      "seed": 867011285,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1588435445,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1644866249723,
+      "updated": 1688312605190,
       "link": null,
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
       "text": "Container",
-      "baseline": 15,
       "textAlign": "center",
       "verticalAlign": "middle",
       "containerId": "Kod3erHdgkFtJcsgqHeQQ",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     },
     {
       "type": "rectangle",
-      "version": 1863,
-      "versionNonce": 2078703957,
+      "version": 1916,
+      "versionNonce": 1103787989,
       "isDeleted": false,
       "id": "8p_AIsxC6UbaWESkrwl4g",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 1035.874140249897,
-      "y": 69.42614920483032,
-      "strokeColor": "#5f3dc4",
+      "y": 67.92614920483035,
+      "strokeColor": "#7950f2",
       "backgroundColor": "#7950f2",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 1712249051,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 842044245,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "id": "AceuLTPujneYjRp_njhIl",
           "type": "text"
-        },
-        {
-          "id": "AceuLTPujneYjRp_njhIl",
-          "type": "text"
-        },
-        {
-          "id": "AceuLTPujneYjRp_njhIl",
-          "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "AceuLTPujneYjRp_njhIl"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 443,
-      "versionNonce": 2092739195,
+      "version": 499,
+      "versionNonce": 1132199771,
       "isDeleted": false,
       "id": "AceuLTPujneYjRp_njhIl",
       "fillStyle": "hachure",
@@ -519,28 +346,29 @@
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
-      "x": 1040.874140249897,
-      "y": 101.01573525083728,
+      "x": 1041.907473074604,
+      "y": 99.51573525083731,
       "strokeColor": "#000000",
       "backgroundColor": "#7950f2",
-      "width": 75,
+      "width": 72.93333435058594,
       "height": 21,
-      "seed": 1770780245,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 30236853,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1644866249723,
+      "updated": 1688312605190,
       "link": null,
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
       "text": "Container",
-      "baseline": 15,
       "textAlign": "center",
       "verticalAlign": "middle",
       "containerId": "8p_AIsxC6UbaWESkrwl4g",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     }
   ],
   "appState": {

docs/images/examples.excalidraw

@@ -5,8 +5,8 @@
   "elements": [
     {
       "type": "rectangle",
-      "version": 623,
-      "versionNonce": 1614613512,
+      "version": 658,
+      "versionNonce": 56883131,
       "isDeleted": false,
       "id": "28knmXv7UWxhQgfO1Ap_e",
       "fillStyle": "hachure",
@@ -23,14 +23,17 @@
       "height": 19.163298353951955,
       "seed": 333697800,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176184,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 724,
-      "versionNonce": 384800888,
+      "version": 759,
+      "versionNonce": 1321514357,
       "isDeleted": false,
       "id": "NWB9SsDdfRnKFM7WSMiDD",
       "fillStyle": "hachure",
@@ -47,14 +50,17 @@
       "height": 8.353548393013705,
       "seed": 1799143800,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176184,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 838,
-      "versionNonce": 1137593096,
+      "version": 873,
+      "versionNonce": 1965246555,
       "isDeleted": false,
       "id": "o8mRXU1Pv0Lht3Kvw3LjM",
       "fillStyle": "hachure",
@@ -71,14 +77,17 @@
       "height": 10.760364257767161,
       "seed": 84161032,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176184,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 817,
-      "versionNonce": 322230024,
+      "version": 852,
+      "versionNonce": 1473789653,
       "isDeleted": false,
       "id": "7XXBg9rngZPBd8BY3k-Dk",
       "fillStyle": "hachure",
@@ -95,14 +104,17 @@
       "height": 326.7582126998559,
       "seed": 1645597304,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641752945859
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 785,
-      "versionNonce": 1088399880,
+      "version": 820,
+      "versionNonce": 1609532667,
       "isDeleted": false,
       "id": "1djoflJbPq1iZYxeG6vvs",
       "fillStyle": "hachure",
@@ -119,14 +131,17 @@
       "height": 8.353548393013705,
       "seed": 1354805512,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1012,
-      "versionNonce": 2110642808,
+      "version": 1047,
+      "versionNonce": 2087951413,
       "isDeleted": false,
       "id": "61SvR173kGBlNiM2hYx-i",
       "fillStyle": "hachure",
@@ -143,14 +158,19 @@
       "height": 35.19095464139164,
       "seed": 497337208,
       "groupIds": [],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 1
+      },
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 1144,
-      "versionNonce": 1288082440,
+      "version": 1179,
+      "versionNonce": 1800088987,
       "isDeleted": false,
       "id": "KweZDYL4pV266i29UQo2O",
       "fillStyle": "hachure",
@@ -167,22 +187,26 @@
       "height": 17,
       "seed": 244666376,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745960277,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.613804582094819,
       "fontFamily": 1,
       "text": "Show more",
-      "baseline": 12,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Show more"
+      "originalText": "Show more",
+      "lineHeight": 1.3477297741025214,
+      "baseline": 12
     },
     {
       "type": "line",
-      "version": 1020,
-      "versionNonce": 354143096,
+      "version": 1055,
+      "versionNonce": 612126101,
       "isDeleted": false,
       "id": "6UHr1F9sbQjfc3Pn3AUC9",
       "fillStyle": "hachure",
@@ -199,9 +223,14 @@
       "height": 7.91335492900009,
       "seed": 1278055544,
       "groupIds": [],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
       "boundElements": [],
-      "updated": 1641745822132,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "startBinding": null,
       "endBinding": null,
       "lastCommittedPoint": null,
@@ -224,11 +253,11 @@
     },
     {
       "type": "rectangle",
-      "version": 1546,
-      "versionNonce": 871548936,
+      "version": 1593,
+      "versionNonce": 711508539,
       "isDeleted": false,
       "id": "pQ7Helj-RkhLonwDn4uxq",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -244,17 +273,20 @@
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822133
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "line",
-      "version": 758,
-      "versionNonce": 1334025336,
+      "version": 806,
+      "versionNonce": 522703605,
       "isDeleted": false,
       "id": "bI-A-gEMD-yvyhVvuajAH",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -270,9 +302,14 @@
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
       "boundElements": [],
-      "updated": 1641745822133,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "startBinding": null,
       "endBinding": null,
       "lastCommittedPoint": null,
@@ -291,11 +328,11 @@
     },
     {
       "type": "line",
-      "version": 768,
-      "versionNonce": 292325128,
+      "version": 816,
+      "versionNonce": 640336603,
       "isDeleted": false,
       "id": "X3L53b-aDRBM9J_i7xfIb",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -311,9 +348,14 @@
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
       "boundElements": [],
-      "updated": 1641745822133,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "startBinding": null,
       "endBinding": null,
       "lastCommittedPoint": null,
@@ -332,11 +374,11 @@
     },
     {
       "type": "rectangle",
-      "version": 1720,
-      "versionNonce": 1858686328,
+      "version": 1768,
+      "versionNonce": 956017749,
       "isDeleted": false,
       "id": "Fk7rL2dsob9lBYBWtqIaE",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -345,24 +387,27 @@
       "x": 909.5034365835013,
       "y": 461.80381850437016,
       "strokeColor": "#000000",
-      "backgroundColor": "#03a9f4",
+      "backgroundColor": "#ffeb3b",
       "width": 12.48277017202394,
       "height": 62.41385086011974,
       "seed": 1096491528,
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822134
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1590,
-      "versionNonce": 928409096,
+      "version": 1638,
+      "versionNonce": 656650107,
       "isDeleted": false,
       "id": "V3UV1tCb1upEwCUx2NF_U",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -371,24 +416,27 @@
       "x": 940.7103620135604,
       "y": 480.5279737624063,
       "strokeColor": "#000000",
-      "backgroundColor": "#ffeb3b",
+      "backgroundColor": "#03a9f4",
       "width": 12.482770172023953,
       "height": 43.68969560208382,
       "seed": 1860559480,
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822134
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1597,
-      "versionNonce": 1546706552,
+      "version": 1650,
+      "versionNonce": 793292539,
       "isDeleted": false,
       "id": "H9TFct1zEuli3oFZ0J6Pi",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -397,24 +445,27 @@
       "x": 971.9172874436208,
       "y": 474.28658867639376,
       "strokeColor": "#000000",
-      "backgroundColor": "#00e676",
+      "backgroundColor": "#7950f2",
       "width": 12.482770172023953,
       "height": 49.9310806880958,
       "seed": 1153016072,
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822134
+      "updated": 1688314179109,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1754,
-      "versionNonce": 1655465224,
+      "version": 1806,
+      "versionNonce": 1066797083,
       "isDeleted": false,
       "id": "Yvl5ZFMXGqZoKBjooHuuL",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -423,21 +474,24 @@
       "x": 1003.1242128736812,
       "y": 468.04520359038156,
       "strokeColor": "#000000",
-      "backgroundColor": "#ff7043",
+      "backgroundColor": "#00e676",
       "width": 12.48277017202394,
       "height": 56.172465774107785,
       "seed": 905106296,
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822134
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 785,
-      "versionNonce": 725387784,
+      "version": 820,
+      "versionNonce": 851598101,
       "isDeleted": false,
       "id": "k2RjGkR4842FWeQqwZNqP",
       "fillStyle": "hachure",
@@ -454,22 +508,26 @@
       "height": 46,
       "seed": 351902216,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746472537,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 36,
       "fontFamily": 1,
       "text": "Results",
-      "baseline": 32,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Results"
+      "originalText": "Results",
+      "lineHeight": 1.2777777777777777,
+      "baseline": 32
     },
     {
       "type": "line",
-      "version": 352,
-      "versionNonce": 1781891704,
+      "version": 387,
+      "versionNonce": 359943355,
       "isDeleted": false,
       "id": "9BG_CdPIU12PIVg57JFjb",
       "fillStyle": "hachure",
@@ -488,9 +546,14 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
       "boundElements": [],
-      "updated": 1641746497832,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "startBinding": null,
       "endBinding": null,
       "lastCommittedPoint": null,
@@ -509,8 +572,8 @@
     },
     {
       "type": "text",
-      "version": 924,
-      "versionNonce": 382863880,
+      "version": 959,
+      "versionNonce": 1682976885,
       "isDeleted": false,
       "id": "ooEpA700FaTXoHICTApB5",
       "fillStyle": "hachure",
@@ -529,22 +592,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972927,
       "fontFamily": 1,
       "text": "0",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "0"
+      "originalText": "0",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 996,
-      "versionNonce": 2125560440,
+      "version": 1031,
+      "versionNonce": 1818295643,
       "isDeleted": false,
       "id": "n9FAYaIZeMXggjqKvdYxs",
       "fillStyle": "hachure",
@@ -563,22 +630,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972932,
       "fontFamily": 1,
       "text": "1",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "1"
+      "originalText": "1",
+      "lineHeight": 1.3556693953949972,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1072,
-      "versionNonce": 454265096,
+      "version": 1107,
+      "versionNonce": 1144805845,
       "isDeleted": false,
       "id": "fQfYp0ljtX1ESRr5wPM9b",
       "fillStyle": "hachure",
@@ -597,22 +668,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "2",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "2"
+      "originalText": "2",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 980,
-      "versionNonce": 2022402936,
+      "version": 1015,
+      "versionNonce": 1873713659,
       "isDeleted": false,
       "id": "6-kghWfO7GdeQUDWs_zBe",
       "fillStyle": "hachure",
@@ -631,22 +706,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972927,
       "fontFamily": 1,
       "text": "ID",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "ID"
+      "originalText": "ID",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 989,
-      "versionNonce": 952969224,
+      "version": 1024,
+      "versionNonce": 463230773,
       "isDeleted": false,
       "id": "UMowu2HWCm_mgorPKfHhX",
       "fillStyle": "hachure",
@@ -665,22 +744,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "_____",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "_____"
+      "originalText": "_____",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1058,
-      "versionNonce": 1677671544,
+      "version": 1093,
+      "versionNonce": 1012900507,
       "isDeleted": false,
       "id": "1_YJ1fjNPFhQdtKIp-DSI",
       "fillStyle": "hachure",
@@ -699,22 +782,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.90383362297293,
       "fontFamily": 1,
       "text": "_____",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "_____"
+      "originalText": "_____",
+      "lineHeight": 1.3556693953949974,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1133,
-      "versionNonce": 1274734344,
+      "version": 1168,
+      "versionNonce": 1960159381,
       "isDeleted": false,
       "id": "O5wbweMweVsQWwdNABtm7",
       "fillStyle": "hachure",
@@ -733,22 +820,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "____",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "____"
+      "originalText": "____",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1033,
-      "versionNonce": 852553080,
+      "version": 1068,
+      "versionNonce": 723949371,
       "isDeleted": false,
       "id": "v3I-eUd4WMJ8VzvC_7rB_",
       "fillStyle": "hachure",
@@ -767,22 +858,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972927,
       "fontFamily": 1,
       "text": "Text",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Text"
+      "originalText": "Text",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1084,
-      "versionNonce": 1360578056,
+      "version": 1119,
+      "versionNonce": 1906167285,
       "isDeleted": false,
       "id": "sdGrFMITxSJsejpV0zIFY",
       "fillStyle": "hachure",
@@ -801,22 +896,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972929,
       "fontFamily": 1,
       "text": "Score",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Score"
+      "originalText": "Score",
+      "lineHeight": 1.3556693953949976,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1007,
-      "versionNonce": 2118418040,
+      "version": 1042,
+      "versionNonce": 1516632027,
       "isDeleted": false,
       "id": "5tPTHdBAi9oW2Cme01uST",
       "fillStyle": "hachure",
@@ -835,22 +934,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972929,
       "fontFamily": 1,
       "text": "0.851",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "0.851"
+      "originalText": "0.851",
+      "lineHeight": 1.3556693953949976,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1082,
-      "versionNonce": 1428342024,
+      "version": 1117,
+      "versionNonce": 1627014997,
       "isDeleted": false,
       "id": "VkvcQQqVWAFSBPXeb-631",
       "fillStyle": "hachure",
@@ -869,22 +972,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972929,
       "fontFamily": 1,
       "text": "0.834",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "0.834"
+      "originalText": "0.834",
+      "lineHeight": 1.3556693953949976,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1154,
-      "versionNonce": 387837816,
+      "version": 1189,
+      "versionNonce": 2016429179,
       "isDeleted": false,
       "id": "r5V4P6WY1YTnZqynbGcPW",
       "fillStyle": "hachure",
@@ -903,22 +1010,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972924,
       "fontFamily": 1,
       "text": "0.653",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "0.653"
+      "originalText": "0.653",
+      "lineHeight": 1.355669395394998,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1129,
-      "versionNonce": 1704256520,
+      "version": 1164,
+      "versionNonce": 356553909,
       "isDeleted": false,
       "id": "tplkVB19dZGWPH04eoN3-",
       "fillStyle": "hachure",
@@ -937,22 +1048,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176187,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972922,
       "fontFamily": 1,
       "text": "Data",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Data"
+      "originalText": "Data",
+      "lineHeight": 1.3556693953949983,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1056,
-      "versionNonce": 362889336,
+      "version": 1091,
+      "versionNonce": 1279786267,
       "isDeleted": false,
       "id": "-O_818hE0e37gLO8X85ED",
       "fillStyle": "hachure",
@@ -971,22 +1086,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176187,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "{____}",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "{____}"
+      "originalText": "{____}",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1083,
-      "versionNonce": 7021320,
+      "version": 1118,
+      "versionNonce": 2048275989,
       "isDeleted": false,
       "id": "q_5ogcScUemL3g6XJyETb",
       "fillStyle": "hachure",
@@ -1005,22 +1124,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176187,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "{____}",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "{____}"
+      "originalText": "{____}",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1076,
-      "versionNonce": 1771204984,
+      "version": 1111,
+      "versionNonce": 35627451,
       "isDeleted": false,
       "id": "hePUy_nbNvIpf6_6PBvbq",
       "fillStyle": "hachure",
@@ -1039,17 +1162,21 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176187,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "{____}",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "{____}"
+      "originalText": "{____}",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     }
   ],
   "appState": {

docs/images/faq.excalidraw

@@ -0,0 +1,367 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "ellipse",
+      "version": 236,
+      "versionNonce": 266746677,
+      "isDeleted": false,
+      "id": "pLSV--mc1HnQu2oyARr4o",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 593.4166948358098,
+      "y": -409.48426474916266,
+      "strokeColor": "#ff7043",
+      "backgroundColor": "#ff7043",
+      "width": 100,
+      "height": 92,
+      "seed": 513066133,
+      "groupIds": [
+        "agHSoiBea3xOkqCmOPa91",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 281,
+      "versionNonce": 1964375707,
+      "isDeleted": false,
+      "id": "n2qByon4fJyq2-FZxQqM0",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 629.5250284234563,
+      "y": -397.41226924952065,
+      "strokeColor": "#ff7043",
+      "backgroundColor": "#2e303e",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 852571451,
+      "groupIds": [
+        "agHSoiBea3xOkqCmOPa91",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    },
+    {
+      "type": "ellipse",
+      "version": 288,
+      "versionNonce": 239275157,
+      "isDeleted": false,
+      "id": "QIRNCrcWupjspegQn6VDV",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 744.4166948358098,
+      "y": -409.48426474916266,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 100,
+      "height": 92,
+      "seed": 1498649243,
+      "groupIds": [
+        "yonc0SR8jSwQLwuVvb9hG",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 372,
+      "versionNonce": 1982180597,
+      "isDeleted": false,
+      "id": "a-CPMQS9ApW4f7p9FjS8m",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.5250284234563,
+      "y": -397.41226924952065,
+      "strokeColor": "#ffd43b",
+      "backgroundColor": "#ffeb3b",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 1679591227,
+      "groupIds": [
+        "yonc0SR8jSwQLwuVvb9hG",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313209123,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    },
+    {
+      "type": "ellipse",
+      "version": 326,
+      "versionNonce": 1069462005,
+      "isDeleted": false,
+      "id": "nsYgl7js3PjMKKTKcLYf1",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 895.4166948358098,
+      "y": -409.48426474916266,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 100,
+      "height": 92,
+      "seed": 384682581,
+      "groupIds": [
+        "M7MHx-aSMrw_q1mhaAuG6",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 371,
+      "versionNonce": 641029083,
+      "isDeleted": false,
+      "id": "a_CBcLegs1ZMstjO9f4bx",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 931.5250284234563,
+      "y": -397.41226924952065,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 1741404085,
+      "groupIds": [
+        "M7MHx-aSMrw_q1mhaAuG6",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    },
+    {
+      "type": "ellipse",
+      "version": 328,
+      "versionNonce": 22981461,
+      "isDeleted": false,
+      "id": "l88Tj2DPIgTUA2Pij6hPA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1046.4166948358097,
+      "y": -409.48426474916266,
+      "strokeColor": "#7950f2",
+      "backgroundColor": "#7950f2",
+      "width": 100,
+      "height": 92,
+      "seed": 701954677,
+      "groupIds": [
+        "vFCiKG5iTbg2K5S2WQSsn",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 373,
+      "versionNonce": 1236541563,
+      "isDeleted": false,
+      "id": "oqmBYU6SpmJhOl9d1JLJy",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1082.5250284234562,
+      "y": -397.41226924952065,
+      "strokeColor": "#7950f2",
+      "backgroundColor": "#7950f2",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 39445461,
+      "groupIds": [
+        "vFCiKG5iTbg2K5S2WQSsn",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    },
+    {
+      "type": "ellipse",
+      "version": 329,
+      "versionNonce": 1937275061,
+      "isDeleted": false,
+      "id": "IFZTHrjZaEk7Avy2zkma-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1197.4166948358097,
+      "y": -409.48426474916266,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 100,
+      "height": 92,
+      "seed": 894921429,
+      "groupIds": [
+        "70qQx7edpvpMfsvEta9mo",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 374,
+      "versionNonce": 2029017371,
+      "isDeleted": false,
+      "id": "p0xoCeTYO1ZVy-xm2oET3",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1233.5250284234562,
+      "y": -397.41226924952065,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 959268917,
+      "groupIds": [
+        "70qQx7edpvpMfsvEta9mo",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

docs/images/flows.excalidraw

@@ -0,0 +1,793 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "text",
+      "version": 603,
+      "versionNonce": 1883892902,
+      "isDeleted": false,
+      "id": "Buic2Lx427wuSIW8P_Rw5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 714,
+      "y": 141,
+      "strokeColor": "#000000",
+      "backgroundColor": "#228be6",
+      "width": 532,
+      "height": 46,
+      "seed": 373648901,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "fontSize": 36,
+      "fontFamily": 1,
+      "text": "What are semantic workflows?",
+      "baseline": 32,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "What are semantic workflows?"
+    },
+    {
+      "type": "rectangle",
+      "version": 1671,
+      "versionNonce": 51714234,
+      "isDeleted": false,
+      "id": "qYd3q0Vjks7VOHUC9RR51",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 425,
+      "y": 339.5,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 290.0000000000001,
+      "height": 46,
+      "seed": 1441952427,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "WPeWn6N4rCHf0jY16N9Ge"
+        }
+      ],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "arrow",
+      "version": 323,
+      "versionNonce": 921790438,
+      "isDeleted": false,
+      "id": "tgnQzXC9s8RY4oImBOXuB",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 562.5285999651802,
+      "y": 272.4578674923631,
+      "strokeColor": "#000",
+      "backgroundColor": "#228be6",
+      "width": 0.4714000348197942,
+      "height": 72.5421325076369,
+      "seed": 650463755,
+      "groupIds": [],
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "B435ajoI5vAQBDvzkd8aY",
+        "focus": 0.05963733900881362,
+        "gap": 5.457867492363107
+      },
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          0.4714000348197942,
+          72.5421325076369
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 1359,
+      "versionNonce": 2032529786,
+      "isDeleted": false,
+      "id": "WPeWn6N4rCHf0jY16N9Ge",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 430,
+      "y": 344.5,
+      "strokeColor": "#000000",
+      "backgroundColor": "#fa5252",
+      "width": 280,
+      "height": 36,
+      "seed": 870516459,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Translate",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "qYd3q0Vjks7VOHUC9RR51",
+      "originalText": "Translate"
+    },
+    {
+      "type": "rectangle",
+      "version": 180,
+      "versionNonce": 159873830,
+      "isDeleted": false,
+      "id": "IxMxusRKX2PpnJT1uY0cC",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 819,
+      "y": 217.5,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 290.0000000000001,
+      "height": 49,
+      "seed": 1364021803,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "SSXjX_URKvcVC8h-Qgz4j"
+        },
+        {
+          "id": "_YNRYTAnVzdhhKxFtkVyg",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 85,
+      "versionNonce": 1360900666,
+      "isDeleted": false,
+      "id": "SSXjX_URKvcVC8h-Qgz4j",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 824,
+      "y": 224,
+      "strokeColor": "#000000",
+      "backgroundColor": "#82c91e",
+      "width": 280,
+      "height": 36,
+      "seed": 948915563,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Extract Text",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "IxMxusRKX2PpnJT1uY0cC",
+      "originalText": "Extract Text"
+    },
+    {
+      "type": "rectangle",
+      "version": 217,
+      "versionNonce": 511256166,
+      "isDeleted": false,
+      "id": "lATIKISgJPOGnUHleuFRH",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 821,
+      "y": 338,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 290.0000000000001,
+      "height": 46,
+      "seed": 1256311595,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "21WiUuyDtpQ9FnJ74REpJ",
+          "type": "text"
+        },
+        {
+          "id": "Q51e0Hav-kYfjX3b8tt-r",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 96,
+      "versionNonce": 2111789818,
+      "isDeleted": false,
+      "id": "21WiUuyDtpQ9FnJ74REpJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 826,
+      "y": 343,
+      "strokeColor": "#000000",
+      "backgroundColor": "#fa5252",
+      "width": 280,
+      "height": 36,
+      "seed": 626266373,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Summarize",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "lATIKISgJPOGnUHleuFRH",
+      "originalText": "Summarize"
+    },
+    {
+      "type": "arrow",
+      "version": 162,
+      "versionNonce": 1452501414,
+      "isDeleted": false,
+      "id": "_YNRYTAnVzdhhKxFtkVyg",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 962.2340210300499,
+      "y": 270,
+      "strokeColor": "#000",
+      "backgroundColor": "#228be6",
+      "width": 0.2659789699500834,
+      "height": 75,
+      "seed": 101249451,
+      "groupIds": [],
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "IxMxusRKX2PpnJT1uY0cC",
+        "focus": 0.012856281024868153,
+        "gap": 3.5
+      },
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          0.2659789699500834,
+          75
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 225,
+      "versionNonce": 502131642,
+      "isDeleted": false,
+      "id": "6B3J0wJfi561c9gMsgtXG",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 824,
+      "y": 455,
+      "strokeColor": "#7950f2",
+      "backgroundColor": "#7950f2",
+      "width": 290.0000000000001,
+      "height": 46,
+      "seed": 1164190923,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "qJDqmDEZF9Ewh2UrjXq5Z",
+          "type": "text"
+        },
+        {
+          "id": "Q51e0Hav-kYfjX3b8tt-r",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 147,
+      "versionNonce": 1896039654,
+      "isDeleted": false,
+      "id": "qJDqmDEZF9Ewh2UrjXq5Z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 829,
+      "y": 460,
+      "strokeColor": "#000000",
+      "backgroundColor": "#fa5252",
+      "width": 280,
+      "height": 36,
+      "seed": 1307943269,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Build Vector Index",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "6B3J0wJfi561c9gMsgtXG",
+      "originalText": "Build Vector Index"
+    },
+    {
+      "type": "arrow",
+      "version": 215,
+      "versionNonce": 124544122,
+      "isDeleted": false,
+      "id": "Q51e0Hav-kYfjX3b8tt-r",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 963.600775377322,
+      "y": 387.5,
+      "strokeColor": "#000",
+      "backgroundColor": "#228be6",
+      "width": 0.29392358842710564,
+      "height": 66.5,
+      "seed": 59173285,
+      "groupIds": [],
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "lATIKISgJPOGnUHleuFRH",
+        "focus": 0.015253485349599789,
+        "gap": 3.5
+      },
+      "endBinding": {
+        "elementId": "6B3J0wJfi561c9gMsgtXG",
+        "focus": -0.039966641220930875,
+        "gap": 1
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.29392358842710564,
+          66.5
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 1258,
+      "versionNonce": 1062582310,
+      "isDeleted": false,
+      "id": "5VuUdI_BsJ5pyE1nTqJUI",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1195,
+      "y": 221,
+      "strokeColor": "#7950f2",
+      "backgroundColor": "#7950f2",
+      "width": 290.0000000000001,
+      "height": 46,
+      "seed": 1044404613,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "bJJ9SGsJsvT071qBBH0w5",
+          "type": "text"
+        },
+        {
+          "id": "Q51e0Hav-kYfjX3b8tt-r",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1443,
+      "versionNonce": 1411237178,
+      "isDeleted": false,
+      "id": "bJJ9SGsJsvT071qBBH0w5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1200,
+      "y": 226,
+      "strokeColor": "#000000",
+      "backgroundColor": "#fa5252",
+      "width": 280,
+      "height": 36,
+      "seed": 128953675,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Run similarity query",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "5VuUdI_BsJ5pyE1nTqJUI",
+      "originalText": "Run similarity query"
+    },
+    {
+      "type": "rectangle",
+      "version": 359,
+      "versionNonce": 1029930726,
+      "isDeleted": false,
+      "id": "pSbgtf1qAB7tWl-pTld7e",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1197,
+      "y": 343,
+      "strokeColor": "#ff7043",
+      "backgroundColor": "#ff7043",
+      "width": 290.0000000000001,
+      "height": 46,
+      "seed": 210689733,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "pQzKSM3audka1kiQm_Sku",
+          "type": "text"
+        },
+        {
+          "id": "Q51e0Hav-kYfjX3b8tt-r",
+          "type": "arrow"
+        },
+        {
+          "id": "k67YzXNtt1GLh4i8Es5zJ",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1673791251799,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 320,
+      "versionNonce": 660337658,
+      "isDeleted": false,
+      "id": "pQzKSM3audka1kiQm_Sku",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1219.5,
+      "y": 348,
+      "strokeColor": "#000",
+      "backgroundColor": "#fa5252",
+      "width": 245,
+      "height": 36,
+      "seed": 1869028363,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791256722,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Send notifications",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "pSbgtf1qAB7tWl-pTld7e",
+      "originalText": "Send notifications"
+    },
+    {
+      "type": "arrow",
+      "version": 329,
+      "versionNonce": 251216122,
+      "isDeleted": false,
+      "id": "k67YzXNtt1GLh4i8Es5zJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1339.1171359117898,
+      "y": 265.5,
+      "strokeColor": "#000",
+      "backgroundColor": "#228be6",
+      "width": 1.0119492860442278,
+      "height": 76.5,
+      "seed": 1656816747,
+      "groupIds": [],
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1673791249246,
+      "link": null,
+      "locked": false,
+      "startBinding": null,
+      "endBinding": {
+        "elementId": "pSbgtf1qAB7tWl-pTld7e",
+        "focus": -0.010690950588675632,
+        "gap": 1
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          1.0119492860442278,
+          76.5
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 320,
+      "versionNonce": 407191226,
+      "isDeleted": false,
+      "id": "kKPPLMCj8QQIJLbW-B5hm",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1180,
+      "y": 451,
+      "strokeColor": "#000",
+      "backgroundColor": "#fab005",
+      "width": 296,
+      "height": 52,
+      "seed": 1079990731,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 1,
+      "text": "- API bindings for JavaScript,\n  Rust, Go and Java",
+      "baseline": 44,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- API bindings for JavaScript,\n  Rust, Go and Java"
+    },
+    {
+      "type": "text",
+      "version": 572,
+      "versionNonce": 1757323750,
+      "isDeleted": false,
+      "id": "1AQ3rj-V4weRtPA8a5-z-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 437.5,
+      "y": 449.5,
+      "strokeColor": "#000",
+      "backgroundColor": "#fab005",
+      "width": 278,
+      "height": 52,
+      "seed": 836512075,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 1,
+      "text": "- Build with Python or YAML\n- Run local or via API",
+      "baseline": 44,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Build with Python or YAML\n- Run local or via API"
+    },
+    {
+      "type": "rectangle",
+      "version": 304,
+      "versionNonce": 23156602,
+      "isDeleted": false,
+      "id": "B435ajoI5vAQBDvzkd8aY",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 426,
+      "y": 221,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 290.0000000000001,
+      "height": 46,
+      "seed": 942981672,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "AoGnxEHn4x-zq2-0C0VrT",
+          "type": "text"
+        },
+        {
+          "id": "Q51e0Hav-kYfjX3b8tt-r",
+          "type": "arrow"
+        },
+        {
+          "id": "tgnQzXC9s8RY4oImBOXuB",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 183,
+      "versionNonce": 1100838182,
+      "isDeleted": false,
+      "id": "AoGnxEHn4x-zq2-0C0VrT",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 431,
+      "y": 226,
+      "strokeColor": "#000000",
+      "backgroundColor": "#fa5252",
+      "width": 280,
+      "height": 36,
+      "seed": 604886104,
+      "groupIds": [],
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1673791247706,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Summarize",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "B435ajoI5vAQBDvzkd8aY",
+      "originalText": "Summarize"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#3030"
+  },
+  "files": {}
+}

docs/images/format.excalidraw

@@ -0,0 +1,446 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 564,
+      "versionNonce": 555415043,
+      "index": "aZV",
+      "isDeleted": false,
+      "id": "c8q6be5K81xmCJ-ukSRnZ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 518,
+      "y": -130.82625000000002,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 261,
+      "height": 183,
+      "seed": 443859885,
+      "groupIds": [
+        "2ZFdM1ONmx5lh5aHlDXqH"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584360105,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 518,
+      "versionNonce": 1253679021,
+      "index": "aa",
+      "isDeleted": false,
+      "id": "uKvu2zcnLI9S3D-aFErJQ",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 616.6083333333333,
+      "y": -129.82625000000002,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#03a9f4",
+      "width": 53.78333333333333,
+      "height": 35,
+      "seed": 848196621,
+      "groupIds": [
+        "2ZFdM1ONmx5lh5aHlDXqH"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584360105,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 5,
+      "text": "ANN",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "ANN",
+      "autoResize": true,
+      "lineHeight": 1.25
+    },
+    {
+      "type": "text",
+      "version": 480,
+      "versionNonce": 944506682,
+      "index": "ab",
+      "isDeleted": false,
+      "id": "DCqImwTRCBh0nmPIv-3zR",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 536.8249969482422,
+      "y": -85.17374999999998,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#03a9f4",
+      "width": 213.35000610351562,
+      "height": 125,
+      "seed": 1618350307,
+      "groupIds": [
+        "2ZFdM1ONmx5lh5aHlDXqH"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724587151711,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 5,
+      "text": "- Faiss\n- HNSWLib\n- Annoy\n- NumPy\n- Postgres (pgvector)",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Faiss\n- HNSWLib\n- Annoy\n- NumPy\n- Postgres (pgvector)",
+      "autoResize": true,
+      "lineHeight": 1.25
+    },
+    {
+      "type": "rectangle",
+      "version": 780,
+      "versionNonce": 1085123203,
+      "index": "abV",
+      "isDeleted": false,
+      "id": "Jo5LO2bSebtz54pWb3XHT",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 810.5,
+      "y": -131.82625000000002,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 282.9999999999999,
+      "height": 182,
+      "seed": 848620899,
+      "groupIds": [
+        "hCyHVfABhzmE_8xEhUbGD"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584372798,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 717,
+      "versionNonce": 708473645,
+      "index": "ac",
+      "isDeleted": false,
+      "id": "wX4udz8nceRrNC3uUDLYf",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 886.2583312988281,
+      "y": -121.82625000000002,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#03a9f4",
+      "width": 131.48333740234375,
+      "height": 35,
+      "seed": 16206083,
+      "groupIds": [
+        "hCyHVfABhzmE_8xEhUbGD"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584372798,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 5,
+      "text": "Database",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "Database",
+      "autoResize": true,
+      "lineHeight": 1.25
+    },
+    {
+      "type": "text",
+      "version": 722,
+      "versionNonce": 1181998310,
+      "index": "ad",
+      "isDeleted": false,
+      "id": "-5HdWsMbtE_7EnREtXBfc",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 831.1999969482422,
+      "y": -77.17374999999998,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#03a9f4",
+      "width": 241.60000610351562,
+      "height": 75,
+      "seed": 1976381603,
+      "groupIds": [
+        "hCyHVfABhzmE_8xEhUbGD"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724587158353,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 5,
+      "text": "- SQLite\n- DuckDB\n- Postgres (SQLAlchemy)",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- SQLite\n- DuckDB\n- Postgres (SQLAlchemy)",
+      "autoResize": true,
+      "lineHeight": 1.25
+    },
+    {
+      "type": "rectangle",
+      "version": 900,
+      "versionNonce": 1747610221,
+      "index": "adV",
+      "isDeleted": false,
+      "id": "AK1nt4FrnmjinS2uZVZ2g",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1125,
+      "y": -132.32625000000002,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 266.9999999999999,
+      "height": 180,
+      "seed": 961514755,
+      "groupIds": [
+        "KOc5NUqUBL03A8qlCEUgw"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584372798,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 828,
+      "versionNonce": 1440317155,
+      "index": "ae",
+      "isDeleted": false,
+      "id": "yeFldcgggtNS8S44yWdtq",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1218.7916666666667,
+      "y": -129.32625000000002,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#ffeb3b",
+      "width": 79.41666666666667,
+      "height": 35,
+      "seed": 41743523,
+      "groupIds": [
+        "KOc5NUqUBL03A8qlCEUgw"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584372798,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 5,
+      "text": "Graph",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "Graph",
+      "autoResize": true,
+      "lineHeight": 1.25
+    },
+    {
+      "type": "text",
+      "version": 937,
+      "versionNonce": 766285005,
+      "index": "af",
+      "isDeleted": false,
+      "id": "FSHrZ4DMc-ubEPZmgY7M3",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1128.8999938964844,
+      "y": -78.67374999999998,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#ffeb3b",
+      "width": 259.20001220703125,
+      "height": 75,
+      "seed": 2094819395,
+      "groupIds": [
+        "KOc5NUqUBL03A8qlCEUgw"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584372798,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 5,
+      "text": "- NetworkX (MessagePack)\n- Postgres (grand-graph)\n",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- NetworkX (MessagePack)\n- Postgres (grand-graph)\n",
+      "autoResize": true,
+      "lineHeight": 1.25
+    },
+    {
+      "type": "rectangle",
+      "version": 784,
+      "versionNonce": 868973283,
+      "index": "afV",
+      "isDeleted": false,
+      "id": "C_7EOPOUjQk0kE44hF9WC",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1423.5,
+      "y": -130.82625000000002,
+      "strokeColor": "#9775fa",
+      "backgroundColor": "#9775fa",
+      "width": 290.9999999999999,
+      "height": 174,
+      "seed": 606086243,
+      "groupIds": [
+        "-E2Q3MkgggNSCRYPrSvhr"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584154381,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 712,
+      "versionNonce": 1558730957,
+      "index": "ag",
+      "isDeleted": false,
+      "id": "nbfo2fj-5l1xNK7r9SnT8",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1520.2333335876465,
+      "y": -127.82625000000002,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#ffeb3b",
+      "width": 97.53333282470703,
+      "height": 35,
+      "seed": 704880643,
+      "groupIds": [
+        "-E2Q3MkgggNSCRYPrSvhr"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584154381,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 5,
+      "text": "Scoring",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "Scoring",
+      "autoResize": true,
+      "lineHeight": 1.25
+    },
+    {
+      "type": "text",
+      "version": 832,
+      "versionNonce": 2104267693,
+      "index": "ah",
+      "isDeleted": false,
+      "id": "6a1UVKWBLqgnaAvPSuSfw",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1431.6750030517578,
+      "y": -76.17374999999998,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#ffeb3b",
+      "width": 274.6499938964844,
+      "height": 75,
+      "seed": 154576803,
+      "groupIds": [
+        "-E2Q3MkgggNSCRYPrSvhr"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1724584347937,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 5,
+      "text": "- Local index (MessagePack)\n- Postgres\n",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Local index (MessagePack)\n- Postgres\n",
+      "autoResize": true,
+      "lineHeight": 1.25
+    }
+  ],
+  "appState": {
+    "gridSize": 20,
+    "gridStep": 5,
+    "gridModeEnabled": false,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

docs/images/install.excalidraw

@@ -5,11 +5,11 @@
   "elements": [
     {
       "type": "rectangle",
-      "version": 1431,
-      "versionNonce": 726709768,
+      "version": 1435,
+      "versionNonce": 1860045045,
       "isDeleted": false,
       "id": "LgVOdCQnXRH2GYHIGLwOx",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -25,17 +25,20 @@
       "groupIds": [
         "s5mCNZzJxsETmWhZoD3qR"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641754344717
+      "updated": 1688311151676,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1531,
-      "versionNonce": 2009304696,
+      "version": 1534,
+      "versionNonce": 2090527963,
       "isDeleted": false,
       "id": "GG_mFtlk6nEiYCQ8vhzjZ",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -51,41 +54,48 @@
       "groupIds": [
         "Rezl6DUXkEY8GT9kuDOQX"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641754344717
+      "updated": 1688311151677,
+      "link": null,
+      "locked": false
     },
     {
-      "id": "NLfZWFLgvJj3ccuF5yxD8",
       "type": "text",
-      "x": 793,
-      "y": 216.86805555555557,
-      "width": 80,
-      "height": 16,
-      "angle": 0,
-      "strokeColor": "#343a40",
-      "backgroundColor": "transparent",
+      "version": 293,
+      "versionNonce": 1618682453,
+      "isDeleted": false,
+      "id": "NLfZWFLgvJj3ccuF5yxD8",
       "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
-      "groupIds": [],
-      "strokeSharpness": "sharp",
+      "angle": 0,
+      "x": 793,
+      "y": 216.86805555555557,
+      "strokeColor": "#343a40",
+      "backgroundColor": "transparent",
+      "width": 80,
+      "height": 16,
       "seed": 297813880,
-      "version": 288,
-      "versionNonce": 145457528,
-      "isDeleted": false,
-      "boundElements": null,
-      "updated": 1641754464371,
-      "text": "Installing.......",
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688311151677,
+      "link": null,
+      "locked": false,
       "fontSize": 12.242424242424246,
       "fontFamily": 1,
+      "text": "Installing.......",
       "textAlign": "left",
       "verticalAlign": "top",
-      "baseline": 11,
       "containerId": null,
-      "originalText": "Installing......."
+      "originalText": "Installing.......",
+      "lineHeight": 1.3069306930693065,
+      "baseline": 11
     }
   ],
   "appState": {

docs/images/models.excalidraw

@@ -0,0 +1,531 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "line",
+      "version": 1608,
+      "versionNonce": 602661205,
+      "isDeleted": false,
+      "id": "yVN5CxzfpwAZ0J0GEsEim",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1702.2261496777674,
+      "y": -5.340583377694145,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 191.85486538200286,
+      "height": 186.6125755399512,
+      "seed": 1276926237,
+      "groupIds": [
+        "ZD4Gm6iTkUS0-nxoYPoXN"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688309214322,
+      "link": null,
+      "locked": false,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": null,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          0,
+          0
+        ],
+        [
+          -4.946284888409469,
+          1.310194580443298
+        ],
+        [
+          -71.92848036094448,
+          33.259712239166916
+        ],
+        [
+          -71.92848036094448,
+          33.259712239166916
+        ],
+        [
+          -76.36916810848369,
+          36.77184458512375
+        ],
+        [
+          -78.84005658267724,
+          41.86612631945928
+        ],
+        [
+          -95.23551981468633,
+          113.57223168992945
+        ],
+        [
+          -95.23551981468633,
+          113.57223168992945
+        ],
+        [
+          -95.35947746324585,
+          118.64172405287644
+        ],
+        [
+          -93.4324997019961,
+          123.35061560390292
+        ],
+        [
+          -92.71129163005317,
+          124.36030246259705
+        ],
+        [
+          -46.433776844719105,
+          181.9006920694186
+        ],
+        [
+          -46.433776844719105,
+          181.9006920694186
+        ],
+        [
+          -41.973552779866964,
+          185.39027898361584
+        ],
+        [
+          -36.44504150185128,
+          186.6125755399512
+        ],
+        [
+          37.77928857887258,
+          186.61257553995074
+        ],
+        [
+          37.77928857887258,
+          186.61257553995074
+        ],
+        [
+          43.320564233786016,
+          185.34295850958426
+        ],
+        [
+          47.78003745293803,
+          181.84659678914795
+        ],
+        [
+          94.03350816005033,
+          124.29419365112881
+        ],
+        [
+          94.03350816005033,
+          124.29419365112881
+        ],
+        [
+          96.49538791875702,
+          119.21118606558457
+        ],
+        [
+          96.43753267834124,
+          113.56021815873193
+        ],
+        [
+          79.9338983420456,
+          41.794005010746
+        ],
+        [
+          79.9338983420456,
+          41.794005010746
+        ],
+        [
+          77.45849189744823,
+          36.69972327641062
+        ],
+        [
+          73.02231208993142,
+          33.18759093045337
+        ],
+        [
+          6.148302879201523,
+          1.3162045700933498
+        ],
+        [
+          6.148302879201523,
+          1.3162045700933498
+        ],
+        [
+          -0.060096314222328626,
+          -3.469446951953614e-18
+        ],
+        [
+          0,
+          0
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 2051,
+      "versionNonce": 216616981,
+      "isDeleted": false,
+      "id": "ZTKEryJuyeKfeG4nAI4_e",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 2003.5691665051115,
+      "y": 81.12265056081542,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 30.25969046240687,
+      "height": 75.64922615601705,
+      "seed": 1188509340,
+      "groupIds": [
+        "SFux9iuPQVutxOMe0wlW3"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688309215087,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "rectangle",
+      "version": 2192,
+      "versionNonce": 1390810587,
+      "isDeleted": false,
+      "id": "CckqOoWddQBHuvidbpk4P",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 2066.323943610997,
+      "y": 9.547190432552021,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 30.25969046240682,
+      "height": 151.29845231203416,
+      "seed": 331107492,
+      "groupIds": [
+        "SFux9iuPQVutxOMe0wlW3"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "leDc6Y5qdU0OH3j90C072",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1688308769366,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "rectangle",
+      "version": 2072,
+      "versionNonce": 1184041301,
+      "isDeleted": false,
+      "id": "J8ox4BBdzEg-04DGRL-jh",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 2123.5254786767036,
+      "y": 54.936726126162256,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 30.259690462406844,
+      "height": 105.90891661842389,
+      "seed": 1316437916,
+      "groupIds": [
+        "SFux9iuPQVutxOMe0wlW3"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "5pptt-wBgcgbxd5LUka3S",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1688308769366,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "rectangle",
+      "version": 2259,
+      "versionNonce": 1727832699,
+      "isDeleted": false,
+      "id": "V54vdkZmQeI13AEahvNPV",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 2177.649637157211,
+      "y": 22.527960822066646,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 30.25969046240682,
+      "height": 136.16860708083075,
+      "seed": 1025829916,
+      "groupIds": [
+        "SFux9iuPQVutxOMe0wlW3"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688308769366,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "arrow",
+      "version": 442,
+      "versionNonce": 1922817787,
+      "isDeleted": false,
+      "id": "Inol-LWi8GThocPMvSZGX",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1374.656480958798,
+      "y": 99.23064378133208,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "transparent",
+      "width": 216.36132864000353,
+      "height": 0.12505261106101684,
+      "seed": 1653816315,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688309230624,
+      "link": null,
+      "locked": false,
+      "startBinding": null,
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          216.36132864000353,
+          -0.12505261106101684
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 917,
+      "versionNonce": 151223483,
+      "isDeleted": false,
+      "id": "5pptt-wBgcgbxd5LUka3S",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1802.88085185622,
+      "y": 95.10577155105176,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#34bbde",
+      "width": 184.23585514045453,
+      "height": 3.170483019063738,
+      "seed": 1717004827,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688309218767,
+      "link": null,
+      "locked": false,
+      "startBinding": null,
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          184.23585514045453,
+          3.170483019063738
+        ]
+      ]
+    },
+    {
+      "type": "line",
+      "version": 4226,
+      "versionNonce": 789213499,
+      "isDeleted": false,
+      "id": "pikmP5MLN0MZlpnSik5KN",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1235.1617590461694,
+      "y": 25.06522848370164,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 127.43688004487302,
+      "height": 164.47752527518452,
+      "seed": 844586005,
+      "groupIds": [
+        "xkmNAKuTv_iYXPLj0sxbd"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688309232141,
+      "link": null,
+      "locked": false,
+      "startBinding": null,
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": null,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          0.4202213858663673,
+          124.31122212244284
+        ],
+        [
+          0.019665374373081804,
+          138.46351494441197
+        ],
+        [
+          6.563281167657464,
+          144.5785456702537
+        ],
+        [
+          29.351096662508862,
+          149.75384171231534
+        ],
+        [
+          67.86895989535736,
+          151.36535158937204
+        ],
+        [
+          104.67023109801114,
+          148.79234001109992
+        ],
+        [
+          124.22301744981843,
+          142.63902935804376
+        ],
+        [
+          126.98067338289118,
+          137.45172392705157
+        ],
+        [
+          127.36797617588668,
+          126.05781617962573
+        ],
+        [
+          127.06398851859365,
+          10.429017682904831
+        ],
+        [
+          126.37870276277025,
+          -0.4957733094390701
+        ],
+        [
+          118.19596944321839,
+          -6.601710860670866
+        ],
+        [
+          100.96487933911183,
+          -10.137946798406993
+        ],
+        [
+          61.697553127560816,
+          -13.11217368581247
+        ],
+        [
+          30.215116414714352,
+          -11.338635495813458
+        ],
+        [
+          5.454393748609269,
+          -5.323010354025768
+        ],
+        [
+          -0.06890386898634299,
+          -0.074694110077688
+        ],
+        [
+          0,
+          0
+        ]
+      ]
+    },
+    {
+      "type": "ellipse",
+      "version": 5067,
+      "versionNonce": 1912378357,
+      "isDeleted": false,
+      "id": "sPZv-4m6hv0RYSW1Kzrrb",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1235.9555759014816,
+      "y": 13.472824531333586,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "transparent",
+      "width": 126.61947902597214,
+      "height": 25.607837035548464,
+      "seed": 194842997,
+      "groupIds": [
+        "xkmNAKuTv_iYXPLj0sxbd"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688309232141,
+      "link": null,
+      "locked": false
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

docs/images/query.excalidraw

@@ -0,0 +1,243 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 3038,
+      "versionNonce": 1386987419,
+      "isDeleted": false,
+      "id": "pjs0X-Y0kuuXB-vpf_dvP",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 934.5,
+      "y": -240.29250000000002,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 312,
+      "height": 146,
+      "seed": 1534167490,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "id": "of5l7HpIH0kPvZ95rOMwM",
+          "type": "text"
+        },
+        {
+          "id": "Qzp41i_jzQIBlAB_qFKFH",
+          "type": "arrow"
+        },
+        {
+          "id": "of5l7HpIH0kPvZ95rOMwM",
+          "type": "text"
+        },
+        {
+          "type": "text",
+          "id": "of5l7HpIH0kPvZ95rOMwM"
+        }
+      ],
+      "updated": 1669033845173,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 2554,
+      "versionNonce": 1319492501,
+      "isDeleted": false,
+      "id": "of5l7HpIH0kPvZ95rOMwM",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 939.5,
+      "y": -235.29250000000002,
+      "strokeColor": "#000",
+      "backgroundColor": "#fa5252",
+      "width": 260,
+      "height": 130,
+      "seed": 1972792158,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1669033845173,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 1,
+      "text": "SELECT\n  count(*), flag\nFROM txtai\nGROUP BY flag\nORDER BY count(*) DESC",
+      "baseline": 122,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": "pjs0X-Y0kuuXB-vpf_dvP",
+      "originalText": "SELECT\n  count(*), flag\nFROM txtai\nGROUP BY flag\nORDER BY count(*) DESC"
+    },
+    {
+      "type": "rectangle",
+      "version": 3354,
+      "versionNonce": 1004999739,
+      "isDeleted": false,
+      "id": "xMKXZw8aL27lv21eVGy_C",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 581,
+      "y": -238.29250000000002,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 302,
+      "height": 140,
+      "seed": 1704428866,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "id": "tlu8eCXVp5ApnEG74CF38",
+          "type": "text"
+        },
+        {
+          "id": "Qzp41i_jzQIBlAB_qFKFH",
+          "type": "arrow"
+        },
+        {
+          "type": "text",
+          "id": "tlu8eCXVp5ApnEG74CF38"
+        }
+      ],
+      "updated": 1669033845173,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 2750,
+      "versionNonce": 686792949,
+      "isDeleted": false,
+      "id": "tlu8eCXVp5ApnEG74CF38",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 586,
+      "y": -233.29250000000002,
+      "strokeColor": "#000",
+      "backgroundColor": "#fa5252",
+      "width": 290,
+      "height": 130,
+      "seed": 92268510,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1669033845173,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 1,
+      "text": "SELECT\n  text, flag, entry\nFROM txtai\nWHERE\n  similar('text') AND flag = 1",
+      "baseline": 122,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": "xMKXZw8aL27lv21eVGy_C",
+      "originalText": "SELECT\n  text, flag, entry\nFROM txtai\nWHERE\n  similar('text') AND flag = 1"
+    },
+    {
+      "type": "rectangle",
+      "version": 3169,
+      "versionNonce": 1477367003,
+      "isDeleted": false,
+      "id": "Tsn_ukSxtC7CIwEMUMHxI",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1301.5,
+      "y": -236.79250000000002,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 304,
+      "height": 140,
+      "seed": 919682718,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "id": "t3OC1akz9Oe2w6N8OCavg",
+          "type": "text"
+        },
+        {
+          "id": "Qzp41i_jzQIBlAB_qFKFH",
+          "type": "arrow"
+        },
+        {
+          "id": "t3OC1akz9Oe2w6N8OCavg",
+          "type": "text"
+        },
+        {
+          "id": "t3OC1akz9Oe2w6N8OCavg",
+          "type": "text"
+        },
+        {
+          "type": "text",
+          "id": "t3OC1akz9Oe2w6N8OCavg"
+        }
+      ],
+      "updated": 1669033845173,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 2769,
+      "versionNonce": 345619029,
+      "isDeleted": false,
+      "id": "t3OC1akz9Oe2w6N8OCavg",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1306.5,
+      "y": -231.79250000000002,
+      "strokeColor": "#000",
+      "backgroundColor": "#fa5252",
+      "width": 259,
+      "height": 130,
+      "seed": 1222509122,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1669033845173,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 1,
+      "text": "SELECT\n  translation(text, 'fr')\nFROM txtai\nWHERE\n  similar('feel good story')",
+      "baseline": 122,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": "Tsn_ukSxtC7CIwEMUMHxI",
+      "originalText": "SELECT\n  translation(text, 'fr')\nFROM txtai\nWHERE\n  similar('feel good story')"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

docs/images/search.excalidraw

@@ -0,0 +1,555 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "text",
+      "version": 354,
+      "versionNonce": 1260634512,
+      "isDeleted": false,
+      "id": "Buic2Lx427wuSIW8P_Rw5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 829,
+      "y": 184,
+      "strokeColor": "#000000",
+      "backgroundColor": "#228be6",
+      "width": 452,
+      "height": 46,
+      "seed": 373648901,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1658676789638,
+      "link": null,
+      "locked": false,
+      "fontSize": 36,
+      "fontFamily": 1,
+      "text": "What is semantic search?",
+      "baseline": 32,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "What is semantic search?"
+    },
+    {
+      "type": "rectangle",
+      "version": 786,
+      "versionNonce": 1756557200,
+      "isDeleted": false,
+      "id": "U2NgEIEiFpAlwmv5Xnyzr",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 50,
+      "angle": 0,
+      "x": 555.5,
+      "y": 425.30499999999995,
+      "strokeColor": "#7950f2",
+      "backgroundColor": "#7950f2",
+      "width": 995.3286713286714,
+      "height": 339.0000000000001,
+      "seed": 1946478225,
+      "groupIds": [
+        "C_65R9XVeMQED2nMfIW2D"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1658676804492,
+      "link": "",
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 569,
+      "versionNonce": 1675727216,
+      "isDeleted": false,
+      "id": "fbSO8bnZmAdvIXZxBAtHY",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 570.7237762237763,
+      "y": 439.50080419580416,
+      "strokeColor": "#000",
+      "backgroundColor": "#03a9f4",
+      "width": 967,
+      "height": 312,
+      "seed": 1586673137,
+      "groupIds": [
+        "C_65R9XVeMQED2nMfIW2D"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1658676804492,
+      "link": null,
+      "locked": false,
+      "fontSize": 20.27972027972028,
+      "fontFamily": 1,
+      "text": "Query                  Best Match\n----------------------------------------------------\nfeel good story        Maine man wins $1M from $25 lottery ticket\nclimate change         Canada's last fully intact ice shelf has suddenly collapsed, forming a \n                       Manhattan-sized iceberg\npublic health story     US tops 5 million confirmed virus cases\nwar                    Beijing mobilises invasion craft along coast as Taiwan tensions escalate\nwildlife                 The National Park Service warns against sacrificing slower friends in a\n                        bear attack\nasia                   Beijing mobilises invasion craft along coast as Taiwan tensions escalate\nlucky                   Maine man wins $1M from $25 lottery ticket\ndishonest junk          Make huge profits without work, earn up to $100,000 a day",
+      "baseline": 304,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "Query                  Best Match\n----------------------------------------------------\nfeel good story        Maine man wins $1M from $25 lottery ticket\nclimate change         Canada's last fully intact ice shelf has suddenly collapsed, forming a \n                       Manhattan-sized iceberg\npublic health story     US tops 5 million confirmed virus cases\nwar                    Beijing mobilises invasion craft along coast as Taiwan tensions escalate\nwildlife                 The National Park Service warns against sacrificing slower friends in a\n                        bear attack\nasia                   Beijing mobilises invasion craft along coast as Taiwan tensions escalate\nlucky                   Maine man wins $1M from $25 lottery ticket\ndishonest junk          Make huge profits without work, earn up to $100,000 a day"
+    },
+    {
+      "type": "rectangle",
+      "version": 952,
+      "versionNonce": 581259632,
+      "isDeleted": false,
+      "id": "UO6MS3wSDu7yg2421__LI",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 933,
+      "y": 267,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 214,
+      "height": 49,
+      "seed": 1629565989,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "8sp7H8ijWBlh6aMgZ0XTP"
+        },
+        {
+          "id": "Qzp41i_jzQIBlAB_qFKFH",
+          "type": "arrow"
+        },
+        {
+          "id": "SJ0F0Y81z9hir5qQWAJjk",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1658676831306,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "rectangle",
+      "version": 1611,
+      "versionNonce": 1109835152,
+      "isDeleted": false,
+      "id": "qYd3q0Vjks7VOHUC9RR51",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 549,
+      "y": 267.5,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 219,
+      "height": 52,
+      "seed": 1441952427,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "WPeWn6N4rCHf0jY16N9Ge"
+        },
+        {
+          "id": "Qzp41i_jzQIBlAB_qFKFH",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1658676831306,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1297,
+      "versionNonce": 1725539184,
+      "isDeleted": false,
+      "id": "WPeWn6N4rCHf0jY16N9Ge",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 554,
+      "y": 275.5,
+      "strokeColor": "#000",
+      "backgroundColor": "#fa5252",
+      "width": 209,
+      "height": 36,
+      "seed": 870516459,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1658676831306,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Vectorize",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "qYd3q0Vjks7VOHUC9RR51",
+      "originalText": "Vectorize"
+    },
+    {
+      "type": "rectangle",
+      "version": 1198,
+      "versionNonce": 1201179536,
+      "isDeleted": false,
+      "id": "5VuUdI_BsJ5pyE1nTqJUI",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1332,
+      "y": 268,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 218,
+      "height": 49,
+      "seed": 1044404613,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "id": "bJJ9SGsJsvT071qBBH0w5",
+          "type": "text"
+        },
+        {
+          "id": "bJJ9SGsJsvT071qBBH0w5",
+          "type": "text"
+        },
+        {
+          "id": "Q51e0Hav-kYfjX3b8tt-r",
+          "type": "arrow"
+        },
+        {
+          "type": "text",
+          "id": "bJJ9SGsJsvT071qBBH0w5"
+        },
+        {
+          "id": "SJ0F0Y81z9hir5qQWAJjk",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1658676831306,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1381,
+      "versionNonce": 78886256,
+      "isDeleted": false,
+      "id": "bJJ9SGsJsvT071qBBH0w5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1337,
+      "y": 274.5,
+      "strokeColor": "#000",
+      "backgroundColor": "#fa5252",
+      "width": 208,
+      "height": 36,
+      "seed": 128953675,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1658676831306,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Search",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "5VuUdI_BsJ5pyE1nTqJUI",
+      "originalText": "Search"
+    },
+    {
+      "type": "text",
+      "version": 942,
+      "versionNonce": 429955472,
+      "isDeleted": false,
+      "id": "8sp7H8ijWBlh6aMgZ0XTP",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 938,
+      "y": 273.5,
+      "strokeColor": "#000",
+      "backgroundColor": "transparent",
+      "width": 204,
+      "height": 36,
+      "seed": 1854823263,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1658676831306,
+      "link": null,
+      "locked": false,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Index",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "UO6MS3wSDu7yg2421__LI",
+      "originalText": "Index"
+    },
+    {
+      "type": "text",
+      "version": 382,
+      "versionNonce": 1098614640,
+      "isDeleted": false,
+      "id": "jWJpSXHkTCzRTCA4tbAgv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 548.5,
+      "y": 347.30499999999995,
+      "strokeColor": "#000",
+      "backgroundColor": "#03a9f4",
+      "width": 296,
+      "height": 42,
+      "seed": 1241563487,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1658676831307,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": "- Transform input into numbers\n- Similar concepts have similar values",
+      "baseline": 36,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Transform input into numbers\n- Similar concepts have similar values"
+    },
+    {
+      "type": "text",
+      "version": 381,
+      "versionNonce": 1266083728,
+      "isDeleted": false,
+      "id": "qEnmXs0P_MQE8r4c4OWGh",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 931.5,
+      "y": 346.2074999999999,
+      "strokeColor": "#000",
+      "backgroundColor": "#f44336",
+      "width": 230,
+      "height": 42,
+      "seed": 1038536465,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1658676831307,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": "- Save vectors\n- Store content with vectors",
+      "baseline": 36,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Save vectors\n- Store content with vectors"
+    },
+    {
+      "type": "text",
+      "version": 491,
+      "versionNonce": 2037979504,
+      "isDeleted": false,
+      "id": "1q8bzjK8lnKUZj8_A9v7D",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1243,
+      "y": 349.2074999999999,
+      "strokeColor": "#000",
+      "backgroundColor": "#f44336",
+      "width": 333,
+      "height": 42,
+      "seed": 304472945,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1658676831307,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": "- Find similar vectors with cosine similarity\n- Add rule-based filters using content",
+      "baseline": 36,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Find similar vectors with cosine similarity\n- Add rule-based filters using content"
+    },
+    {
+      "type": "arrow",
+      "version": 1813,
+      "versionNonce": 1230206352,
+      "isDeleted": false,
+      "id": "Qzp41i_jzQIBlAB_qFKFH",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 770.5,
+      "y": 289.8470411964629,
+      "strokeColor": "#000",
+      "backgroundColor": "#f44336",
+      "width": 158.1310513485223,
+      "height": 0.5692601572380909,
+      "seed": 660786897,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "round",
+      "boundElements": [],
+      "updated": 1658676831307,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "qYd3q0Vjks7VOHUC9RR51",
+        "focus": -0.15367587596362536,
+        "gap": 2.5
+      },
+      "endBinding": {
+        "elementId": "UO6MS3wSDu7yg2421__LI",
+        "focus": 0.027437144815141,
+        "gap": 4.3689486514776945
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          158.1310513485223,
+          0.5692601572380909
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 1962,
+      "versionNonce": 1072982896,
+      "isDeleted": false,
+      "id": "SJ0F0Y81z9hir5qQWAJjk",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1149.5,
+      "y": 292.6790761701911,
+      "strokeColor": "#000",
+      "backgroundColor": "#f44336",
+      "width": 181.5,
+      "height": 1.5898915058209013,
+      "seed": 899541905,
+      "groupIds": [
+        "3sURMvhuRfR0M-Q3VRPbg"
+      ],
+      "strokeSharpness": "round",
+      "boundElements": [],
+      "updated": 1658676831307,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "UO6MS3wSDu7yg2421__LI",
+        "focus": 0.08406032225724415,
+        "gap": 2.5
+      },
+      "endBinding": {
+        "elementId": "5VuUdI_BsJ5pyE1nTqJUI",
+        "focus": 0.09327847520504394,
+        "gap": 1
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          181.5,
+          -1.5898915058209013
+        ]
+      ]
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

docs/images/why.excalidraw

@@ -5,8 +5,8 @@
   "elements": [
     {
       "type": "rectangle",
-      "version": 633,
-      "versionNonce": 705603336,
+      "version": 639,
+      "versionNonce": 1876686474,
       "isDeleted": false,
       "id": "mG5J3YEl3JUm8tWD2gLax",
       "fillStyle": "hachure",
@@ -15,22 +15,29 @@
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
-      "x": 840.1017043524607,
+      "x": 840.0920889678455,
       "y": 323.8650942251963,
       "strokeColor": "#ff7043",
       "backgroundColor": "#ff7043",
       "width": 412.0000000000001,
       "height": 54.999999999999964,
       "seed": 322487816,
-      "groupIds": [],
+      "groupIds": [
+        "VYua_NGvuS1eX_KPHCgOc"
+      ],
       "strokeSharpness": "sharp",
       "boundElements": [],
-      "updated": 1641744014625
+      "updated": 1731084295790,
+      "index": "a4",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 202,
-      "versionNonce": 1087974776,
+      "version": 210,
+      "versionNonce": 82579222,
       "isDeleted": false,
       "id": "N9ytCRthOA395FeHYtiWS",
       "fillStyle": "hachure",
@@ -39,17 +46,19 @@
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
-      "x": 1010.3997812755383,
+      "x": 1019.5920889678455,
       "y": 335.8362480713501,
       "strokeColor": "#343a40",
       "backgroundColor": "#03a9f4",
       "width": 53,
       "height": 36,
       "seed": 1396884744,
-      "groupIds": [],
+      "groupIds": [
+        "VYua_NGvuS1eX_KPHCgOc"
+      ],
       "strokeSharpness": "sharp",
       "boundElements": [],
-      "updated": 1641744014625,
+      "updated": 1731084295790,
       "fontSize": 28,
       "fontFamily": 1,
       "text": "API",
@@ -57,138 +66,19 @@
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "API"
+      "originalText": "API",
+      "index": "a5",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false,
+      "autoResize": true,
+      "lineHeight": 1.2857142857142858
     },
     {
       "type": "rectangle",
-      "version": 741,
-      "versionNonce": 917800456,
-      "isDeleted": false,
-      "id": "uFbzRXgEesWgll3llORrW",
-      "fillStyle": "hachure",
-      "strokeWidth": 1,
-      "strokeStyle": "solid",
-      "roughness": 1,
-      "opacity": 100,
-      "angle": 0,
-      "x": 839.3613197370764,
-      "y": 387.49009422519623,
-      "strokeColor": "#00e676",
-      "backgroundColor": "#00e676",
-      "width": 184,
-      "height": 60.846153846153825,
-      "seed": 531948552,
-      "groupIds": [],
-      "strokeSharpness": "sharp",
-      "boundElements": [
-        {
-          "type": "text",
-          "id": "Ctud-Ap642NinH0-64fKT"
-        }
-      ],
-      "updated": 1641744014625
-    },
-    {
-      "type": "text",
-      "version": 509,
-      "versionNonce": 171632136,
-      "isDeleted": false,
-      "id": "Ctud-Ap642NinH0-64fKT",
-      "fillStyle": "cross-hatch",
-      "strokeWidth": 1,
-      "strokeStyle": "solid",
-      "roughness": 1,
-      "opacity": 100,
-      "angle": 0,
-      "x": 844.3613197370764,
-      "y": 399.91317114827314,
-      "strokeColor": "#000000",
-      "backgroundColor": "#03a9f4",
-      "width": 174,
-      "height": 36,
-      "seed": 720254072,
-      "groupIds": [],
-      "strokeSharpness": "sharp",
-      "boundElements": [],
-      "updated": 1641743551018,
-      "fontSize": 28,
-      "fontFamily": 1,
-      "text": "Pipeline",
-      "baseline": 25,
-      "textAlign": "center",
-      "verticalAlign": "middle",
-      "containerId": "uFbzRXgEesWgll3llORrW",
-      "originalText": "Pipeline"
-    },
-    {
-      "type": "rectangle",
-      "version": 726,
-      "versionNonce": 558153336,
-      "isDeleted": false,
-      "id": "_zQO_kyS5Pvq1zDhEn9hv",
-      "fillStyle": "hachure",
-      "strokeWidth": 1,
-      "strokeStyle": "solid",
-      "roughness": 1,
-      "opacity": 100,
-      "angle": 0,
-      "x": 1032.3805505063074,
-      "y": 389.3362480713501,
-      "strokeColor": "#ffeb3b",
-      "backgroundColor": "#ffeb3b",
-      "width": 219,
-      "height": 56,
-      "seed": 1988829960,
-      "groupIds": [],
-      "strokeSharpness": "sharp",
-      "boundElements": [
-        {
-          "id": "tL_gw7MpliV1bDFJp8Cql",
-          "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "tL_gw7MpliV1bDFJp8Cql"
-        }
-      ],
-      "updated": 1641744014625
-    },
-    {
-      "type": "text",
-      "version": 514,
-      "versionNonce": 1694557448,
-      "isDeleted": false,
-      "id": "tL_gw7MpliV1bDFJp8Cql",
-      "fillStyle": "cross-hatch",
-      "strokeWidth": 1,
-      "strokeStyle": "solid",
-      "roughness": 1,
-      "opacity": 100,
-      "angle": 0,
-      "x": 1037.2555505063074,
-      "y": 399.3362480713501,
-      "strokeColor": "#000000",
-      "backgroundColor": "#03a9f4",
-      "width": 209,
-      "height": 36,
-      "seed": 274132856,
-      "groupIds": [],
-      "strokeSharpness": "sharp",
-      "boundElements": [],
-      "updated": 1641743551018,
-      "fontSize": 28,
-      "fontFamily": 1,
-      "text": "Workflow",
-      "baseline": 25,
-      "textAlign": "center",
-      "verticalAlign": "middle",
-      "containerId": "_zQO_kyS5Pvq1zDhEn9hv",
-      "originalText": "Workflow"
-    },
-    {
-      "type": "rectangle",
-      "version": 455,
-      "versionNonce": 289834248,
+      "version": 460,
+      "versionNonce": 1551291798,
       "isDeleted": false,
       "id": "QKBksUpSSix31GSKFO1uV",
       "fillStyle": "hachure",
@@ -197,8 +87,8 @@
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
-      "x": 838.5920889678456,
-      "y": 456.87470960981165,
+      "x": 837.5920889678456,
+      "y": 456.8747096098117,
       "strokeColor": "#03a9f4",
       "backgroundColor": "#03a9f4",
       "width": 417,
@@ -210,22 +100,19 @@
         {
           "id": "4o-d8ZzuqEpB7ROBAyReL",
           "type": "text"
-        },
-        {
-          "id": "4o-d8ZzuqEpB7ROBAyReL",
-          "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "4o-d8ZzuqEpB7ROBAyReL"
         }
       ],
-      "updated": 1641744014625
+      "updated": 1731084298697,
+      "index": "a6",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 258,
-      "versionNonce": 347014152,
+      "version": 263,
+      "versionNonce": 1011157002,
       "isDeleted": false,
       "id": "4o-d8ZzuqEpB7ROBAyReL",
       "fillStyle": "cross-hatch",
@@ -234,8 +121,8 @@
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
-      "x": 843.4670889678456,
-      "y": 469.29778653288855,
+      "x": 842.4670889678456,
+      "y": 469.2977865328886,
       "strokeColor": "#343a40",
       "backgroundColor": "#03a9f4",
       "width": 407,
@@ -244,7 +131,7 @@
       "groupIds": [],
       "strokeSharpness": "sharp",
       "boundElements": [],
-      "updated": 1641743551019,
+      "updated": 1731084298697,
       "fontSize": 28,
       "fontFamily": 1,
       "text": "Embeddings",
@@ -252,11 +139,251 @@
       "textAlign": "center",
       "verticalAlign": "middle",
       "containerId": "QKBksUpSSix31GSKFO1uV",
-      "originalText": "Embeddings"
+      "originalText": "Embeddings",
+      "index": "a7",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false,
+      "autoResize": true,
+      "lineHeight": 1.2857142857142858
+    },
+    {
+      "type": "rectangle",
+      "version": 847,
+      "versionNonce": 266827407,
+      "isDeleted": false,
+      "id": "uFbzRXgEesWgll3llORrW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 969.4670889678455,
+      "y": 387.5069211482732,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 135,
+      "height": 60.846153846153825,
+      "seed": 531948552,
+      "groupIds": [
+        "DlS9eiqyrIHp45FmS35GN"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "Ctud-Ap642NinH0-64fKT"
+        }
+      ],
+      "updated": 1731149308913,
+      "index": "a8",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 617,
+      "versionNonce": 1566326241,
+      "isDeleted": false,
+      "id": "Ctud-Ap642NinH0-64fKT",
+      "fillStyle": "cross-hatch",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 988.8004223011789,
+      "y": 399.9299980713501,
+      "strokeColor": "#000000",
+      "backgroundColor": "#03a9f4",
+      "width": 96.33333333333333,
+      "height": 36,
+      "seed": 720254072,
+      "groupIds": [
+        "DlS9eiqyrIHp45FmS35GN"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1731149308913,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Pipeline",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "uFbzRXgEesWgll3llORrW",
+      "originalText": "Pipeline",
+      "index": "a9",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false,
+      "autoResize": true,
+      "lineHeight": 1.2857142857142858
+    },
+    {
+      "type": "rectangle",
+      "version": 827,
+      "versionNonce": 1640467681,
+      "isDeleted": false,
+      "id": "_zQO_kyS5Pvq1zDhEn9hv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1111.4863197370764,
+      "y": 388.1766044061918,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 141,
+      "height": 59.76470588235287,
+      "seed": 1988829960,
+      "groupIds": [
+        "DlS9eiqyrIHp45FmS35GN"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "id": "tL_gw7MpliV1bDFJp8Cql",
+          "type": "text"
+        }
+      ],
+      "updated": 1731149323000,
+      "index": "aA",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 617,
+      "versionNonce": 1776634049,
+      "isDeleted": false,
+      "id": "tL_gw7MpliV1bDFJp8Cql",
+      "fillStyle": "cross-hatch",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1122.7113197370763,
+      "y": 400.0589573473683,
+      "strokeColor": "#000000",
+      "backgroundColor": "#03a9f4",
+      "width": 118.55,
+      "height": 36,
+      "seed": 274132856,
+      "groupIds": [
+        "DlS9eiqyrIHp45FmS35GN"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1731149323001,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Workflow",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "_zQO_kyS5Pvq1zDhEn9hv",
+      "originalText": "Workflow",
+      "index": "aB",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false,
+      "autoResize": true,
+      "lineHeight": 1.2857142857142858
+    },
+    {
+      "type": "rectangle",
+      "version": 910,
+      "versionNonce": 391148577,
+      "isDeleted": false,
+      "id": "STAkXjIBdAZgXMIKZfGnx",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 839.6978581986145,
+      "y": 387.38672884058093,
+      "strokeColor": "#9775fa",
+      "backgroundColor": "#9775fa",
+      "width": 121,
+      "height": 60.846153846153825,
+      "seed": 362222090,
+      "groupIds": [
+        "DlS9eiqyrIHp45FmS35GN"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "1iyFs-1PG5afe6QKq1s34"
+        }
+      ],
+      "updated": 1731149289560,
+      "index": "aC",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 690,
+      "versionNonce": 456727151,
+      "isDeleted": false,
+      "id": "1iyFs-1PG5afe6QKq1s34",
+      "fillStyle": "cross-hatch",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 861.8895238480286,
+      "y": 399.80980576365783,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#03a9f4",
+      "width": 76.61666870117188,
+      "height": 36,
+      "seed": 565985482,
+      "groupIds": [
+        "DlS9eiqyrIHp45FmS35GN"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1731149289560,
+      "fontSize": 28,
+      "fontFamily": 1,
+      "text": "Agent",
+      "baseline": 25,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "STAkXjIBdAZgXMIKZfGnx",
+      "originalText": "Agent",
+      "index": "aD",
+      "frameId": null,
+      "roundness": null,
+      "link": null,
+      "locked": false,
+      "autoResize": true,
+      "lineHeight": 1.2857142857142858
     }
   ],
   "appState": {
-    "gridSize": null,
+    "gridSize": 20,
+    "gridStep": 5,
+    "gridModeEnabled": false,
     "viewBackgroundColor": "#ffffff"
   },
   "files": {}

docs/index.md

@@ -4,59 +4,57 @@
     <img src="https://raw.githubusercontent.com/neuml/txtai/master/logo.png"/>
 </p>
 
-<h3 align="center">
-    <p>Build AI-powered semantic search applications</p>
-</h3>
+<p align="center">
+    <b>All-in-one AI framework</b>
+</p>
 
 <p align="center">
     <a href="https://github.com/neuml/txtai/releases">
         <img src="https://img.shields.io/github/release/neuml/txtai.svg?style=flat&color=success" alt="Version"/>
     </a>
-    <a href="https://github.com/neuml/txtai/releases">
-        <img src="https://img.shields.io/github/release-date/neuml/txtai.svg?style=flat&color=blue" alt="GitHub Release Date"/>
+    <a href="https://github.com/neuml/txtai">
+        <img src="https://img.shields.io/github/last-commit/neuml/txtai.svg?style=flat&color=blue" alt="GitHub last commit"/>
     </a>
     <a href="https://github.com/neuml/txtai/issues">
         <img src="https://img.shields.io/github/issues/neuml/txtai.svg?style=flat&color=success" alt="GitHub issues"/>
     </a>
-    <a href="https://github.com/neuml/txtai">
-        <img src="https://img.shields.io/github/last-commit/neuml/txtai.svg?style=flat&color=blue" alt="GitHub last commit"/>
+    <a href="https://join.slack.com/t/txtai/shared_invite/zt-37c1zfijp-Y57wMty6YOx_hyIHEQvQJA">
+        <img src="https://img.shields.io/badge/slack-join-blue?style=flat&logo=slack&logocolor=white" alt="Join Slack"/>
     </a>
     <a href="https://github.com/neuml/txtai/actions?query=workflow%3Abuild">
         <img src="https://github.com/neuml/txtai/workflows/build/badge.svg" alt="Build Status"/>
     </a>
     <a href="https://coveralls.io/github/neuml/txtai?branch=master">
-        <img src="https://img.shields.io/coveralls/github/neuml/txtai" alt="Coverage Status">
+        <img src="https://img.shields.io/coverallsCoverage/github/neuml/txtai" alt="Coverage Status">
     </a>
 </p>
 
--------------------------------------------------------------------------------------------------------------------------------------------------------
+txtai is an all-in-one AI framework for semantic search, LLM orchestration and language model workflows.
 
-txtai executes machine-learning workflows to transform data and build AI-powered semantic search applications.
+![architecture](images/architecture.png#gh-light-mode-only)
+![architecture](images/architecture-dark.png#gh-dark-mode-only)
 
-![demo](https://raw.githubusercontent.com/neuml/txtai/master/demo.gif)
+The key component of txtai is an embeddings database, which is a union of vector indexes (sparse and dense), graph networks and relational databases.
 
-Traditional search systems use keywords to find data. Semantic search applications have an understanding of natural language and identify results that have the same meaning, not necessarily the same keywords.
+This foundation enables vector search and/or serves as a powerful knowledge source for large language model (LLM) applications.
 
-Backed by state-of-the-art machine learning models, data is transformed into vector representations for search (also known as embeddings). Innovation is happening at a rapid pace, models can understand concepts in documents, audio, images and more.
+Build autonomous agents, retrieval augmented generation (RAG) processes, multi-model workflows and more.
 
 Summary of txtai features:
 
-- 🔎 Large-scale similarity search with multiple index backends ([Faiss](https://github.com/facebookresearch/faiss), [Annoy](https://github.com/spotify/annoy), [Hnswlib](https://github.com/nmslib/hnswlib))
-- 📄 Create embeddings for text snippets, documents, audio, images and video. Supports transformers and word vectors.
-- 💡 Machine-learning pipelines to run extractive question-answering, zero-shot labeling, transcription, translation, summarization and text extraction
-- ↪️️ Workflows that join pipelines together to aggregate business logic. txtai processes can be microservices or full-fledged indexing workflows.
-- 🔗 API bindings for [JavaScript](https://github.com/neuml/txtai.js), [Java](https://github.com/neuml/txtai.java), [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go)
-- ☁️ Cloud-native architecture that scales out with container orchestration systems (e.g. Kubernetes)
+- 🔎 Vector search with SQL, object storage, topic modeling, graph analysis and multimodal indexing
+- 📄 Create embeddings for text, documents, audio, images and video
+- 💡 Pipelines powered by language models that run LLM prompts, question-answering, labeling, transcription, translation, summarization and more
+- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be simple microservices or multi-model workflows.
+- 🤖 Agents that intelligently connect embeddings, pipelines, workflows and other agents together to autonomously solve complex problems
+- ⚙️ Web and Model Context Protocol (MCP) APIs. Bindings available for [JavaScript](https://github.com/neuml/txtai.js), [Java](https://github.com/neuml/txtai.java), [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go).
+- 🔋 Batteries included with defaults to get up and running fast
+- ☁️ Run local or scale out with container orchestration
 
-Applications range from similarity search to complex NLP-driven data extractions to generate structured databases. The following applications are powered by txtai.
+txtai is built with Python 3.11+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
 
-![apps](https://raw.githubusercontent.com/neuml/txtai/master/apps.jpg)
+!!! note
 
-| Application  | Description  |
-|:----------|:-------------|
-| [paperai](https://github.com/neuml/paperai) | AI-powered literature discovery and review engine for medical/scientific papers |
-| [tldrstory](https://github.com/neuml/tldrstory) | AI-powered understanding of headlines and story text |
-| [neuspo](https://neuspo.com) | Fact-driven, real-time sports event and news site |
-| [codequestion](https://github.com/neuml/codequestion) | Ask coding questions directly from the terminal |
+    [NeuML](https://neuml.com) is the company behind txtai and we provide AI consulting services around our stack. [Schedule a meeting](https://cal.com/neuml/intro) or [send a message](mailto:[email protected]) to learn more.
 
-txtai is built with Python 3.7+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi)
+    We're also building an easy and secure way to run hosted txtai applications with [txtai.cloud](https://txtai.cloud).

docs/install.md

@@ -5,45 +5,35 @@
 
 The easiest way to install is via pip and PyPI
 
-    pip install txtai
+```
+pip install txtai
+```
 
-Python 3.7+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
-
-## Install from source
-
-txtai can also be installed directly from GitHub to access the latest, unreleased features.
-
-    pip install git+https://github.com/neuml/txtai
-
-## Environment specific prerequisites
-
-Additional environment specific prerequisites are below.
-
-### Linux
-
-Optional audio transcription requires a [system library to be installed](https://github.com/bastibe/python-soundfile#installation)
-
-### macOS
-
-Run `brew install libomp` see [this link](https://github.com/kyamagu/faiss-wheels#prerequisite)
-
-### Windows
-
-Optional dependencies require [C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/)
+Python 3.11+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
 
 ## Optional dependencies
 
 txtai has the following optional dependencies that can be installed as extras. The patterns below are supported
 in setup.py install_requires sections.
 
+_Note: Extras are provided for convenience. Alternatively, individual packages can be installed to limit dependencies._
+
 ### All
 
-Install all dependencies (mirrors txtai < 3.2)
+Install all dependencies.
 
 ```
 pip install txtai[all]
 ```
 
+### ANN
+
+Additional ANN backends.
+
+```
+pip install txtai[ann]
+```
+
 ### API
 
 Serve txtai via a web API.
@@ -60,17 +50,33 @@ Interface with cloud compute.
 pip install txtai[cloud]
 ```
 
+### Console
+
+Command line index query console.
+
+```
+pip install txtai[console]
+```
+
 ### Database
 
-Additional content storage options
+Additional content storage options.
 
 ```
 pip install txtai[database]
 ```
 
+### Graph
+
+Topic modeling, data connectivity and network analysis.
+
+```
+pip install txtai[graph]
+```
+
 ### Model
 
-Additional non-standard models
+Additional non-standard models.
 
 ```
 pip install txtai[model]
@@ -84,12 +90,22 @@ All pipelines - default install comes with most common pipelines.
 pip install txtai[pipeline]
 ```
 
-### Similarity
+More granular extras are available for pipeline categories: `pipeline-audio`, `pipeline-data`, `pipeline-image`, `pipeline-llm`, `pipeline-text`, and `pipeline-train`.
 
-Word vectors, support for sentence-transformers models not on the HF Hub and additional ANN libraries.
+### Scoring
+
+Additional scoring methods.
 
 ```
-pip install txtai[similarity]
+pip install txtai[scoring]
+```
+
+### Vectors
+
+Additional vector methods.
+
+```
+pip install txtai[vectors]
 ```
 
 ### Workflow
@@ -100,8 +116,83 @@ All workflow tasks - default install comes with most common workflow tasks.
 pip install txtai[workflow]
 ```
 
+### Combining dependencies
+
 Multiple dependencies can be specified at the same time.
 
 ```
 pip install txtai[pipeline,workflow]
 ```
+
+## Environment specific prerequisites
+
+Additional environment specific prerequisites are below.
+
+### Linux
+
+The AudioStream and Microphone pipelines require the [PortAudio](https://python-sounddevice.readthedocs.io/en/0.5.0/installation.html) system library.
+
+The Transcription pipeline requires the [SoundFile](https://github.com/bastibe/python-soundfile#installation) system library.
+
+The LiteRT LLM pipeline requires libegl1, libgles2 and libvulkan1.
+
+### macOS
+
+Older versions of Faiss have a runtime dependency on `libomp` for macOS. Run `brew install libomp` in this case.
+
+The AudioStream and Microphone pipelines require the [PortAudio](https://python-sounddevice.readthedocs.io/en/0.5.0/installation.html) system library. Run `brew install portaudio`.
+
+### Windows
+
+Optional dependencies require [C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/)
+
+The [txtai build workflow](https://github.com/neuml/txtai/blob/master/.github/workflows/build.yml) occasionally has work arounds for other known but temporary dependency issues. The [FAQ](../faq) also has a list of common problems, including common installation issues.
+
+## CPU-only
+
+The default install adds PyTorch with GPU support. There are a number of dependencies that come with that. When running in a CPU-only environment, the CPU-only PyTorch package can be installed with txtai as follows.
+
+```
+pip install txtai torch==[version]+cpu \
+-f https://download.pytorch.org/whl/torch
+```
+
+Where `[version]` is the version of PyTorch (such as 2.4.1). The [txtai-cpu](https://hub.docker.com/r/neuml/txtai-cpu) image on Docker Hub uses this method to reduce the image size.
+
+## Install from source
+
+txtai can also be installed directly from GitHub to access the latest, unreleased features.
+
+```
+pip install git+https://github.com/neuml/txtai
+```
+
+Extras can be installed from GitHub by adding `#egg=txtai[<name-of-extra>]` to the end of the above URL.
+
+## Conda
+
+A [community-supported txtai package](https://anaconda.org/conda-forge/txtai) is available via conda-forge.
+
+```
+conda install -c conda-forge txtai
+```
+
+## Minimal install
+
+A lightweight zero dependency minimal install package is available. This is helpful if the default libraries are too heavy or there are no plans to use them.
+
+All the same extras mentioned above are available in this version of the package.
+
+```
+# Lightweight minimal install
+pip install txtai_minimal
+
+# Same as standard `txtai` install
+pip install txtai_minimal[default]
+```
+
+Note that in order to use the `Embeddings` or `LLM` interface without `torch` requires either `llama.cpp`, `litellm` or `litert`.
+
+## Run with containers
+
+Docker images are available for txtai. [See this section](../cloud) for more information on container-based installs.