Fix incompatibility between latest skl2onnx and txtai, closes #729

Ensure config.path = None and config.path missing mean the same thing

.github/workflows/build.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Install Python - Linux
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.8
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Python - macOS/Windows
@@ -34,13 +34,17 @@ jobs:
           java-version: "8"
 
       - name: Install dependencies - Linux
-        run: sudo apt-get install libsndfile1
+        run: sudo apt-get update && sudo apt-get install libsndfile1
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install dependencies - macOS
         run: |
-          brew install libomp
-          pip install torch==1.12.1 torchvision==0.13.1
+          echo "OMP_NUM_THREADS=1" >> $GITHUB_ENV
+          echo "PYTORCH_MPS_DISABLE=1" >> $GITHUB_ENV
+          echo "LLAMA_NO_METAL=1" >> $GITHUB_ENV
+          echo "ACCELERATE_USE_CPU=1" >> $GITHUB_ENV
+          echo "TIKA_STARTUP_SLEEP=30" >> $GITHUB_ENV
+          echo "TIKA_STARTUP_MAX_RETRY=10" >> $GITHUB_ENV
         if: matrix.os == 'macos-latest'
 
       - name: Install dependencies - Windows
@@ -49,7 +53,7 @@ jobs:
 
       - name: Build
         run: |
-          pip install -U pip wheel
+          pip install -U wheel
           pip install .[all,dev]
           python -c "import nltk; nltk.download('punkt')"
           python --version

.github/workflows/docs.yml

@@ -10,7 +10,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.8
       - run: |
           pip install -U pip wheel
           pip install .[all,dev]

.gitignore

@@ -7,3 +7,4 @@ __pycache__/
 .coverage
 .coverage.*
 *.pyc
+.vscode/

.pre-commit-config.yaml

@@ -1,13 +1,13 @@
 repos:
   - repo: https://github.com/pycqa/pylint
-    rev: v2.12.1
+    rev: v2.16.2
     hooks:
     - id: pylint
       args:
       - -d import-error
       - -d duplicate-code
   - repo: https://github.com/ambv/black
-    rev: 22.3.0
+    rev: 23.1.0
     hooks:
     - id: black
       language_version: python3

.pylintrc

@@ -15,6 +15,3 @@ min-public-methods=0
 
 [FORMAT]
 max-line-length=150
-
-[MESSAGES CONTROL]
-disable=R0201

CITATION.cff

@@ -1,8 +1,8 @@
 cff-version: 1.2.0
 date-released: 2020-08-11
 message: "If you use this software, please cite it as below."
-title: "txtai"
-abstract: "Build AI-powered semantic search applications"
+title: "txtai: the all-in-one embeddings database"
+abstract: "txtai is an all-in-one open-source embeddings database for semantic search, LLM orchestration and language model workflows"
 url: "https://github.com/neuml/txtai"
 authors:
 - family-names: "Mezzetti"

Makefile

@@ -14,10 +14,16 @@ export TOKENIZERS_PARALLELISM := false
 # Default python executable if not provided
 PYTHON ?= python
 
+# Check for wget
+WGET := $(shell wget --version 2> /dev/null)
+ifndef WGET
+    $(error "Required binary `wget` not found, please install wget OS package")
+endif
+
 # Download test data
 data:
 	mkdir -p /tmp/txtai
-	wget -N https://github.com/neuml/txtai/releases/download/v3.5.0/tests.tar.gz -P /tmp
+	wget -N https://github.com/neuml/txtai/releases/download/v6.2.0/tests.tar.gz -P /tmp
 	tar -xvzf /tmp/tests.tar.gz -C /tmp
 
 # Unit tests
@@ -26,5 +32,20 @@ test:
 
 # Run tests while calculating code coverage
 coverage:
-	coverage run -m unittest discover -v -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testann -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testapi.testauthorization -k testapi.testcluster -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testapi.testencoding -k testapi.testextension -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testapp -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testarchive -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testcloud -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testconsole -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testdatabase -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testembeddings -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testgraph -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testmodels -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testoptional -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testpipeline -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testscoring -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testvectors -s ${TEST_DIR}
+	coverage run -m unittest discover -v -k testworkflow -s ${TEST_DIR}
 	coverage combine

README.md

@@ -2,9 +2,9 @@
     <img src="https://raw.githubusercontent.com/neuml/txtai/master/logo.png"/>
 </p>
 
-<h3 align="center">
-    <p>Build AI-powered semantic search applications</p>
-</h3>
+<p align="center">
+    <b>All-in-one embeddings database</b>
+</p>
 
 <p align="center">
     <a href="https://github.com/neuml/txtai/releases">
@@ -27,64 +27,46 @@
     </a>
 </p>
 
--------------------------------------------------------------------------------------------------------------------------------------------------------
+txtai is an all-in-one embeddings database for semantic search, LLM orchestration and language model workflows.
 
-txtai executes machine-learning workflows to transform data and build AI-powered semantic search applications.
+![architecture](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/architecture.png#gh-light-mode-only)
+![architecture](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/architecture-dark.png#gh-dark-mode-only)
 
-![demo](https://raw.githubusercontent.com/neuml/txtai/master/demo.gif)
+Embeddings databases are a union of vector indexes (sparse and dense), graph networks and relational databases. This enables vector search with SQL, topic modeling, retrieval augmented generation and more.
 
-Traditional search systems use keywords to find data. Semantic search applications have an understanding of natural language and identify results that have the same meaning, not necessarily the same keywords.
-
-![search](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/search.png#gh-light-mode-only)
-![search](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/search-dark.png#gh-dark-mode-only)
-
-Backed by state-of-the-art machine learning models, data is transformed into vector representations for search (also known as embeddings). Innovation is happening at a rapid pace, models can understand concepts in documents, audio, images and video.
+Embeddings databases can stand on their own and/or serve as a powerful knowledge source for large language model (LLM) prompts.
 
 Summary of txtai features:
 
-- 🔎 Large-scale similarity search with multiple index backends ([Faiss](https://github.com/facebookresearch/faiss), [Annoy](https://github.com/spotify/annoy), [Hnswlib](https://github.com/nmslib/hnswlib)) and support for external vector databases
-- 📄 Create embeddings for text snippets, documents, audio, images and video
-- 💡 Machine-learning pipelines that run question-answering, labeling, transcription, translation, summarization, LLM prompts and more
-- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be microservices or full-fledged indexing workflows.
+- 🔎 Vector search with SQL, object storage, topic modeling, graph analysis and multimodal indexing
+- 📄 Create embeddings for text, documents, audio, images and video
+- 💡 Pipelines powered by language models that run LLM prompts, question-answering, labeling, transcription, translation, summarization and more
+- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be simple microservices or multi-model workflows.
 - ⚙️ Build with Python or YAML. API bindings available for [JavaScript](https://github.com/neuml/txtai.js), [Java](https://github.com/neuml/txtai.java), [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go).
-- ☁️ Cloud-native architecture that scales out with container orchestration systems (e.g. Kubernetes)
+- ☁️ Run local or scale out with container orchestration
 
-Applications range from similarity search to NLP-driven data extractions that generate structured data. Semantic workflows transform and find data driven by user intent.
+txtai is built with Python 3.8+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
 
-![flows](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/flows.png#gh-light-mode-only)
-![flows](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/flows-dark.png#gh-dark-mode-only)
-
-The following applications are powered by txtai.
-
-![apps](https://raw.githubusercontent.com/neuml/txtai/master/apps.jpg)
-
-| Application  | Description  |
-|:----------|:-------------|
-| [paperai](https://github.com/neuml/paperai) | Semantic search and workflows for medical/scientific papers |
-| [codequestion](https://github.com/neuml/codequestion) | Semantic search for developers |
-| [tldrstory](https://github.com/neuml/tldrstory) | Semantic search for headlines and story text |
-| [neuspo](https://neuspo.com) | Fact-driven, real-time sports event and news site |
-
-txtai is built with Python 3.7+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi)
+*Interested in an easy and secure way to run hosted txtai applications? Then join the [txtai.cloud](https://txtai.cloud) preview to learn more.*
 
 ## Why txtai?
 
 ![why](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/why.png#gh-light-mode-only)
 ![why](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/why-dark.png#gh-dark-mode-only)
 
-In addition to traditional search systems, a growing number of semantic search solutions are available, so why txtai?
+New vector databases, LLM frameworks and everything in between are sprouting up daily. Why build with txtai?
 
 - Up and running in minutes with [pip](https://neuml.github.io/txtai/install/) or [Docker](https://neuml.github.io/txtai/cloud/)
 ```python
 # Get started in a couple lines
-from txtai.embeddings import Embeddings
+import txtai
 
-embeddings = Embeddings({"path": "sentence-transformers/all-MiniLM-L6-v2"})
-embeddings.index([(0, "Correct", None), (1, "Not what we hoped", None)])
+embeddings = txtai.Embeddings()
+embeddings.index(["Correct", "Not what we hoped"])
 embeddings.search("positive", 1)
-#[(0, 0.2986203730106354)]
+#[(0, 0.29862046241760254)]
 ```
-- Build applications in your programming language of choice via the API
+- Built-in API makes it easy to develop applications using your programming language of choice
 ```yaml
 # app.yml
 embeddings:
@@ -94,11 +76,84 @@ embeddings:
 CONFIG=app.yml uvicorn "txtai.api:app"
 curl -X GET "http://localhost:8000/search?query=positive"
 ```
-- Connect machine learning models together to build intelligent data processing workflows
-- Works with both small and big data - scale when needed
-- Supports micromodels all the way up to large language models (LLMs)
-- Low footprint - install additional dependencies when you need them
-- [Learn by example](#examples) - notebooks cover all available functionality
+- Run local - no need to ship data off to disparate remote services
+- Work with micromodels all the way up to large language models (LLMs)
+- Low footprint - install additional dependencies and scale up when needed
+- [Learn by example](https://neuml.github.io/txtai/examples) - notebooks cover all available functionality
+
+## Use Cases
+
+The following sections introduce common txtai use cases. A comprehensive set of over 50 [example notebooks and applications](https://neuml.github.io/txtai/examples) are also available.
+
+### Semantic Search
+
+Build semantic/similarity/vector/neural search applications.
+
+![demo](https://raw.githubusercontent.com/neuml/txtai/master/demo.gif)
+
+Traditional search systems use keywords to find data. Semantic search has an understanding of natural language and identifies results that have the same meaning, not necessarily the same keywords.
+
+![search](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/search.png#gh-light-mode-only)
+![search](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/search-dark.png#gh-dark-mode-only)
+
+Get started with the following examples.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Introducing txtai](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=SIezMnVdmMs) | Overview of the functionality provided by txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |
+| [Similarity search with images](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) | Embed images and text into the same space for search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |
+| [Build a QA database](https://github.com/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) | Question matching with semantic search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) |
+| [Semantic Graphs](https://github.com/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) | Explore topics, data connectivity and run network analysis| [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) |
+
+### LLM Orchestration
+
+LLM chains, retrieval augmented generation (RAG), chat with your data, pipelines and workflows that interface with large language models (LLMs).
+
+#### Chains
+
+Integrate LLM chains (known as workflows in txtai), multiple LLM agents and self-critique.
+
+![llm](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/llm.png)
+
+See below to learn more.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
+| [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
+| [Build knowledge graphs with LLMs](https://github.com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) | Build knowledge graphs with LLM-driven entity extraction | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |
+
+#### Retrieval augmented generation
+
+Retrieval augmented generation (RAG) reduces the risk of LLM hallucinations by constraining the output with a knowledge base as context. RAG is commonly used to "chat with your data".
+
+![rag](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/rag.png#gh-light-mode-only)
+![rag](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/rag-dark.png#gh-dark-mode-only)
+
+A novel feature of txtai is that it can provide both an answer and source citation.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Advanced RAG with graph path traversal](https://github.com/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) | Graph path traversal to collect complex sets of data for advanced RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) |
+| [Advanced RAG with guided generation](https://github.com/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) | Retrieval Augmented and Guided Generation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) |
+| [RAG with llama.cpp and external API services](https://github.com/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) | RAG with additional vector and LLM frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) |
+
+### Language Model Workflows
+
+Language model workflows, also known as semantic workflows, connect language models together to build intelligent applications.
+
+![flows](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/flows.png#gh-light-mode-only)
+![flows](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/flows-dark.png#gh-dark-mode-only)
+
+While LLMs are powerful, there are plenty of smaller, more specialized models that work better and faster for specific tasks. This includes models for extractive question-answering, automatic summarization, text-to-speech, transcription and translation.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) [▶️](https://www.youtube.com/watch?v=UBMPDCn1gEU) | Simple yet powerful constructs to efficiently process data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
+| [Building abstractive text summaries](https://github.com/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) | Run abstractive text summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) |
+| [Transcribe audio to text](https://github.com/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) | Convert audio files to text | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) |
+| [Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) | Streamline machine translation and language detection | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |
 
 ## Installation
 
@@ -111,119 +166,68 @@ The easiest way to install is via pip and PyPI
 pip install txtai
 ```
 
-Python 3.7+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
+Python 3.8+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
 
-See the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering
-[optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies), [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites), [installing from source](https://neuml.github.io/txtai/install/#install-from-source), [conda support](https://neuml.github.io/txtai/install/#conda) and how to [run with containers](https://neuml.github.io/txtai/cloud).
+See the detailed [install instructions](https://neuml.github.io/txtai/install) for more information covering [optional dependencies](https://neuml.github.io/txtai/install/#optional-dependencies), [environment specific prerequisites](https://neuml.github.io/txtai/install/#environment-specific-prerequisites), [installing from source](https://neuml.github.io/txtai/install/#install-from-source), [conda support](https://neuml.github.io/txtai/install/#conda) and how to [run with containers](https://neuml.github.io/txtai/cloud).
 
-## Examples
+## Model guide
 
-![examples](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/examples.png#gh-light-mode-only)
-![examples](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/examples-dark.png#gh-dark-mode-only)
+![models](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/models.png)
 
-The examples directory has a series of notebooks and applications giving an overview of txtai. See the sections below.
+See the table below for the current recommended models. These models all allow commercial use and offer a blend of speed and performance.
 
-### Semantic Search
+| Component                                                                     | Model(s)                                                                 |
+| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------ |
+| [Embeddings](https://neuml.github.io/txtai/embeddings)                        | [all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) | 
+| [Image Captions](https://neuml.github.io/txtai/pipeline/image/caption)        | [BLIP](https://hf.co/Salesforce/blip-image-captioning-base)              |
+| [Labels - Zero Shot](https://neuml.github.io/txtai/pipeline/text/labels)      | [BART-Large-MNLI](https://hf.co/facebook/bart-large)                     |
+| [Labels - Fixed](https://neuml.github.io/txtai/pipeline/text/labels)          | Fine-tune with [training pipeline](https://neuml.github.io/txtai/pipeline/train/trainer)          |
+| [Large Language Model (LLM)](https://neuml.github.io/txtai/pipeline/text/llm) | [Mistral 7B OpenOrca](https://hf.co/Open-Orca/Mistral-7B-OpenOrca)       |
+| [Summarization](https://neuml.github.io/txtai/pipeline/text/summary)          | [DistilBART](https://hf.co/sshleifer/distilbart-cnn-12-6)                |
+| [Text-to-Speech](https://neuml.github.io/txtai/pipeline/audio/texttospeech)   | [ESPnet JETS](https://hf.co/NeuML/ljspeech-jets-onnx)                    |
+| [Transcription](https://neuml.github.io/txtai/pipeline/audio/transcription)   | [Whisper](https://hf.co/openai/whisper-base)                             | 
+| [Translation](https://neuml.github.io/txtai/pipeline/text/translation)        | [OPUS Model Series](https://hf.co/Helsinki-NLP)                          |
 
-Build semantic/similarity/vector/neural search applications.
+Models can be loaded as either a path from the Hugging Face Hub or a local directory. Model paths are optional, defaults are loaded when not specified. For tasks with no recommended model, txtai uses the default models as shown in the Hugging Face Tasks guide.
 
-| Notebook  | Description  |       |
-|:----------|:-------------|------:|
-| [Introducing txtai](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=SIezMnVdmMs) | Overview of the functionality provided by txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |
-| [Build an Embeddings index with Hugging Face Datasets](https://github.com/neuml/txtai/blob/master/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb) | Index and search Hugging Face Datasets | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb) |
-| [Build an Embeddings index from a data source](https://github.com/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb)  | Index and search a data source with word embeddings | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb) |
-| [Add semantic search to Elasticsearch](https://github.com/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb)  | Add semantic search to existing search systems | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb) |
-| [Similarity search with images](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) | Embed images and text into the same space for search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |
-| [Distributed embeddings cluster](https://github.com/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) | Distribute an embeddings index across multiple data nodes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) |
-| [What's new in txtai 4.0](https://github.com/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) | Content storage, SQL, object storage, reindex and compressed indexes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) |
-| [Anatomy of a txtai index](https://github.com/neuml/txtai/blob/master/examples/29_Anatomy_of_a_txtai_index.ipynb) | Deep dive into the file formats behind a txtai embeddings index | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/29_Anatomy_of_a_txtai_index.ipynb) |
-| [Custom Embeddings SQL functions](https://github.com/neuml/txtai/blob/master/examples/30_Embeddings_SQL_custom_functions.ipynb) | Add user-defined functions to Embeddings SQL | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/30_Embeddings_SQL_custom_functions.ipynb) |
-| [Model explainability](https://github.com/neuml/txtai/blob/master/examples/32_Model_explainability.ipynb) | Explainability for semantic search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/32_Model_explainability.ipynb) |
-| [Query translation](https://github.com/neuml/txtai/blob/master/examples/33_Query_translation.ipynb) | Domain-specific natural language queries with query translation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/33_Query_translation.ipynb) |
-| [Build a QA database](https://github.com/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) | Question matching with semantic search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) |
-| [Embeddings components](https://github.com/neuml/txtai/blob/master/examples/37_Embeddings_index_components.ipynb) | Composable search with vector, SQL and scoring components | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/37_Embeddings_index_components.ipynb) |
-| [Semantic Graphs](https://github.com/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) | Explore topics, data connectivity and run network analysis| [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) |
-| [Topic Modeling with BM25](https://github.com/neuml/txtai/blob/master/examples/39_Classic_Topic_Modeling_with_BM25.ipynb) | Topic modeling backed by a BM25 index | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/39_Classic_Topic_Modeling_with_BM25.ipynb) |
-| [Prompt-driven search with LLMs](https://github.com/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) | Embeddings-guided and Prompt-driven search with Large Language Models (LLMs) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) |
+See the following links to learn more.
 
-### Pipelines
+- [Hugging Face Tasks](https://hf.co/tasks)
+- [Hugging Face Model Hub](https://hf.co/models)
+- [MTEB Leaderboard](https://hf.co/spaces/mteb/leaderboard)
+- [LMSYS LLM Leaderboard](https://chat.lmsys.org/?leaderboard)
+- [Open LLM Leaderboard](https://hf.co/spaces/HuggingFaceH4/open_llm_leaderboard)
 
-Transform data with NLP-backed pipelines.
+## Powered by txtai
 
-| Notebook  | Description  |       |
-|:----------|:-------------|------:|
-| [Extractive QA with txtai](https://github.com/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) | Introduction to extractive question-answering with txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) |
-| [Extractive QA with Elasticsearch](https://github.com/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) | Run extractive question-answering queries with Elasticsearch | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) |
-| [Extractive QA to build structured data](https://github.com/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) | Build structured datasets using extractive question-answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) |
-| [Apply labels with zero shot classification](https://github.com/neuml/txtai/blob/master/examples/07_Apply_labels_with_zero_shot_classification.ipynb) | Use zero shot learning for labeling, classification and topic modeling | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/07_Apply_labels_with_zero_shot_classification.ipynb) |
-| [Building abstractive text summaries](https://github.com/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) | Run abstractive text summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) |
-| [Extract text from documents](https://github.com/neuml/txtai/blob/master/examples/10_Extract_text_from_documents.ipynb) | Extract text from PDF, Office, HTML and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/10_Extract_text_from_documents.ipynb) |
-| [Text to speech generation](https://github.com/neuml/txtai/blob/master/examples/40_Text_to_Speech_Generation.ipynb) | Generate speech from text | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/40_Text_to_Speech_Generation.ipynb) |
-| [Transcribe audio to text](https://github.com/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) | Convert audio files to text | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) |
-| [Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) | Streamline machine translation and language detection | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |
-| [Generate image captions and detect objects](https://github.com/neuml/txtai/blob/master/examples/25_Generate_image_captions_and_detect_objects.ipynb) | Captions and object detection for images | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/25_Generate_image_captions_and_detect_objects.ipynb) |
-| [Near duplicate image detection](https://github.com/neuml/txtai/blob/master/examples/31_Near_duplicate_image_detection.ipynb) | Identify duplicate and near-duplicate images | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/31_Near_duplicate_image_detection.ipynb) |
-| [API Gallery](https://github.com/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) | Using txtai in JavaScript, Java, Rust and Go | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) |
+The following applications are powered by txtai.
 
-### Workflows
+![apps](https://raw.githubusercontent.com/neuml/txtai/master/apps.jpg)
 
-Efficiently process data at scale.
+| Application  | Description  |
+|:------------ |:-------------|
+| [txtchat](https://github.com/neuml/txtchat) | Retrieval Augmented Generation (RAG) powered search |
+| [paperai](https://github.com/neuml/paperai) | Semantic search and workflows for medical/scientific papers |
+| [codequestion](https://github.com/neuml/codequestion) | Semantic search for developers |
+| [tldrstory](https://github.com/neuml/tldrstory) | Semantic search for headlines and story text |
 
-| Notebook  | Description  |       |
-|:----------|:-------------|------:|
-| [Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) [▶️](https://www.youtube.com/watch?v=UBMPDCn1gEU) | Simple yet powerful constructs to efficiently process data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
-| [Transform tabular data with composable workflows](https://github.com/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) | Transform, index and search tabular data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) |
-| [Tensor workflows](https://github.com/neuml/txtai/blob/master/examples/23_Tensor_workflows.ipynb) | Performant processing of large tensor arrays | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/23_Tensor_workflows.ipynb) |
-| [Entity extraction workflows](https://github.com/neuml/txtai/blob/master/examples/26_Entity_extraction_workflows.ipynb) | Identify entity/label combinations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/26_Entity_extraction_workflows.ipynb) |
-| [Workflow Scheduling](https://github.com/neuml/txtai/blob/master/examples/27_Workflow_scheduling.ipynb) | Schedule workflows with cron expressions | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/27_Workflow_scheduling.ipynb) |
-| [Push notifications with workflows](https://github.com/neuml/txtai/blob/master/examples/28_Push_notifications_with_workflows.ipynb) | Generate and push notifications with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/28_Push_notifications_with_workflows.ipynb) |
-| [Pictures are a worth a thousand words](https://github.com/neuml/txtai/blob/master/examples/35_Pictures_are_worth_a_thousand_words.ipynb) | Generate webpage summary images with DALL-E mini | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/35_Pictures_are_worth_a_thousand_words.ipynb) |
-| [Run txtai with native code](https://github.com/neuml/txtai/blob/master/examples/36_Run_txtai_in_native_code.ipynb) | Execute workflows in native code with the Python C API | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/36_Run_txtai_in_native_code.ipynb) |
+In addition to this list, there are also many other [open-source projects](https://github.com/neuml/txtai/network/dependents), [published research](https://scholar.google.com/scholar?q=txtai&hl=en&as_ylo=2022) and closed proprietary/commercial projects that have built on txtai in production.
 
-### Model Training
+## Further Reading
 
-Train NLP models.
+![further](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/further.png#gh-light-mode-only)
+![further](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/further-ghdark.png#gh-dark-mode-only)
 
-| Notebook  | Description  |       |
-|:----------|:-------------|------:|
-| [Train a text labeler](https://github.com/neuml/txtai/blob/master/examples/16_Train_a_text_labeler.ipynb) | Build text sequence classification models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/16_Train_a_text_labeler.ipynb) |
-| [Train without labels](https://github.com/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) | Use zero-shot classifiers to train new models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/17_Train_without_labels.ipynb) |
-| [Train a QA model](https://github.com/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) | Build and fine-tune question-answering models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/19_Train_a_QA_model.ipynb) |
-| [Train a language model from scratch](https://github.com/neuml/txtai/blob/master/examples/41_Train_a_language_model_from_scratch.ipynb) | Build new language models | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/41_Train_a_language_model_from_scratch.ipynb) |
-| [Export and run models with ONNX](https://github.com/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) | Export models with ONNX, run natively in JavaScript, Java and Rust | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) |
-| [Export and run other machine learning models](https://github.com/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) | Export and run models from scikit-learn, PyTorch and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) |
-
-### Applications
-
-Series of example applications with txtai. Links to hosted versions on [Hugging Face Spaces](https://hf.co/spaces) also provided.
-
-| Application  | Description  |       |
-|:-------------|:-------------|------:|
-| [Basic similarity search](https://github.com/neuml/txtai/blob/master/examples/similarity.py) | Basic similarity search example. Data from the original txtai demo. |[🤗](https://hf.co/spaces/NeuML/similarity)|
-| [Book search](https://github.com/neuml/txtai/blob/master/examples/books.py) | Book similarity search application. Index book descriptions and query using natural language statements. |*Local run only*|
-| [Image search](https://github.com/neuml/txtai/blob/master/examples/images.py) | Image similarity search application. Index a directory of images and run searches to identify images similar to the input query. |[🤗](https://hf.co/spaces/NeuML/imagesearch)|
-| [Summarize an article](https://github.com/neuml/txtai/blob/master/examples/article.py) | Summarize an article. Workflow that extracts text from a webpage and builds a summary. |[🤗](https://hf.co/spaces/NeuML/articlesummary)|
-| [Wiki search](https://github.com/neuml/txtai/blob/master/examples/wiki.py) | Wikipedia search application. Queries Wikipedia API and summarizes the top result. |[🤗](https://hf.co/spaces/NeuML/wikisummary)|
-| [Workflow builder](https://github.com/neuml/txtai/blob/master/examples/workflows.py) | Build and execute txtai workflows. Connect summarization, text extraction, transcription, translation and similarity search pipelines together to run unified workflows. |[🤗](https://hf.co/spaces/NeuML/txtai)|
-
-### Documentation
-
-[Full documentation on txtai](https://neuml.github.io/txtai) including configuration settings for pipelines, workflows, indexing and the API.
-
-### Further Reading
-
-![further](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/further.png)
-
-- [Introducing txtai, AI-powered semantic search built on Transformers](https://medium.com/neuml/introducing-txtai-an-ai-powered-search-engine-built-on-transformers-37674be252ec)
+- [Introducing txtai, the all-in-one embeddings database](https://medium.com/neuml/introducing-txtai-the-all-in-one-embeddings-database-c721f4ff91ad)
 - [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
-- [What's new in txtai 5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
+- [What's new in txtai 7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
 - [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9)
-- [Run machine-learning workflows to transform data and build AI-powered semantic search applications with txtai](https://medium.com/neuml/run-machine-learning-workflows-to-transform-data-and-build-ai-powered-text-indices-with-txtai-43d769b566a7)
-- [Semantic search on the cheap](https://medium.com/neuml/semantic-search-on-the-cheap-55940c0fcdab)
-- [Serverless vector search with txtai](https://medium.com/neuml/serverless-vector-search-with-txtai-96f6163ab972)
-- [Insights from the txtai console](https://medium.com/neuml/insights-from-the-txtai-console-d307c28e149e)
-- [The big and small of txtai](https://medium.com/neuml/the-big-and-small-of-txtai-4ca405c1b82)
+- [Running txtai at scale](https://medium.com/neuml/running-at-scale-with-txtai-71196cdd99f9)
 
-### Contributing
+## Documentation
+
+[Full documentation on txtai](https://neuml.github.io/txtai) including configuration settings for embeddings, pipelines, workflows, API and a FAQ with common questions/issues is available.
+
+## Contributing
 
 For those who would like to contribute to txtai, please see [this guide](https://github.com/neuml/.github/blob/master/CONTRIBUTING.md).

docker/aws/workflow.py

@@ -8,6 +8,7 @@ from txtai.api import API
 
 APP = None
 
+
 # pylint: disable=W0603,W0613
 def handler(event, context):
     """

docker/base/Dockerfile

@@ -8,7 +8,7 @@ ARG GPU
 # Target CPU architecture
 ARG TARGETARCH
 
-# Set Python version (i.e. 3, 3.7, 3.8)
+# Set Python version (i.e. 3, 3.8)
 ARG PYTHON_VERSION=3
 
 # List of txtai components to install
@@ -27,7 +27,7 @@ RUN \
     # Install txtai project and dependencies
     ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python && \
     python -m pip install --no-cache-dir -U pip wheel setuptools && \
-    if [ -z ${GPU} ] && { [ -z ${TARGETARCH} ] || [ ${TARGETARCH} = "amd64" ] ;}; then pip install --no-cache-dir pip install torch==1.12.1+cpu torchvision==0.13.1+cpu -f https://download.pytorch.org/whl/torch_stable.html; fi && \
+    if [ -z ${GPU} ] && { [ -z ${TARGETARCH} ] || [ ${TARGETARCH} = "amd64" ] ;}; then pip install --no-cache-dir torch==2.2.2+cpu torchvision==0.17.2+cpu -f https://download.pytorch.org/whl/torch_stable.html; fi && \
     python -m pip install --no-cache-dir txtai${COMPONENTS} && \
     python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download('punkt')" && \
     \

docs/api/customization.md

@@ -0,0 +1,25 @@
+# Customization
+
+The txtai API has a number of features out of the box that are designed to help get started quickly. API services can also be augmented with custom code and functionality. The two main ways to do this are with extensions and dependencies.
+
+Extensions add a custom endpoint. Dependencies add middleware that executes with each request. See the sections below for more.
+
+## Extensions
+
+While the API is extremely flexible and complex logic can be executed through YAML-driven workflows, some may prefer to create an endpoint in Python. API extensions define custom Python endpoints that interact with txtai applications. 
+
+See the link below for a detailed example.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Custom API Endpoints](https://github.com/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) | Extend the API with custom endpoints | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) |
+
+## Dependencies
+
+txtai has a default API token authorization method that works well in many cases. Dependencies can also add custom logic with each request. This could be an additional authorization step and/or an authentication method. 
+
+See the link below for a detailed example.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [API Authorization and Authentication](https://github.com/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) | Add authorization, authentication and middleware dependencies to the API | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) |

docs/api/index.md

@@ -73,7 +73,7 @@ Assuming this YAML content is stored in a file named config.yml, the following c
 CONFIG=config.yml uvicorn "txtai.api:app"
 ```
 
-uvicorn is a full-featured production ready server with support for SSL and more. See the [uvicorn deployment guide](https://www.uvicorn.org/deployment/) for details.
+Uvicorn is a full-featured production-ready server. See the [Uvicorn deployment guide](https://www.uvicorn.org/deployment/) for more on configuration options.
 
 ## Connect to API
 
@@ -114,6 +114,7 @@ The API can be containerized and run. This will bring up an API instance without
 
 The following programming languages have bindings with the txtai API:
 
+- [Python](https://github.com/neuml/txtai.py)
 - [JavaScript](https://github.com/neuml/txtai.js)
 - [Java](https://github.com/neuml/txtai.java)
 - [Rust](https://github.com/neuml/txtai.rs)

docs/api/methods.md

@@ -1,13 +1,15 @@
 # Methods
 
 ::: txtai.api.API
-    selection:
+    options:
         inherited_members: true
         filters:
             - "!__del__"
             - "!flows"
             - "!function"
+            - "!indexes"
             - "!limit"
             - "!pipes"
             - "!read"
             - "!resolve"
+            - "!weights"

docs/api/security.md

@@ -0,0 +1,33 @@
+# Security
+
+The default implementation of an API service runs via HTTP and is fully open. If the service is being run as a prototype on an internal network, that may be fine. In most scenarios, the connection should at least be encrypted. Authorization is another built-in feature that requires a valid API token with each request. See below for more.
+
+## HTTPS
+
+The default API service command starts a Uvicorn server as a HTTP service on port 8000. To run a HTTPS service, consider the following options.
+
+- [TLS Proxy Server](https://fastapi.tiangolo.com/deployment/https/). *Recommended choice*. With this configuration, the txtai API service runs as a HTTP service only accessible on the localhost/local network. The proxy server handles all encryption and redirects requests to local services. See this [example configuration](https://www.uvicorn.org/deployment/#running-behind-nginx) for more.
+
+- [Uvicorn SSL Certificate](https://www.uvicorn.org/deployment/). Another option is setting the SSL certificate on the Uvicorn service. This works in simple situations but gets complex when hosting multiple txtai or other related services.
+
+## Authorization
+
+Authorization requires a valid API token with each API request. This token is sent as a HTTP `Authorization` header. 
+
+*Server*
+```bash
+CONFIG=config.yml TOKEN=<sha256 encoded token> uvicorn "txtai.api:app"
+```
+
+*Client*
+```bash
+curl \
+  -X POST "http://localhost:8000/workflow" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <token>" \ 
+  -d '{"name":"sumfrench", "elements": ["https://github.com/neuml/txtai"]}'
+```
+
+It's important to note that HTTPS **must** be enabled using one of the methods mentioned above. Otherwise, tokens will be exchanged as clear text. 
+
+Authentication and Authorization can be fully customized. See the [dependencies](../customization#dependencies) section for more.

docs/cloud.md

@@ -3,11 +3,12 @@
 ![cloud](images/cloud.png#only-light)
 ![cloud](images/cloud-dark.png#only-dark)
 
-Scalable cloud-native applications can be built with txtai. The following container runtimes are supported.
+Scalable cloud-native applications can be built with txtai. The following cloud runtimes are supported.
 
 - Container Orchestration Systems (i.e. Kubernetes)
 - Docker Engine
 - Serverless Compute
+- txtai.cloud (coming in 2024)
 
 Images for txtai are available on Docker Hub for [CPU](https://hub.docker.com/r/neuml/txtai-cpu) and [GPU](https://hub.docker.com/r/neuml/txtai-gpu) installs. The CPU install is recommended when GPUs aren't available given the image is half the size.
 
@@ -29,8 +30,8 @@ Examples build commands below.
 # Get Dockerfile
 wget https://raw.githubusercontent.com/neuml/txtai/master/docker/base/Dockerfile
 
-# Build Ubuntu 18.04 image running Python 3.7
-docker build -t txtai --build-arg BASE_IMAGE=ubuntu:18.04 --build-arg PYTHON_VERSION=3.7 .
+# Build Ubuntu 20.04 image running Python 3.8
+docker build -t txtai --build-arg BASE_IMAGE=ubuntu:20.04 --build-arg PYTHON_VERSION=3.8 .
 
 # Build image with GPU support
 docker build -t txtai --build-arg GPU=1 .
@@ -175,3 +176,7 @@ If successful, a local API instance is now running in a "serverless" fashion. Th
 txtai scales with container orchestration systems. This can be self-hosted or with a cloud provider such as [Amazon Elastic Kubernetes Service](https://aws.amazon.com/eks/), [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine) and [Azure Kubernetes Service](https://azure.microsoft.com/en-us/services/kubernetes-service/). There are also other smaller providers with a managed Kubernetes offering.
 
 A full example covering how to build a serverless txtai application on Kubernetes with Knative [can be found here](https://medium.com/neuml/serverless-vector-search-with-txtai-96f6163ab972).
+
+## txtai.cloud
+
+[txtai.cloud](https://txtai.cloud) is a planned effort that will offer an easy and secure way to run hosted txtai applications.

docs/embeddings/configuration/ann.md

@@ -0,0 +1,83 @@
+# ANN
+
+Approximate Nearest Neighbor (ANN) index configuration for storing vector embeddings.
+
+## backend
+```yaml
+backend: faiss|hnsw|annoy|numpy|torch|pgvector|custom
+```
+
+Sets the ANN backend. Defaults to `faiss`. Additional backends are available via the [ann](../../../install/#ann) extras package. Set custom backends via setting this parameter to the fully resolvable class string.
+
+Backend-specific settings are set with a corresponding configuration object having the same name as the backend (i.e. annoy, faiss, or hnsw). These are optional and set to defaults if omitted.
+
+### faiss
+```yaml
+faiss:
+    components: comma separated list of components - defaults to "IDMap,Flat" for small
+                indices and "IVFx,Flat" for larger indexes where
+                x = min(4 * sqrt(embeddings count), embeddings count / 39)
+                automatically calculates number of IVF cells when omitted (supports "IVF,Flat")
+    nprobe: search probe setting (int) - defaults to x/16 (as defined above)
+            for larger indexes
+    nflip: same as nprobe - only used with binary hash indexes
+    quantize: store vectors with x-bit precision vs 32-bit (boolean|int)
+              true sets 8-bit precision, false disables, int sets specified
+              precision
+    mmap: load as on-disk index (boolean) - trade query response time for a
+          smaller RAM footprint, defaults to false
+    sample: percent of data to use for model training (0.0 - 1.0)
+            reduces indexing time for larger (>1M+ row) indexes, defaults to 1.0
+```
+
+Faiss supports both floating point and binary indexes. Floating point indexes are the default. Binary indexes are used when indexing scalar-quantized datasets.
+
+See the following Faiss documentation links for more information.
+
+- [Guidelines for choosing an index](https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index)
+- [Index configuration summary](https://github.com/facebookresearch/faiss/wiki/Faiss-indexes)
+- [Index Factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory)
+- [Binary Indexes](https://github.com/facebookresearch/faiss/wiki/Binary-indexes)
+- [Search Tuning](https://github.com/facebookresearch/faiss/wiki/Faster-search)
+
+### hnsw
+```yaml
+hnsw:
+    efconstruction:  ef_construction param for init_index (int) - defaults to 200
+    m: M param for init_index (int) - defaults to 16
+    randomseed: random-seed param for init_index (int) - defaults to 100
+    efsearch: ef search param (int) - defaults to None and not set
+```
+
+See [Hnswlib documentation](https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md) for more information on these parameters.
+
+### annoy
+```yaml
+annoy:
+    ntrees: number of trees (int) - defaults to 10
+    searchk: search_k search setting (int) - defaults to -1
+```
+
+See [Annoy documentation](https://github.com/spotify/annoy#full-python-api) for more information on these parameters. Note that annoy indexes can not be modified after creation, upserts/deletes and other modifications are not supported.
+
+### numpy
+```yaml
+numpy:
+    quantize: number of quantized bits when working with quantized vectors (int)
+```
+
+The NumPy backend is a k-nearest neighbors backend. It's designed for simplicity and works well with smaller datasets.
+
+The `torch` backend supports the same options. The only difference is that the vectors can be search using GPUs.
+
+### pgvector
+```yaml
+pgvector:
+    url: database url connection string, alternatively can be set via
+         ANN_URL environment variable
+    table: database table to store vectors - defaults to `vectors`
+    efconstruction:  ef_construction param (int) - defaults to 200
+    m: M param for init_index (int) - defaults to 16
+```
+
+The pgvector backend stores embeddings in a Postgres database. See the [pgvector documentation](https://github.com/pgvector/pgvector-python?tab=readme-ov-file#sqlalchemy) for more information on these parameters. See the [SQLAlchemy](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls) documentation for more information on how to construct url connection strings.

docs/embeddings/configuration/cloud.md

@@ -1,59 +1,98 @@
 # Cloud
 
-This section describes parameters used to sync compressed indexes with cloud storage. These parameters are only enabled if an embeddings index is stored as compressed. They are set via the [embeddings.load](../../methods/#txtai.embeddings.base.Embeddings.load) and [embeddings.save](../../methods/#txtai.embeddings.base.Embeddings.save) methods.
+The following describes parameters used to sync indexes with cloud storage. Cloud object storage, the [Hugging Face Hub](https://huggingface.co/models) and custom providers are all supported.
+
+Parameters are set via the [embeddings.load](../../methods/#txtai.embeddings.base.Embeddings.load) and [embeddings.save](../../methods/#txtai.embeddings.base.Embeddings.save) methods.
 
 ## provider
 ```yaml
 provider: string
 ```
 
-The cloud storage provider, see [full list of providers here](https://libcloud.readthedocs.io/en/stable/storage/supported_providers.html).
+Cloud provider. Can be one of the following:
+
+- Cloud object storage. Set to one of these [providers](https://libcloud.readthedocs.io/en/stable/storage/supported_providers.html).
+
+- Hugging Face Hub. Set to `huggingface-hub`.
+
+- Custom providers. Set to the full class path of the custom provider.
 
 ## container
 ```yaml
 container: string
 ```
 
-Container/bucket/directory name.
+Container/bucket/directory/repository name. Embeddings will be stored in the container with the filename specified by the `path` configuration.
 
-## key
+## Cloud object storage configuration
+
+In addition to the above common configuration, the cloud object storage provider has the following additional configuration parameters. Note that some cloud providers do not need any of these parameters and can use implicit authentication with service accounts.
+
+See the [libcloud documentation](https://libcloud.readthedocs.io/en/stable/apidocs/libcloud.common.html#module-libcloud.common.base) for more information on these parameters.
+
+### key
 ```yaml
 key: string
 ```
 
-Provider-specific access key. Can also be set via ACCESS_KEY environment variable. Ensure the configuration file is secured if added to the file.
+Provider-specific access key. Can also be set via `ACCESS_KEY` environment variable. Ensure the configuration file is secured if added to the file. When using implicit authentication, set this to a value such as 'using-implicit-auth'.
 
-## secret
+### secret
 ```yaml
 secret: string
 ```
 
-Provider-specific access secret. Can also be set via ACCESS_SECRET environment variable. Ensure the configuration file is secured if added to the file.
+Provider-specific access secret. Can also be set via `ACCESS_SECRET` environment variable. Ensure the configuration file is secured if added to the file. When using implicit authentication, this option is not required.
 
-## host
+### host
 ```yaml
 host: string
 ```
 
 Optional server host name. Set when using a local cloud storage server.
 
-## port
+### port
 ```yaml
 port: int
 ```
 
 Optional server port. Set when using a local cloud storage server.
 
-## token
+### token
 ```yaml
 token: string
 ```
 
 Optional temporary session token
 
-## region
+### region
 ```yaml
 region: string
 ```
 
 Optional parameter to specify the storage region, provider-specific.
+
+## Hugging Face Hub configuration
+
+The huggingface-hub provider supports the following additional configuration parameters. More on these parameters can be found in the [Hugging Face Hub's documentation](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/overview).
+
+### revision
+```yaml
+revision: string
+```
+
+Optional Git revision id which can be a branch name, a tag, or a commit hash
+
+### cache
+```yaml
+cache: string
+```
+
+Path to the folder where cached files are stored
+
+### token
+```yaml
+token: string|boolean
+```
+
+Token to be used for the download. If set to True, the token will be read from the Hugging Face config folder.

docs/embeddings/configuration/database.md

@@ -0,0 +1,60 @@
+# Database
+
+Databases store metadata, text and binary content.
+
+## content
+```yaml
+content: boolean|sqlite|duckdb|client|url|custom
+```
+
+Enables content storage. When true, the default storage engine, `sqlite` will be used to save metadata.
+
+Client-server connections are supported with either `client` or a full connection URL. When set to `client`, the CLIENT_URL environment variable must be set to the full connection URL. See the [SQLAlchemy](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls) documentation for more information on how to construct connection strings for client-server databases.
+
+Add custom storage engines via setting this parameter to the fully resolvable class string.
+
+Content storage specific settings are set with a corresponding configuration object having the same name as the content storage engine (i.e. duckdb or sqlite). These are optional and set to defaults if omitted.
+
+### sqlite
+```yaml
+sqlite:
+    wal: enable write-ahead logging - allows concurrent read/write operations,
+         defaults to false
+```
+
+## objects
+```yaml
+objects: boolean|image|pickle
+```
+
+Enables object storage. Supports storing binary content. Requires content storage to also be enabled.
+
+Object encoding options are:
+
+- `standard`: Default encoder when boolean set. Encodes and decodes objects as byte arrays.
+- `image`: Image encoder. Encodes and decodes objects as image objects.
+- `pickle`: Pickle encoder. Encodes and decodes objects with the pickle module. Supports arbitrary objects.
+
+## functions
+```yaml
+functions: list
+```
+
+List of functions with user-defined SQL functions, only used when [content](#content) is enabled. Each list element must be one of the following:
+
+- function
+- callable object
+- dict with fields for name, argcount and function
+
+[An example can be found here](../../query#custom-sql-functions).
+
+## query
+```yaml
+query:
+    path: sets the path for the query model - this can be any model on the
+          Hugging Face Model Hub or a local file path.
+    prefix: text prefix to prepend to all inputs
+    maxlength: maximum generated sequence length
+```
+
+Query translation model. Translates natural language queries to txtai compatible SQL statements.

docs/embeddings/configuration/general.md

@@ -0,0 +1,47 @@
+# General
+
+General configuration options that don't fit elsewhere.
+
+## keyword
+```yaml
+keyword: boolean
+```
+
+Enables sparse keyword indexing for this embeddings.
+
+## hybrid
+```yaml
+hybrid: boolean
+```
+
+Enables hybrid (sparse + dense) indexing for this embeddings.
+
+## indexes
+```yaml
+indexes: dict
+```
+
+Key value pairs defining subindexes for this embeddings. Each key is the index name and the value is the full configuration. This configuration can use any of the available configurations in a standard embeddings instance.
+
+## autoid
+```yaml
+format: int|uuid function
+```
+
+Sets the auto id generation method. When this is not set, an autogenerated numeric sequence is used. This also supports [UUID generation functions](https://docs.python.org/3/library/uuid.html#uuid.uuid1). For example, setting this value to `uuid4` will generate random UUIDs. Setting this to `uuid5` will generate deterministic UUIDs for each input data row.
+
+## columns
+```yaml
+columns:
+    text: name of the text column
+    object: name of the object column
+```
+
+Sets the `text` and `object` column names. Defaults to `text` and `object` if not provided.
+
+## format
+```yaml
+format: json|pickle
+```
+
+Sets the configuration storage format. Defaults to `json`.

docs/embeddings/configuration/graph.md

@@ -0,0 +1,70 @@
+# Graph
+
+Enable graph storage via the `graph` parameter. This component requires the [graph](../../../install/#graph) extras package.
+
+When enabled, a graph network is built using the embeddings index. Graph nodes are synced with each embeddings index operation (index/upsert/delete). Graph edges are created using the embeddings index upon completion of each index/upsert/delete embeddings index call.
+
+## backend
+```yaml
+backend: networkx|rdbms|custom
+```
+
+Sets the graph backend. Defaults to `networkx`.
+
+Add custom graph storage engines via setting this parameter to the fully resolvable class string.
+
+The `rdbms` backend has the following additional settings.
+
+### rdbms
+```yaml
+url: database url connection string, alternatively can be set via the
+     GRAPH_URL environment variable
+nodes: table to store node data, defaults to `nodes`
+edges: table to store edge data, defaults to `edges`
+```
+
+## batchsize
+```yaml
+batchsize: int
+```
+
+Batch query size, used to query embeddings index - defaults to 256.
+
+## limit
+```yaml
+limit: int
+```
+
+Maximum number of results to return per embeddings query - defaults to 15.
+
+## minscore
+```yaml
+minscore: float
+```
+
+Minimum score required to consider embeddings query matches - defaults to 0.1.
+
+## approximate
+```yaml
+approximate: boolean
+```
+
+When true, queries only run for nodes without edges - defaults to true.
+
+## topics
+```yaml
+topics:
+    algorithm: community detection algorithm (string), options are
+               louvain (default), greedy, lpa
+    level: controls number of topics (string), options are best (default) or first
+    resolution: controls number of topics (int), larger values create more
+                topics (int), defaults to 100
+    labels: scoring index method used to build topic labels (string)
+            options are bm25 (default), tfidf, sif
+    terms: number of frequent terms to use for topic labels (int), defaults to 4
+    stopwords: optional list of stop words to exclude from topic labels
+    categories: optional list of categories used to group topics, allows
+                granular topics with broad categories grouping topics
+```
+
+Enables topic modeling. Defaults are tuned so that in most cases these values don't need to be changed (except for categories). These parameters are available for advanced use cases where one wants full control over the community detection process.

docs/embeddings/configuration/index.md

@@ -1,205 +1,51 @@
 # Configuration
 
-This following describes available embeddings configuration. These parameters are set via the [Embeddings constructor](../methods#txtai.embeddings.base.Embeddings.__init__).
+The following describes available embeddings configuration. These parameters are set in the [Embeddings constructor](../methods#txtai.embeddings.base.Embeddings.__init__) via either the `config` parameter or as keyword arguments.
 
-## path
-```yaml
-path: string
+Configuration is designed to be optional and set only when needed. Out of the box, sensible defaults are picked to get up and running fast. For example:
+
+```python
+from txtai.embeddings import Embeddings
+
+embeddings = Embeddings()
 ```
 
-Sets the path for a vectors model. When using a transformers/sentence-transformers model, this can be any model on the
-[Hugging Face Model Hub](https://huggingface.co/models) or a local file path. Otherwise, it must be a local file path to a word embeddings model.
+Creates a new embeddings instance, using [all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) as the vector model, [Faiss](https://faiss.ai/) as the ANN index backend and content disabled.
 
-## method
-```yaml
-method: transformers|sentence-transformers|words|external
+```python
+from txtai.embeddings import Embeddings
+
+embeddings = Embeddings(content=True)
 ```
 
-Sentence embeddings method to use. If the method is not provided, it is inferred using the `path`.
+Is the same as above except it adds in [SQLite](https://www.sqlite.org/index.html) for content storage. 
 
-`sentence-transformers` and `words` require the [similarity](../../install/#similarity) extras package to be installed.
+The following sections link to all the available configuration options.
 
-### transformers
+## [ANN](./ann)
 
-Builds sentence embeddings using a transformers model. While this can be any transformers model, it works best with
-[models trained](https://huggingface.co/models?pipeline_tag=sentence-similarity) to build sentence embeddings.
+The default vector index backend is Faiss.
 
-### sentence-transformers
+## [Cloud](./cloud)
 
-Same as transformers but loads models with the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) library.
+Embeddings databases can optionally be synced with cloud storage.
 
-### words
+## [Database](./database)
 
-Builds sentence embeddings using a word embeddings model. Transformers models are the preferred vector backend in most cases. Word embeddings models may be deprecated in the future.
+Content storage is disabled by default. When enabled, SQLite is the default storage engine.
 
-#### storevectors
-```yaml
-storevectors: boolean
-```
+## [General](./general)
 
-Enables copying of a vectors model set in path into the embeddings models output directory on save. This option enables a fully encapsulated index with no external file dependencies.
+General configuration that doesn't fit elsewhere.
 
-#### scoring
-```yaml
-scoring: bm25|tfidf|sif
-```
+## [Graph](./graph)
 
-A scoring model builds weighted averages of word vectors for a given sentence. Supports BM25, TF-IDF and SIF (smooth inverse frequency) methods. If a scoring method is not provided, mean sentence embeddings are built.
+An accomplying graph index can be created with an embeddings database. This enables topic modeling, path traversal and more. [NetworkX](https://github.com/networkx/networkx) is the default graph index.
 
-#### pca
-```yaml
-pca: int
-```
+## [Scoring](./scoring)
 
-Removes _n_ principal components from generated sentence embeddings. When enabled, a TruncatedSVD model is built to help with dimensionality reduction. After pooling of vectors creates a single sentence embedding, this method is applied.
+Sparse keyword indexing and word vectors term weighting.
 
-### external
+## [Vectors](./vectors)
 
-Sentence embeddings are loaded via an external model or API. Requires setting the [transform](#transform) parameter to a function that translates data into vectors.
-
-#### transform
-```yaml
-transform: function
-```
-
-When method is `external`, this function transforms input content into embeddings. The input to this function is a list of data. This method must return either a numpy array or list of numpy arrays.
-
-## batch
-```yaml
-batch: int
-```
-
-Sets the transform batch size. This parameter controls how input streams are chunked and vectorized.
-
-## encodebatch
-```yaml
-encodebatch: int
-```
-
-Sets the encode batch size. This parameter controls the underlying vector model batch size. This often corresponds to a GPU batch size, which controls GPU memory usage.
-
-## tokenize
-```yaml
-tokenize: boolean
-```
-
-Enables string tokenization (defaults to false). This method applies tokenization rules that only work with English language text and may increase the quality of
-English language sentence embeddings in some situations.
-
-## backend
-```yaml
-backend: faiss|hnsw|annoy
-```
-
-Approximate Nearest Neighbor (ANN) index backend for storing generated sentence embeddings. `Defaults to faiss`. Additional backends require the
-[similarity](../../install/#similarity) extras package to be installed.
-
-Backend-specific settings are set with a corresponding configuration object having the same name as the backend (i.e. annoy, faiss, or hnsw). None of these are required and are set to defaults if omitted.
-
-### faiss
-```yaml
-faiss:
-    components: comma separated list of components - defaults to "Flat" for small
-                indices and "IVFx,Flat" for larger indexes where
-                x = 4 * sqrt(embeddings count)
-    nprobe: search probe setting (int) - defaults to x/16 (as defined above)
-            for larger indexes
-    quantize: store vectors with 8-bit precision vs 32-bit (boolean)
-              defaults to false
-    mmap: load as on-disk index (boolean) - trade query response time for a
-          smaller RAM footprint, defaults to false
-```
-
-See the following Faiss documentation links for more information.
-
-- [Guidelines for choosing an index](https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index)
-- [Index configuration summary](https://github.com/facebookresearch/faiss/wiki/Faiss-indexes)
-- [Index Factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory)
-- [Search Tuning](https://github.com/facebookresearch/faiss/wiki/Faster-search)
-
-### hnsw
-```yaml
-hnsw:
-    efconstruction:  ef_construction param for init_index (int) - defaults to 200
-    m: M param for init_index (int) - defaults to 16
-    randomseed: random-seed param for init_index (int) - defaults to 100
-    efsearch: ef search param (int) - defaults to None and not set
-```
-
-See [Hnswlib documentation](https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md) for more information on these parameters.
-
-### annoy
-```yaml
-annoy:
-    ntrees: number of trees (int) - defaults to 10
-    searchk: search_k search setting (int) - defaults to -1
-```
-
-See [Annoy documentation](https://github.com/spotify/annoy#full-python-api) for more information on these parameters. Note that annoy indexes can not be modified after creation, upserts/deletes and other modifications are not supported.
-
-## content
-```yaml
-content: string|boolean
-```
-
-Enables content storage. When true, the default content storage engine will be used. Otherwise, the string must specify the supported content storage engine to use.
-
-## functions
-```yaml
-functions: list
-```
-
-List of functions with user-defined SQL functions, only used when [content](#content) is enabled. Each list element must be one of the following:
-
-- function
-- callable object
-- dict with fields for name, argcount and function
-
-[An example can be found here](../query#custom-sql-functions).
-
-## query
-```yaml
-query:
-    path: sets the path for the query model - this can be any model on the
-          Hugging Face Model Hub or a local file path.
-    prefix: text prefix to prepend to all inputs
-    maxlength: maximum generated sequence length
-```
-
-Query translation model. Translates natural language queries to txtai compatible SQL statements.
-
-## graph
-```yaml
-graph:
-    backend: graph network backend (string), defaults to "networkx"
-    batchsize: batch query size, used to query embeddings index (int)
-               defaults to 256
-    limit: maximum number of results to return per embeddings query (int)
-           defaults to 15
-    minscore: minimum score required to consider embeddings query matches (float)
-              defaults to 0.1
-    approximate: when true, queries only run for nodes without edges (boolean)
-                 defaults to true
-    topics: see below
-```
-
-Enables graph storage. When set, a graph network is built using the embeddings index. Graph nodes are synced with each embeddings index operation (index/upsert/delete). Graph edges are created using the embeddings index upon completion of each index/upsert/delete embeddings index call.
-
-Defaults are tuned so that in most cases these values don't need to be changed. 
-
-### topics
-```yaml
-topics:
-    algorithm: community detection algorithm (string), options are
-               louvain (default), greedy, lpa
-    level: controls number of topics (string), options are best (default) or first
-    resolution: controls number of topics (int), larger values create more
-                topics (int), defaults to 100
-    labels: scoring index method used to build topic labels (string)
-            options are bm25 (default), tfidf, sif
-    terms: number of frequent terms to use for topic labels (int), defaults to 4
-    categories: optional list of categories used to group topics, allows
-                granular topics with broad categories grouping topics
-```
-
-Enables topic modeling. Defaults are tuned so that in most cases these values don't need to be changed (except for categories). These parameters are available for advanced use cases where one wants full control over the community detection process.
+Vector search is enabled by converting text and other binary data into embeddings vectors. These vectors are then stored in an ANN index. The vector model is optional and a default model is used when not provided.

docs/embeddings/configuration/scoring.md

@@ -0,0 +1,32 @@
+# Scoring
+
+Enable scoring support via the `scoring` parameter.
+
+This scoring instance can serve two purposes, depending on the settings.
+
+One use case is building sparse/keyword indexes. This occurs when the `terms` parameter is set to `True`.
+
+The other use case is with word vector term weighting. This feature has been available since the initial version but isn't quite as common anymore.
+
+The following covers the available options.
+
+## method
+```yaml
+method: bm25|tfidf|sif|custom
+```
+
+Sets the scoring method. Add custom scoring via setting this parameter to the fully resolvable class string.
+
+## terms
+```yaml
+terms: boolean
+```
+
+Enables term frequency sparse arrays for a scoring instance. This is the backend for sparse keyword indexes.
+
+## normalize
+```yaml
+normalize: boolean
+```
+
+Enables normalized scoring (ranging from 0 to 1). When enabled, statistics from the index will be used to calculate normalized scores.

docs/embeddings/configuration/vectors.md

@@ -0,0 +1,139 @@
+# Vectors
+
+The following covers available vector model configuration options.
+
+## path
+```yaml
+path: string
+```
+
+Sets the path for a vectors model. When using a transformers/sentence-transformers model, this can be any model on the
+[Hugging Face Hub](https://huggingface.co/models) or a local file path. Otherwise, it must be a local file path to a word embeddings model.
+
+## method
+```yaml
+method: transformers|sentence-transformers|llama.cpp|litellm|external|words
+```
+
+Embeddings method to use. If the method is not provided, it is inferred using the `path`.
+
+`sentence-transformers`, `llama.cpp`, `litellm` and `words` require the [vectors](../../../install/#vectors) extras package to be installed.
+
+### transformers
+
+Builds embeddings using a transformers model. While this can be any transformers model, it works best with
+[models trained](https://huggingface.co/models?pipeline_tag=sentence-similarity) to build embeddings.
+
+Both `mean` and `cls` pooling are supported and automatically infered from the model. The pooling method can be overwritten by changing the method
+from `transformers` to `meanpooling` or `clspooling` respectively.
+
+### sentence-transformers
+
+Same as transformers but loads models with the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) library.
+
+### llama.cpp
+
+Builds embeddings using a llama.cpp model. Supports both local and remote GGUF paths on the HF Hub.
+
+### litellm
+
+Builds embeddings using a LiteLLM model. See the [LiteLLM documentation](https://litellm.vercel.app/docs/providers) for the options available with LiteLLM models.
+
+### words
+
+Builds embeddings using a word embeddings model. Transformers models are the preferred vector backend in most cases. Word embeddings models may be deprecated in the future.
+
+#### storevectors
+```yaml
+storevectors: boolean
+```
+
+Enables copying of a vectors model set in path into the embeddings models output directory on save. This option enables a fully encapsulated index with no external file dependencies.
+
+#### pca
+```yaml
+pca: int
+```
+
+Removes _n_ principal components from generated embeddings. When enabled, a TruncatedSVD model is built to help with dimensionality reduction. After pooling of vectors creates a single embedding, this method is applied.
+
+### external
+
+Embeddings are created via an external model or API. Requires setting the [transform](#transform) parameter to a function that translates data into embeddings.
+
+#### transform
+```yaml
+transform: function
+```
+
+When method is `external`, this function transforms input content into embeddings. The input to this function is a list of data. This method must return either a numpy array or list of numpy arrays.
+
+## gpu
+```yaml
+gpu: boolean|int|string|device
+```
+
+Set the target device. Supports true/false, device id, device string and torch device instance. This is automatically derived if omitted.
+
+## batch
+```yaml
+batch: int
+```
+
+Sets the transform batch size. This parameter controls how input streams are chunked and vectorized.
+
+## encodebatch
+```yaml
+encodebatch: int
+```
+
+Sets the encode batch size. This parameter controls the underlying vector model batch size. This often corresponds to a GPU batch size, which controls GPU memory usage.
+
+## dimensionality
+```yaml
+dimensionality: int
+```
+
+Enables truncation of vectors to this dimensionality. This is only useful for models trained to store more important information in earlier dimensions such as [Matryoshka Representation Learning (MRL)](https://huggingface.co/blog/matryoshka).
+
+## quantize
+```yaml
+quantize: int|boolean
+```
+
+Enables scalar quantization at the specified precision. Supports 1-bit through 8-bit quantization. Scalar quantization transforms continuous floating point values
+to discrete unsigned integers. Quantized data storage is only supported by the `faiss`, `numpy` and `torch` ANN backends.
+
+This parameter supports booleans for backwards compatability. When set to true/false, this flag sets [faiss.quantize](../ann/#faiss).
+
+## instructions
+```yaml
+instructions:
+    query: prefix for queries
+    data: prefix for indexing
+```
+
+Instruction-based models use prefixes to modify how embeddings are computed. This is especially useful with asymmetric search, which is when the query and indexed data are of vastly different lengths. In other words, short queries with long documents.
+
+[E5-base](https://huggingface.co/intfloat/e5-base) is an example of a model that accepts instructions. It takes `query: ` and `passage: ` prefixes and uses those to generate embeddings that work well for asymmetric search.
+
+## models
+```yaml
+models: dict
+```
+
+Loads and stores vector models in this cache. This is primarily used with subindexes but can be set on any embeddings instance. This prevents the same model from being loaded multiple times when working with multiple embeddings instances.
+
+## tokenize
+```yaml
+tokenize: boolean
+```
+
+Enables string tokenization (defaults to false). This method applies tokenization rules that only work with English language text. It's not recommended for use with recent vector models.
+
+## vectors
+```yaml
+vectors: dict
+```
+
+Passes these additional parameters to the underlying vector model.

docs/embeddings/index.md

@@ -3,7 +3,7 @@
 ![embeddings](../images/embeddings.png#only-light)
 ![embeddings](../images/embeddings-dark.png#only-dark)
 
-Embeddings is the engine that delivers semantic search. Data is transformed into embeddings vectors where similar concepts will produce similar vectors. Indexes both large and small are built with these vectors. The indexes are used to find results that have the same meaning, not necessarily the same keywords.
+Embeddings databases are the engine that delivers semantic search. Data is transformed into embeddings vectors where similar concepts will produce similar vectors. Indexes both large and small are built with these vectors. The indexes are used to find results that have the same meaning, not necessarily the same keywords.
 
 The following code snippet shows how to build and search an embeddings index.
 
@@ -11,7 +11,7 @@ The following code snippet shows how to build and search an embeddings index.
 from txtai.embeddings import Embeddings
 
 # Create embeddings model, backed by sentence-transformers & transformers
-embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2"})
+embeddings = Embeddings(path="sentence-transformers/nli-mpnet-base-v2")
 
 data = [
   "US tops 5 million confirmed virus cases",
@@ -24,10 +24,10 @@ data = [
   "Make huge profits without work, earn up to $100,000 a day"
 ]
 
-# Create an index for the list of text
-embeddings.index([(uid, text, None) for uid, text in enumerate(data)])
+# Index the list of text
+embeddings.index(data)
 
-print("%-20s %s" % ("Query", "Best Match"))
+print(f"{'Query':20} Best Match")
 print("-" * 50)
 
 # Run an embeddings search for each query
@@ -38,30 +38,32 @@ for query in ("feel good story", "climate change", "public health story", "war",
     uid = embeddings.search(query, 1)[0][0]
 
     # Print text
-    print("%-20s %s" % (query, data[uid]))
+    print(f"{query:20} {data[uid]}")
 ```
 
 ## Build
 
-An embeddings instance can be created as follows:
+An embeddings instance is [configuration-driven](configuration) based on what is passed in the constructor. Vectors are stored with the option to also [store content](configuration/database#content). Content storage enables additional filtering and data retrieval options.
+
+The example above sets a specific embeddings vector model via the [path](configuration/vectors/#path) parameter. An embeddings instance with no configuration can also be created.
 
 ```python
-embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2"})
+embeddings = Embeddings()
 ```
 
-The example above builds a transformers-based embeddings instance. In this case, when loading and searching for data, a transformers model is used to vectorize data.
-
-The embeddings instance is [configuration-driven](configuration) based on what is passed in the constructor. Embeddings indexes store vectors and can optionally [store content](configuration#content). Content storage enables additional filtering and data retrieval options.
+In this case, when loading and searching for data, the [default transformers vector model](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) is used to vectorize data. See the [model guide](../models) for current model recommentations.
 
 ## Index
 
 After creating a new embeddings instance, the next step is adding data to it.
 
 ```python
-embeddings.index([(uid, text, None) for uid, text in enumerate(data)])
+embeddings.index(rows)
 ```
 
-The index method takes an iterable collection of tuples with three values. 
+The index method takes an iterable and supports the following formats for each element.
+
+- `(id, data, tags)` - default processing format
 
 | Element     | Description                                                   |
 | ----------- | ------------------------------------------------------------- |
@@ -69,7 +71,15 @@ The index method takes an iterable collection of tuples with three values.
 | data        | input data to index, can be text, a dictionary or object      |
 | tags        | optional tags string, used to mark/label data as it's indexed |
 
-When the data element is a dictionary and it has a field named `text`, that will be used for indexing.
+- `(id, data)`
+
+  Same as above but without tags.
+
+- `data`
+
+Single element to index. In this case, unique id's will automatically be generated. Note that for generated id's, [upsert](methods/#txtai.embeddings.base.Embeddings.upsert) and [delete](methods/#txtai.embeddings.base.Embeddings.delete) calls require a separate search to get the target ids.
+
+When the data field is a dictionary, text is passed via the `text` key, binary objects via the `object` key. Note that [content](configuration/database#content) must be enabled to store metadata and [objects](configuration/database#objects) to store binary object data. The `id` and `tags` keys will be extracted, if provided.
 
 The input iterable can be a list or generator. [Generators](https://wiki.python.org/moin/Generators) help with indexing very large datasets as only portions of the data is in memory at any given time.
 
@@ -83,7 +93,7 @@ Once data is indexed, it is ready for search.
 embeddings.search(query, limit)
 ```
 
-The search method takes two parameters, the query and query limit. The results format is different based on whether [content](configuration/#content) is stored or not.
+The search method takes two parameters, the query and query limit. The results format is different based on whether [content](configuration/database#content) is stored or not.
 
 - List of `(id, score)` when content is _not_ stored
 - List of `{**query columns}` when content is stored

docs/embeddings/indexing.md

@@ -3,23 +3,25 @@
 ![indexing](../images/indexing.png#only-light)
 ![indexing](../images/indexing-dark.png#only-dark)
 
-This section gives an in-depth overview on how to index data with txtai. We'll cover vectorization, indexing, updating and deleting data.
+This section gives an in-depth overview on how to index data with txtai. We'll cover vectorization, indexing/updating/deleting data and the various components of an embeddings database.
 
 ## Vectorization
 
-The most compute intensive step in building an index is vectorization. The [path](../configuration#path) parameter sets the path to the vector model. There is logic to automatically detect the vector model [method](../configuration#method) but it can also be set directly.
+The most compute intensive step in building an index is vectorization. The [path](../configuration/vectors#path) parameter sets the path to the vector model. There is logic to automatically detect the vector model [method](../configuration/vectors#method) but it can also be set directly.
 
-The [batch](../configuration#batch) and [encodebatch](../configuration#encodebatch) parameters control the vectorization process. Larger values for `batch` will pass larger batches to the vectorization method. Larger values for `encodebatch` will pass larger batches for each vector encode call. In the case of GPU vector models, larger values will consume more GPU memory.
+The [batch](../configuration/vectors#batch) and [encodebatch](../configuration/vectors#encodebatch) parameters control the vectorization process. Larger values for `batch` will pass larger batches to the vectorization method. Larger values for `encodebatch` will pass larger batches for each vector encode call. In the case of GPU vector models, larger values will consume more GPU memory.
 
-Data is buffered to temporary storage during indexing as embeddings vectors can be quite large (for example 768 dimensions of float32 is 768 * 4 = 3072 bytes per vector). Once vectorization is complete, a mmapped array is created with all vectors for [Approximate Nearest Neighbor (ANN)](../configuration#backend) indexing.
+Data is buffered to temporary storage during indexing as embeddings vectors can be quite large (for example 768 dimensions of float32 is 768 * 4 = 3072 bytes per vector). Once vectorization is complete, a mmapped array is created with all vectors for [Approximate Nearest Neighbor (ANN)](../configuration/vectors#backend) indexing.
+
+The terms `ANN` and `dense vector index` are used interchangeably throughout txtai's documentation.
 
 ## Setting a backend
 
-As mentioned above, computed vectors are stored in an ANN. There are various index [backends](../configuration#backend) that can be configured. Faiss is the default backend.
+As mentioned above, computed vectors are stored in an ANN. There are various index [backends](../configuration/ann#backend) that can be configured. Faiss is the default backend.
 
 ## Content storage
 
-Embeddings indexes can optionally [store content](../configuration#content). When this is enabled, the input content is saved in a database alongside the computed vectors. This enables filtering on additional fields and content retrieval.
+Embeddings indexes can optionally [store content](../configuration/database#content). When this is enabled, the input content is saved in a database alongside the computed vectors. This enables filtering on additional fields and content retrieval.
 
 ## Index vs Upsert
 
@@ -70,17 +72,17 @@ embeddings.delete(ids)
 
 ## Reindex
 
-When [content storage](../configuration#content) is enabled, [reindex](../methods#txtai.embeddings.base.Embeddings.reindex) can be called to rebuild the index with new settings. For example, the backend can be switched from faiss to hnsw or the vector model can be updated. This prevents having to go back to the original raw data. 
+When [content storage](../configuration/database#content) is enabled, [reindex](../methods#txtai.embeddings.base.Embeddings.reindex) can be called to rebuild the index with new settings. For example, the backend can be switched from faiss to hnsw or the vector model can be updated. This prevents having to go back to the original raw data. 
 
 ```python
-embeddings.reindex({"path": "sentence-transformers/all-MiniLM-L6-v2", "backend": "hnsw"})
+embeddings.reindex(path="sentence-transformers/all-MiniLM-L6-v2", backend="hnsw")
 ```
 
 ## Graph
 
 Dimensionality reduction with UMAP combined with HDBSCAN is a popular topic modeling method found in a number of libraries. txtai takes a different approach with a semantic graph.
 
-Enabling a [graph network](../configuration#graph) adds a semantic graph at index time as data is being vectorized. Vector embeddings are used to create relationships in the graph. Finally, community detection algorithms build topic clusters. Semantic graphs can also be used to analyze data connectivity.
+Enabling a [graph network](../configuration/graph) adds a semantic graph at index time as data is being vectorized. Vector embeddings are used to create relationships in the graph. Finally, community detection algorithms build topic clusters. Semantic graphs can also be used to analyze data connectivity.
 
 This approach has the advantage of only having to vectorize data once. It also has the advantage of better topic precision given there isn't a dimensionality reduction operation (UMAP). Semantic graph examples are shown below.
 
@@ -104,15 +106,25 @@ embeddings.graph.showpath(id1, id2)
 
 Graphs are persisted alongside an embeddings index. Each save and load will also save and load the graph.
 
-## Scoring
+## Sparse vectors
 
-When using [word vector backed models](../configuration#words) with scoring set, a separate call is required before calling `index` as follows:
+Scoring instances can create a standalone [keyword](../configuration/general#keyword) or sparse index (BM25, TF-IDF). This enables [hybrid](../configuration/general/#hybrid) search when there is an available dense vector index.
+
+The terms `sparse vector index`, `keyword index`, `terms index` and `scoring index` are used interchangeably throughout txtai's documentation.
+
+See [this link](../../examples/#semantic-search) to learn more.
+
+## Subindexes
+
+An embeddings instance can optionally have associated [subindexes](../configuration/general/#indexes), which are also embeddings databases. This enables indexing additional fields, vector models and much more.
+
+## Word vectors
+
+When using [word vector backed models](../configuration/vectors#words) with scoring set, a separate call is required before calling `index` as follows:
 
 ```python
-embeddings.score([(uid, text, None) for uid, text in enumerate(data)])
-embeddings.index([(uid, text, None) for uid, text in enumerate(data)])
+embeddings.score(rows)
+embeddings.index(rows)
 ```
 
-Two calls are required to support generator-backed iteration of data. The scoring index requires a separate full-pass of the data.
-
-Scoring instances can also create a standalone keyword-based index (BM25, TF-IDF). See [this link](../../examples/#semantic-search) to learn more.
+Both calls are required to support generator-backed iteration of data with word vectors models.

docs/embeddings/methods.md

@@ -1,12 +1,22 @@
 # Methods
 
 ::: txtai.embeddings.Embeddings
-    selection:
+    options:
         filters:
+            - "!columns"
+            - "!createann"
+            - "!createcloud"
             - "!createdatabase"
             - "!creategraph"
+            - "!createids"
+            - "!createindexes"
+            - "!createscoring"
             - "!checkarchive"
             - "!configure"
+            - "!defaultallowed"
+            - "!defaults"
+            - "!initindex"
+            - "!loadconfig"
             - "!loadquery"
             - "!loadvectors"
-            - "!normalize"
+            - "!saveconfig"

docs/embeddings/query.md

@@ -18,7 +18,7 @@ The queries above [search](../methods#txtai.embeddings.base.Embeddings.search) t
 
 ## SQL
 
-txtai supports more complex queries with SQL. This is only supported if [content storage](../configuration/#content) is enabled. txtai has a translation layer that analyzes input SQL statements and combines similarity results with content stored in a relational database.
+txtai supports more complex queries with SQL. This is only supported if [content storage](../configuration/database#content) is enabled. txtai has a translation layer that analyzes input SQL statements and combines similarity results with content stored in a relational database.
 
 SQL queries are run through `embeddings.search` like natural language queries but the examples below only show the SQL query for conciseness.
 
@@ -32,43 +32,50 @@ The similar clause is a txtai function that enables similarity searches with SQL
 
 ```sql
 SELECT id, text, score FROM txtai WHERE similar('feel good story')
-SELECT id, text, score FROM txtai WHERE similar('feel good story')
 ```
 
-The similar clause takes two arguments:
+The similar clause takes the following arguments:
 
 ```sql
-similar("query", "number of candidates")
+similar("query", "number of candidates", "index", "weights")
 ```
 
 | Argument              | Description                            |
 | --------------------- | ---------------------------------------|
 | query                 | natural language query to run          |
 | number of candidates  | number of candidate results to return  |
+| index                 | target index name                      |
+| weights               | hybrid score weights                   |
 
-The txtai query layer has to join results from two separate components, a relational store and a similarity index. With a similar clause, a similarity search is run and those ids are fed to the underlying database query.
+The txtai query layer joins results from two separate components, a relational store and a similarity index. With a similar clause, a similarity search is run and those ids are fed to the underlying database query.
 
 The number of candidates should be larger than the desired number of results when applying additional filter clauses. This ensures that `limit` results are still returned after applying additional filters. If the number of candidates is not specified, it is defaulted as follows:
 
 - For a single query filter clause, the default is the query limit
 - With multiple filtering clauses, the default is 10x the query limit
 
+The index name is only applicable when [subindexes](../configuration/general/#indexes) are enabled. This specifies the index to use for the query.
+
+Weights sets the hybrid score weights when an index has both a sparse and dense index.
+
 ### Dynamic columns
 
-Content can be indexed in multiple ways when content storage is enabled. [Remember that input documents](../#index) take the form of `(id, data, tags)` tuples. If data is a string, then content is primarily filtered with similar clauses. If data is a dictionary, then all fields in the dictionary are indexed and searchable.
+Content can be indexed in multiple ways when content storage is enabled. [Remember that input documents](../#index) take the form of `(id, data, tags)` tuples. If data is a string or binary content, it's indexed and searchable with `similar()` clauses.
+
+If data is a dictionary, then all fields in the dictionary are stored and available via SQL. The `text` field or [field specified in the index configuration](../configuration/general/#columns) is indexed and searchable with `similar()` clauses.
 
 For example:
 
 ```python
-embeddings.index([(0, {"text": "text to index", "flag": True,
-                       "entry": "2022-01-01"}, None)])
+embeddings.index([{"text": "text to index", "flag": True,
+                   "actiondate": "2022-01-01"}])
 ```
 
 With the above input data, queries can now have more complex filters.
 
 ```sql
-SELECT text, flag, entry FROM txtai WHERE similar('query') AND flag = 1
-AND entry >= '2022-01-01'
+SELECT text, flag, actiondate FROM txtai WHERE similar('query') AND flag = 1
+AND actiondate >= '2022-01-01'
 ```
 
 txtai's query layer automatically detects columns and translates queries into a format that can be understood by the underlying database.
@@ -76,16 +83,30 @@ txtai's query layer automatically detects columns and translates queries into a
 Nested dictionaries/JSON is supported and can be escaped with bracket statements.
 
 ```python
-embeddings.index([(0, {"text": "text to index",
-                       "parent": {"child element": "abc"}}, None)])
+embeddings.index([{"text": "text to index",
+                   "parent": {"child element": "abc"}}])
 ```
 
 ```sql
-SELECT text FROM txtai WHERE [parent.child element] ='abc'
+SELECT text FROM txtai WHERE [parent.child element] = 'abc'
 ```
 
 Note the bracket statement escaping the nested column with spaces in the name.
 
+### Bind parameters
+
+txtai has support for SQL bind parameters.
+
+```python
+# Query with a bind parameter for similar clause
+query = "SELECT id, text, score FROM txtai WHERE similar(:x)"
+results = embeddings.search(query, parameters={"x": "feel good story"})
+
+# Query with a bind parameter for column filter
+query = "SELECT text, flag, actiondate FROM txtai WHERE flag = :x"
+results = embeddings.search(query, parameters={"x": 1})
+```
+
 ### Aggregation queries
 
 The goal of txtai's query language is to closely support all functions in the underlying database engine. The main challenge is ensuring dynamic columns are properly escaped into the engines native query function. 
@@ -104,16 +125,23 @@ SELECT count(*), flag FROM txtai GROUP BY flag ORDER BY count(*) DESC
 txtai has support for storing and retrieving binary objects. Binary objects can be retrieved as shown in the example below.
 
 ```python
+# Create embeddings index with content and object storage enabled
+embeddings = Embeddings(content=True, objects=True)
+
 # Get an image
-request = open("demo.gif")
+request = open("demo.gif", "rb")
 
 # Insert record
 embeddings.index([("txtai", {"text": "txtai executes machine-learning workflows.",
-                             "object": request.read()}, None)])
+                             "object": request.read()})])
 
 # Query txtai and get associated object
-query = "select object from txtai where similar('machine learning') limit 1"
+query = "SELECT object FROM txtai WHERE similar('machine learning') LIMIT 1"
 result = embeddings.search(query)[0]["object"]
+
+# Query binary content with a bind parameter
+query = "SELECT object FROM txtai WHERE similar(:x) LIMIT 1"
+results = embeddings.search(query, parameters={"x": request.read()})
 ```
 
 ## Custom SQL functions
@@ -125,19 +153,19 @@ Custom, user-defined SQL functions extend selection, filtering and ordering clau
 translate = Translation()
 
 # Create embeddings index
-embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2",
-                         "content": True,
-                         "functions": [translate]})
+embeddings = Embeddings(path="sentence-transformers/nli-mpnet-base-v2",
+                        content=True,
+                        functions=[translate]})
 
 # Run a search using a custom SQL function
 embeddings.search("""
-select
+SELECT
   text,
   translation(text, 'de', null) 'text (DE)',
   translation(text, 'es', null) 'text (ES)',
   translation(text, 'fr', null) 'text (FR)'
-from txtai where similar('feel good story')
-limit 1
+FROM txtai WHERE similar('feel good story')
+LIMIT 1
 """)
 ```
 
@@ -156,12 +184,53 @@ select id, text, score from txtai where similar('feel good story') and
 entry >= date('now', '-1 day')
 ```
 
-This requires setting a [query translation model](../configuration#query). The default query translation model is [t5-small-txtsql](https://huggingface.co/NeuML/t5-small-txtsql) but this can easily be finetuned to handle different use cases.
+This requires setting a [query translation model](../configuration/database#query). The default query translation model is [t5-small-txtsql](https://huggingface.co/NeuML/t5-small-txtsql) but this can easily be finetuned to handle different use cases.
+
+## Hybrid search
+
+When an embeddings database has both a sparse and dense index, both indexes will be queried and the results will be equally weighted unless otherwise specified.
+
+```python
+embeddings.search("query", weights=0.5)
+embeddings.search("SELECT id, text, score FROM txtai WHERE similar('query', 0.5)")
+```
+
+## Graph search
+
+If an embeddings database has an associated graph network, graph searches can be run. The search syntax below uses [openCypher](https://github.com/opencypher/openCypher). Follow the preceding link to learn more about this syntax.
+
+Additionally, standard embeddings searches can be returned as graphs.
+
+```python
+# Find all paths between id: 0 and id: 5 between 1 and 3 hops away
+embeddings.graph.search("""
+MATCH P=({id: 0})-[*1..3]->({id: 5})
+RETURN P
+""")
+
+# Standard embeddings search as graph
+embeddings.search("query", graph=True)
+```
+
+## Subindexes
+
+Subindexes can be queried as follows:
+
+```python
+# Query with index parameter
+embeddings.search("query", index="subindex1")
+
+# Specify with SQL
+embeddings.search("""
+SELECT id, text, score FROM txtai
+WHERE similar('query', 'subindex1')
+""")
+```
 
 ## Combined index architecture
 
-When content storage is enabled, txtai becomes a dual storage engine. Content is stored in an underlying database (currently supports SQLite) along with an Approximate Nearest Neighbor (ANN) index. These components combine to deliver similarity search alongside traditional structured search.
+txtai has multiple storage and indexing components. Content is stored in an underlying database along with an approximate nearest neighbor (ANN) index, keyword index and graph network. These components combine to deliver similarity search alongside traditional structured search.
 
-The ANN index stores ids and vectors for each input element. When a natural language query is run, the query is translated into a vector and a similarity query finds the best matching ids. When a database is added into the mix, an additional step is applied. This step takes those ids and effectively inserts them as part of the underlying database query.
+The ANN index stores ids and vectors for each input element. When a natural language query is run, the query is translated into a vector and a similarity query finds the best matching ids. When a database is added into the mix, an additional step is executed. This step takes those ids and effectively inserts them as part of the underlying database query. The same steps apply with keyword indexes except a term frequency index is used to find the best matching ids.
 
-Dynamic columns are supported via the underlying engine. For SQLite, data is stored as JSON and dynamic columns are converted into `json_extract` clauses. This same concept can be expanded to other storage engines like PostgreSQL and could even work with NoSQL stores. 
+Dynamic columns are supported via the underlying engine. For SQLite, data is stored as JSON and dynamic columns are converted into `json_extract` clauses. Client-server databases are supported via [SQLAlchemy](https://docs.sqlalchemy.org/en/20/dialects/) and dynamic columns are supported provided the underlying engine has [JSON](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.JSON) support.

docs/examples.md

@@ -3,7 +3,7 @@
 ![examples](images/examples.png#only-light)
 ![examples](images/examples-dark.png#only-dark)
 
-The examples directory has a series of notebooks and applications giving an overview of txtai. See the sections below.
+See below for a comprehensive series of example notebooks and applications covering txtai.
 
 ## Semantic Search
 
@@ -16,21 +16,32 @@ Build semantic/similarity/vector/neural search applications.
 | [Build an Embeddings index from a data source](https://github.com/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb)  | Index and search a data source with word embeddings | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb) |
 | [Add semantic search to Elasticsearch](https://github.com/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb)  | Add semantic search to existing search systems | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/04_Add_semantic_search_to_Elasticsearch.ipynb) |
 | [Similarity search with images](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) | Embed images and text into the same space for search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |
-| [Distributed embeddings cluster](https://github.com/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) | Distribute an embeddings index across multiple data nodes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) |
-| [What's new in txtai 4.0](https://github.com/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) | Content storage, SQL, object storage, reindex and compressed indexes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) |
-| [Anatomy of a txtai index](https://github.com/neuml/txtai/blob/master/examples/29_Anatomy_of_a_txtai_index.ipynb) | Deep dive into the file formats behind a txtai embeddings index | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/29_Anatomy_of_a_txtai_index.ipynb) |
 | [Custom Embeddings SQL functions](https://github.com/neuml/txtai/blob/master/examples/30_Embeddings_SQL_custom_functions.ipynb) | Add user-defined functions to Embeddings SQL | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/30_Embeddings_SQL_custom_functions.ipynb) |
 | [Model explainability](https://github.com/neuml/txtai/blob/master/examples/32_Model_explainability.ipynb) | Explainability for semantic search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/32_Model_explainability.ipynb) |
 | [Query translation](https://github.com/neuml/txtai/blob/master/examples/33_Query_translation.ipynb) | Domain-specific natural language queries with query translation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/33_Query_translation.ipynb) |
 | [Build a QA database](https://github.com/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) | Question matching with semantic search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) |
-| [Embeddings components](https://github.com/neuml/txtai/blob/master/examples/37_Embeddings_index_components.ipynb) | Composable search with vector, SQL and scoring components | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/37_Embeddings_index_components.ipynb) |
 | [Semantic Graphs](https://github.com/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) | Explore topics, data connectivity and run network analysis| [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) |
 | [Topic Modeling with BM25](https://github.com/neuml/txtai/blob/master/examples/39_Classic_Topic_Modeling_with_BM25.ipynb) | Topic modeling backed by a BM25 index | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/39_Classic_Topic_Modeling_with_BM25.ipynb) |
+
+## LLM
+
+LLM chains, retrieval augmented generation (RAG), chat with your data, pipelines and workflows that interface with large language models (LLMs).
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
 | [Prompt-driven search with LLMs](https://github.com/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) | Embeddings-guided and Prompt-driven search with Large Language Models (LLMs) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) |
+| [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
+| [Generate knowledge with Semantic Graphs and RAG](https://github.com/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) | Knowledge exploration and discovery with Semantic Graphs and RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) |
+| [Build knowledge graphs with LLMs](https://github.com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) | Build knowledge graphs with LLM-driven entity extraction | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |
+| [Advanced RAG with graph path traversal](https://github.com/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) | Graph path traversal to collect complex sets of data for advanced RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) |
+| [Advanced RAG with guided generation](https://github.com/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) | Retrieval Augmented and Guided Generation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) |
+| [RAG with llama.cpp and external API services](https://github.com/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) | RAG with additional vector and LLM frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) |
 
 ## Pipelines
 
-Transform data with NLP-backed pipelines.
+Transform data with language model backed pipelines.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
@@ -45,7 +56,6 @@ Transform data with NLP-backed pipelines.
 | [Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) | Streamline machine translation and language detection | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |
 | [Generate image captions and detect objects](https://github.com/neuml/txtai/blob/master/examples/25_Generate_image_captions_and_detect_objects.ipynb) | Captions and object detection for images | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/25_Generate_image_captions_and_detect_objects.ipynb) |
 | [Near duplicate image detection](https://github.com/neuml/txtai/blob/master/examples/31_Near_duplicate_image_detection.ipynb) | Identify duplicate and near-duplicate images | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/31_Near_duplicate_image_detection.ipynb) |
-| [API Gallery](https://github.com/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) | Using txtai in JavaScript, Java, Rust and Go | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) |
 
 ## Workflows
 
@@ -53,7 +63,7 @@ Efficiently process data at scale.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
- [Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) [▶️](https://www.youtube.com/watch?v=UBMPDCn1gEU) | Simple yet powerful constructs to efficiently process data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
+| [Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) [▶️](https://www.youtube.com/watch?v=UBMPDCn1gEU) | Simple yet powerful constructs to efficiently process data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
 | [Transform tabular data with composable workflows](https://github.com/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) | Transform, index and search tabular data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/22_Transform_tabular_data_with_composable_workflows.ipynb) |
 | [Tensor workflows](https://github.com/neuml/txtai/blob/master/examples/23_Tensor_workflows.ipynb) | Performant processing of large tensor arrays | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/23_Tensor_workflows.ipynb) |
 | [Entity extraction workflows](https://github.com/neuml/txtai/blob/master/examples/26_Entity_extraction_workflows.ipynb) | Identify entity/label combinations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/26_Entity_extraction_workflows.ipynb) |
@@ -75,13 +85,53 @@ Train NLP models.
 | [Export and run models with ONNX](https://github.com/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) | Export models with ONNX, run natively in JavaScript, Java and Rust | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/18_Export_and_run_models_with_ONNX.ipynb) |
 | [Export and run other machine learning models](https://github.com/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) | Export and run models from scikit-learn, PyTorch and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/21_Export_and_run_other_machine_learning_models.ipynb) |
 
+## API
+
+Run distributed txtai, integrate with the API and cloud endpoints.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [API Gallery](https://github.com/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) | Using txtai in JavaScript, Java, Rust and Go | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/08_API_Gallery.ipynb) |
+| [Distributed embeddings cluster](https://github.com/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) | Distribute an embeddings index across multiple data nodes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/15_Distributed_embeddings_cluster.ipynb) |
+| [Embeddings in the Cloud](https://github.com/neuml/txtai/blob/master/examples/43_Embeddings_in_the_Cloud.ipynb) | Load and use an embeddings index from the Hugging Face Hub | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/43_Embeddings_in_the_Cloud.ipynb) |
+| [Custom API Endpoints](https://github.com/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) | Extend the API with custom endpoints | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/51_Custom_API_Endpoints.ipynb) |
+| [API Authorization and Authentication](https://github.com/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) | Add authorization, authentication and middleware dependencies to the API | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/54_API_Authorization_and_Authentication.ipynb) |
+
+## Architecture
+
+Project architecture, data formats, external integrations, scale to production, benchmarks, and performance.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Anatomy of a txtai index](https://github.com/neuml/txtai/blob/master/examples/29_Anatomy_of_a_txtai_index.ipynb) | Deep dive into the file formats behind a txtai embeddings index | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/29_Anatomy_of_a_txtai_index.ipynb) |
+| [Embeddings components](https://github.com/neuml/txtai/blob/master/examples/37_Embeddings_index_components.ipynb) | Composable search with vector, SQL and scoring components | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/37_Embeddings_index_components.ipynb) |
+| [Customize your own embeddings database](https://github.com/neuml/txtai/blob/master/examples/45_Customize_your_own_embeddings_database.ipynb) | Ways to combine vector indexes with relational databases | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/45_Customize_your_own_embeddings_database.ipynb) |
+| [Building an efficient sparse keyword index in Python](https://github.com/neuml/txtai/blob/master/examples/47_Building_an_efficient_sparse_keyword_index_in_Python.ipynb) | Fast and accurate sparse keyword indexing | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/48_Benefits_of_hybrid_search.ipynb) |
+| [Benefits of hybrid search](https://github.com/neuml/txtai/blob/master/examples/48_Benefits_of_hybrid_search.ipynb) | Improve accuracy with a combination of semantic and keyword search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/48_Benefits_of_hybrid_search.ipynb) |
+| [External database integration](https://github.com/neuml/txtai/blob/master/examples/49_External_database_integration.ipynb) | Store metadata in PostgreSQL, MariaDB, MySQL and more | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/49_External_database_integration.ipynb) |
+| [All about vector quantization](https://github.com/neuml/txtai/blob/master/examples/50_All_about_vector_quantization.ipynb) | Benchmarking scalar and product quantization methods | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/50_All_about_vector_quantization.ipynb) |
+| [External vectorization](https://github.com/neuml/txtai/blob/master/examples/56_External_vectorization.ipynb) | Vectorization with precomputed embeddings datasets and APIs | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/56_External_vectorization.ipynb) |
+| [Integrate txtai with Postgres](https://github.com/neuml/txtai/blob/master/examples/61_Integrate_txtai_with_Postgres.ipynb) | Persist content, vectors and graph data in Postgres | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/61_Integrate_txtai_with_Postgres.ipynb) |
+
+## Releases
+
+New functionality added in major releases.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [What's new in txtai 4.0](https://github.com/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) | Content storage, SQL, object storage, reindex and compressed indexes | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/24_Whats_new_in_txtai_4_0.ipynb) |
+| [What's new in txtai 6.0](https://github.com/neuml/txtai/blob/master/examples/46_Whats_new_in_txtai_6_0.ipynb) | Sparse, hybrid and subindexes for embeddings, LLM improvements | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/46_Whats_new_in_txtai_6_0.ipynb) |
+| [What's new in txtai 7.0](https://github.com/neuml/txtai/blob/master/examples/59_Whats_new_in_txtai_7_0.ipynb) | Semantic graph 2.0, LoRA/QLoRA training and binary API support | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/59_Whats_new_in_txtai_7_0.ipynb) |
+
 ## Applications
 
-Series of example applications with txtai. Links to hosted versions on [Hugging Face Spaces](https://hf.co/spaces) also provided.
+Series of example applications with txtai. Links to hosted versions on [Hugging Face Spaces](https://hf.co/spaces) are also provided, when available.
 
 | Application  | Description  |       |
 |:-------------|:-------------|------:|
 | [Basic similarity search](https://github.com/neuml/txtai/blob/master/examples/similarity.py) | Basic similarity search example. Data from the original txtai demo. |[🤗](https://hf.co/spaces/NeuML/similarity)|
+| [Baseball stats](https://github.com/neuml/txtai/blob/master/examples/baseball.py) | Match historical baseball player stats using vector search. |[🤗](https://hf.co/spaces/NeuML/baseball)|
+| [Benchmarks](https://github.com/neuml/txtai/blob/master/examples/benchmarks.py) | Calculate performance metrics for the BEIR datasets. |*Local run only*|
 | [Book search](https://github.com/neuml/txtai/blob/master/examples/books.py) | Book similarity search application. Index book descriptions and query using natural language statements. |*Local run only*|
 | [Image search](https://github.com/neuml/txtai/blob/master/examples/images.py) | Image similarity search application. Index a directory of images and run searches to identify images similar to the input query. |[🤗](https://hf.co/spaces/NeuML/imagesearch)|
 | [Summarize an article](https://github.com/neuml/txtai/blob/master/examples/article.py) | Summarize an article. Workflow that extracts text from a webpage and builds a summary. |[🤗](https://hf.co/spaces/NeuML/articlesummary)|

docs/faq.md

@@ -1,7 +1,84 @@
 # FAQ
 
+![faq](images/faq.png)
+
 Below is a list of frequently asked questions and common issues encountered.
 
+## Questions
+
+----------
+
+__Question__
+
+What models are recommended?
+
+__Answer__
+
+See the [model guide](../models).
+
+----------
+
+__Question__
+
+What is the best way to track the progress of an `embeddings.index` call?
+
+__Answer__
+
+Wrap the list or generator passed to the index call with tqdm. See [#478](https://github.com/neuml/txtai/issues/478) for more.
+
+----------
+
+__Question__
+
+What is the best way to analyze the content of a txtai index?
+
+__Answer__
+
+txtai has a console application that makes this easy. Read [this article](https://medium.com/neuml/insights-from-the-txtai-console-d307c28e149e) to learn more.
+
+----------
+
+__Question__
+
+How can models be externally loaded and passed to embeddings and pipelines?
+
+__Answer__
+
+Embeddings example.
+
+```python
+from transformers import AutoModel, AutoTokenizer
+from txtai.embeddings import Embeddings
+
+# Load model externally
+model = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+
+# Pass to embeddings instance
+embeddings = Embeddings(path=model, tokenizer=tokenizer)
+```
+
+LLM pipeline example.
+
+```python
+import torch
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from txtai.pipeline import LLM
+
+# Load Mistral-7B-OpenOrca
+path = "Open-Orca/Mistral-7B-OpenOrca"
+model = AutoModelForCausalLM.from_pretrained(
+  path,
+  torch_dtype=torch.bfloat16,
+)
+tokenizer = AutoTokenizer.from_pretrained(path)
+
+llm = LLM((model, tokenizer))
+```
+
+## Common issues
+
 ----------
 
 __Issue__
@@ -24,16 +101,28 @@ Segmentation faults and similar errors on macOS
 
 __Solution__
 
-Downgrade PyTorch to <= 1.12. See issue [#377](https://github.com/neuml/txtai/issues/377) for more on this issue. 
+Disable OpenMP threading via the environment variable `export OMP_NUM_THREADS=1` or downgrade PyTorch to <= 1.12. See issue [#377](https://github.com/neuml/txtai/issues/377) for more.
 
 ----------
 
 __Issue__
 
-`ContextualVersionConflict` exception when importing certain libraries while running one of the [examples](./examples.md) notebooks on Google Colab
+`ContextualVersionConflict` and/or package METADATA exception while running one of the [examples](../examples) notebooks on Google Colab
 
 __Solution__
 
 Restart the kernel. See issue [#409](https://github.com/neuml/txtai/issues/409) for more on this issue. 
 
-----------
+----------
+
+__Issue__
+
+Error installing optional/extra dependencies such as `pipeline`
+
+__Solution__
+
+The default MacOS shell (zsh) and Windows PowerShell require escaping square brackets
+
+```
+pip install 'txtai[pipeline]'
+```

docs/further.md

@@ -1,13 +1,10 @@
 # Further reading
 
-![further](images/further.png)
+![further](images/further.png#only-light)
+![further](images/further-dark.png#only-dark)
 
-- [Introducing txtai, AI-powered semantic search built on Transformers](https://medium.com/neuml/introducing-txtai-an-ai-powered-search-engine-built-on-transformers-37674be252ec)
+- [Introducing txtai, the all-in-one embeddings database](https://medium.com/neuml/introducing-txtai-the-all-in-one-embeddings-database-c721f4ff91ad)
 - [Tutorial series on Hashnode](https://neuml.hashnode.dev/series/txtai-tutorial) | [dev.to](https://dev.to/neuml/tutorial-series-on-txtai-ibg)
-- [What's new in txtai 5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
+- [What's new in txtai 7.0](https://medium.com/neuml/whats-new-in-txtai-7-0-855ad6a55440) | [6.0](https://medium.com/neuml/whats-new-in-txtai-6-0-7d93eeedf804) | [5.0](https://medium.com/neuml/whats-new-in-txtai-5-0-e5c75a13b101) | [4.0](https://medium.com/neuml/whats-new-in-txtai-4-0-bbc3a65c3d1c)
 - [Getting started with semantic search](https://medium.com/neuml/getting-started-with-semantic-search-a9fd9d8a48cf) | [workflows](https://medium.com/neuml/getting-started-with-semantic-workflows-2fefda6165d9)
-- [Run machine-learning workflows to transform data and build AI-powered semantic search applications with txtai](https://medium.com/neuml/run-machine-learning-workflows-to-transform-data-and-build-ai-powered-text-indices-with-txtai-43d769b566a7)
-- [Semantic search on the cheap](https://medium.com/neuml/semantic-search-on-the-cheap-55940c0fcdab)
-- [Serverless vector search with txtai](https://medium.com/neuml/serverless-vector-search-with-txtai-96f6163ab972)
-- [Insights from the txtai console](https://medium.com/neuml/insights-from-the-txtai-console-d307c28e149e)
-- [The big and small of txtai](https://medium.com/neuml/the-big-and-small-of-txtai-4ca405c1b82)
+- [Running txtai at scale](https://medium.com/neuml/running-at-scale-with-txtai-71196cdd99f9)

docs/images/architecture.excalidraw

@@ -0,0 +1,996 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 4858,
+      "versionNonce": 83494474,
+      "isDeleted": false,
+      "id": "1iE4X4trkcbtJpCDC5ajY",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 27.525089187647836,
+      "y": -558.1200877100962,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 227.65938859429855,
+      "height": 57.830603028426594,
+      "seed": 1321467630,
+      "groupIds": [
+        "MnVd9k06uFuFRzqXmnEvR",
+        "NFKrTdLl_zy4zOvDGWVBD"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "id": "qROQdhT3FWGzgS0JOQXlg",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 5224,
+      "versionNonce": 1682430806,
+      "isDeleted": false,
+      "id": "ZiUda1q4lvcNM9G2Or_d7",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 39.103253497881326,
+      "y": -548.6024245205953,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#343a40",
+      "width": 210.71665954589844,
+      "height": 40,
+      "seed": 2050033394,
+      "groupIds": [
+        "NFKrTdLl_zy4zOvDGWVBD"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "qROQdhT3FWGzgS0JOQXlg",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": "- Semantic, Keyword, Hybrid\n- Search with SQL",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Semantic, Keyword, Hybrid\n- Search with SQL",
+      "lineHeight": 1.25,
+      "baseline": 35
+    },
+    {
+      "type": "rectangle",
+      "version": 2527,
+      "versionNonce": 2136048394,
+      "isDeleted": false,
+      "id": "i9TRMGNTF_ckjPiI7BrdL",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 358.6031681551546,
+      "y": -598.4332496538902,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 348.08489527005264,
+      "height": 200.0037909410919,
+      "seed": 1867912238,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "id": "viSfAjT-8KRTYvsrunjfn",
+          "type": "arrow"
+        },
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        },
+        {
+          "id": "sf5i6bxPYE2ykeh2wWm8m",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411996548,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "rectangle",
+      "version": 1915,
+      "versionNonce": 211681430,
+      "isDeleted": false,
+      "id": "F1r0Plm7hDaeNndIwZAuf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 368.61999967371725,
+      "y": -538.0801566026939,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 160.26930429700263,
+      "height": 62.605196991016655,
+      "seed": 436869746,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "gw7zjxlDiAjodT1x1_fv_"
+        },
+        {
+          "id": "qROQdhT3FWGzgS0JOQXlg",
+          "type": "arrow"
+        },
+        {
+          "id": "viSfAjT-8KRTYvsrunjfn",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1897,
+      "versionNonce": 347110346,
+      "isDeleted": false,
+      "id": "gw7zjxlDiAjodT1x1_fv_",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 406.8879869357439,
+      "y": -522.695916613224,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "transparent",
+      "width": 83.73332977294922,
+      "height": 31.836717012076832,
+      "seed": 1023300530,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false,
+      "fontSize": 25.469373609661467,
+      "fontFamily": 1,
+      "text": "Sparse",
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "F1r0Plm7hDaeNndIwZAuf",
+      "originalText": "Sparse",
+      "lineHeight": 1.25,
+      "baseline": 22
+    },
+    {
+      "type": "rectangle",
+      "version": 2272,
+      "versionNonce": 696090070,
+      "isDeleted": false,
+      "id": "55xWndBPWTiyUuLIS_J8b",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 369.8721036135376,
+      "y": -469.46654385239594,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 157.76509641736197,
+      "height": 57.59678123173532,
+      "seed": 1317384558,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "Qaeh-leNa7YgcPsQbfBcq"
+        },
+        {
+          "id": "qROQdhT3FWGzgS0JOQXlg",
+          "type": "arrow"
+        },
+        {
+          "id": "viSfAjT-8KRTYvsrunjfn",
+          "type": "arrow"
+        },
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 2210,
+      "versionNonce": 1434612362,
+      "isDeleted": false,
+      "id": "Qaeh-leNa7YgcPsQbfBcq",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 412.012986935744,
+      "y": -456.59111621253714,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "transparent",
+      "width": 73.48332977294922,
+      "height": 31.84592595201767,
+      "seed": 1661259762,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false,
+      "fontSize": 25.476740761614135,
+      "fontFamily": 1,
+      "text": "Dense",
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "55xWndBPWTiyUuLIS_J8b",
+      "originalText": "Dense",
+      "lineHeight": 1.25,
+      "baseline": 22
+    },
+    {
+      "type": "rectangle",
+      "version": 2356,
+      "versionNonce": 268129046,
+      "isDeleted": false,
+      "id": "KSf88Asx9gne1WsRdsx6-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 535.1498236698217,
+      "y": -469.46654385239594,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 156.51299247754164,
+      "height": 57.59678123173532,
+      "seed": 1016194418,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "7RaoYDeBRuz16CJo7jaVO"
+        },
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 2343,
+      "versionNonce": 2120716618,
+      "isDeleted": false,
+      "id": "7RaoYDeBRuz16CJo7jaVO",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 576.8063214344714,
+      "y": -456.5934785058263,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "transparent",
+      "width": 73.19999694824219,
+      "height": 31.850650538596007,
+      "seed": 1307924782,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991785,
+      "link": null,
+      "locked": false,
+      "fontSize": 25.480520430876805,
+      "fontFamily": 1,
+      "text": "Graph",
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "KSf88Asx9gne1WsRdsx6-",
+      "originalText": "Graph",
+      "lineHeight": 1.25,
+      "baseline": 22
+    },
+    {
+      "type": "text",
+      "version": 1738,
+      "versionNonce": 713850966,
+      "isDeleted": false,
+      "id": "gAcYCRjD6VZUsDZce-atm",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 439.98992424347625,
+      "y": -595.9290417742495,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#ffeb3b",
+      "width": 183.39999389648438,
+      "height": 43.823637893711656,
+      "seed": 1369566386,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991786,
+      "link": null,
+      "locked": false,
+      "fontSize": 35.058910314969324,
+      "fontFamily": 1,
+      "text": "Embeddings",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "Embeddings",
+      "lineHeight": 1.25,
+      "baseline": 31
+    },
+    {
+      "type": "rectangle",
+      "version": 2094,
+      "versionNonce": 704590410,
+      "isDeleted": false,
+      "id": "UEE-Z2YT9Y5YvVCJefbd0",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 533.8977197300013,
+      "y": -535.8280526628736,
+      "strokeColor": "#7950f2",
+      "backgroundColor": "#7950f2",
+      "width": 160.26930429700263,
+      "height": 60.10098911137599,
+      "seed": 1015954930,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "type": "text",
+          "id": "Y1uf_NH7fI6arCSGiICiX"
+        },
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691412007529,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 2084,
+      "versionNonce": 1292240278,
+      "isDeleted": false,
+      "id": "Y1uf_NH7fI6arCSGiICiX",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 551.0823711155632,
+      "y": -521.695916613224,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "transparent",
+      "width": 125.9000015258789,
+      "height": 31.836717012076832,
+      "seed": 867545010,
+      "groupIds": [
+        "cNL6-_r-I9wctWvuw8DqS"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991786,
+      "link": null,
+      "locked": false,
+      "fontSize": 25.469373609661467,
+      "fontFamily": 1,
+      "text": "Database",
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": "UEE-Z2YT9Y5YvVCJefbd0",
+      "originalText": "Database",
+      "lineHeight": 1.25,
+      "baseline": 22
+    },
+    {
+      "type": "rectangle",
+      "version": 4488,
+      "versionNonce": 1920419530,
+      "isDeleted": false,
+      "id": "uV8iktEBp-zvkGceO0YNj",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 28.400291679657585,
+      "y": -482.37846781152234,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 226.65938859429855,
+      "height": 53.830603028426594,
+      "seed": 511161586,
+      "groupIds": [
+        "lsnRF8t7e-8bA4k7ZMtQs",
+        "XOEGmXNmM0vVhFi_aLf0h"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "id": "viSfAjT-8KRTYvsrunjfn",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411991786,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 4865,
+      "versionNonce": 144499414,
+      "isDeleted": false,
+      "id": "VU0ZFGV4tB5pR_8AMPehi",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 38.17389738827433,
+      "y": -477.32443222094344,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#343a40",
+      "width": 190.39999389648438,
+      "height": 40,
+      "seed": 660096690,
+      "groupIds": [
+        "XOEGmXNmM0vVhFi_aLf0h"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411991786,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": "- Topics + Relationships\n- Multimodal Indexes",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "- Topics + Relationships\n- Multimodal Indexes",
+      "lineHeight": 1.25,
+      "baseline": 35
+    },
+    {
+      "type": "arrow",
+      "version": 3140,
+      "versionNonce": 1465895050,
+      "isDeleted": false,
+      "id": "qROQdhT3FWGzgS0JOQXlg",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 357.5459819116054,
+      "y": -474.380420640035,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 101.91733646013165,
+      "height": 51.182713481535075,
+      "seed": 1412373166,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691412001500,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "55xWndBPWTiyUuLIS_J8b",
+        "focus": -0.17593583244727756,
+        "gap": 13.269493633705139
+      },
+      "endBinding": {
+        "elementId": "ZiUda1q4lvcNM9G2Or_d7",
+        "focus": -0.7240167179756973,
+        "gap": 5.808732407693981
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -101.91733646013165,
+          -51.182713481535075
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 2221,
+      "versionNonce": 297446922,
+      "isDeleted": false,
+      "id": "viSfAjT-8KRTYvsrunjfn",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 355.69072075221885,
+      "y": -473.6934438187176,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 99.11528692741001,
+      "height": 28.941684185339795,
+      "seed": 769791342,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691412005709,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "55xWndBPWTiyUuLIS_J8b",
+        "focus": 1.1614221413579826,
+        "gap": 14.797915501333051
+      },
+      "endBinding": {
+        "elementId": "uV8iktEBp-zvkGceO0YNj",
+        "focus": 0.737345008582056,
+        "gap": 1.5157535508527076
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -99.11528692741001,
+          28.941684185339795
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 5939,
+      "versionNonce": 1648005846,
+      "isDeleted": false,
+      "id": "CJaOip-NXs_WGJ3YnzrzX",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.2644916835155,
+      "y": -444.3627069906215,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 71.1891690063859,
+      "height": 28.06038115974593,
+      "seed": 1923467222,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411901454,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "SKqPLoIsGXgDL5mhOTqLv",
+        "focus": -0.634641820715333,
+        "gap": 5.063962132776453
+      },
+      "endBinding": {
+        "elementId": "UEE-Z2YT9Y5YvVCJefbd0",
+        "focus": -0.06665280707924877,
+        "gap": 15.270023644140622
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": "arrow",
+      "endArrowhead": null,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -71.1891690063859,
+          -28.06038115974593
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 3429,
+      "versionNonce": 348832458,
+      "isDeleted": false,
+      "id": "7UwPwHCm_ZiMJb1o0x5A8",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 783.2131987136702,
+      "y": -574.1983152922421,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 272.0848952700526,
+      "height": 73.0037909410919,
+      "seed": 155582550,
+      "groupIds": [
+        "kgkwY28q5i7W008rJksMR",
+        "TqYTD07S3yHeAVJOG3yyi"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1591,
+      "versionNonce": 72296150,
+      "isDeleted": false,
+      "id": "fFWZLl9F8IwK5QhXlAssA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 792.6342330724652,
+      "y": -569.5895346298003,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#343a40",
+      "width": 77.1500015258789,
+      "height": 20,
+      "seed": 543556438,
+      "groupIds": [
+        "TqYTD07S3yHeAVJOG3yyi"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": " Prompt >",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": " Prompt >",
+      "lineHeight": 1.25,
+      "baseline": 15
+    },
+    {
+      "type": "text",
+      "version": 1341,
+      "versionNonce": 450634262,
+      "isDeleted": false,
+      "id": "FgFcgIY3s9iRdPKwN4t_r",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 785.1996772966362,
+      "y": -548.1590294764883,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#343a40",
+      "width": 261.5,
+      "height": 41.41205162166277,
+      "seed": 423650058,
+      "groupIds": [
+        "TqYTD07S3yHeAVJOG3yyi"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691412007529,
+      "link": null,
+      "locked": false,
+      "fontSize": 16.56482064866511,
+      "fontFamily": 1,
+      "text": "   Answer the following question\n   using the context below",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "   Answer the following question\n   using the context below",
+      "lineHeight": 1.25,
+      "baseline": 35
+    },
+    {
+      "type": "rectangle",
+      "version": 3467,
+      "versionNonce": 1420811286,
+      "isDeleted": false,
+      "id": "_8pPaWBfdkvpd8B9PQ7wI",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 784.2131987136702,
+      "y": -487.1983152922421,
+      "strokeColor": "#6741d9",
+      "backgroundColor": "#6741d9",
+      "width": 272.0848952700526,
+      "height": 73.0037909410919,
+      "seed": 2066566410,
+      "groupIds": [
+        "FtTm_MWbxe67f2n84saes",
+        "3PAnqhix8z46z6JRJjAO7"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 3
+      },
+      "boundElements": [
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1602,
+      "versionNonce": 1943803978,
+      "isDeleted": false,
+      "id": "apAxY8HSw8w2lfLe1pCXP",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 792.7130504007473,
+      "y": -482.33339890257247,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#6741d9",
+      "width": 75.73332977294922,
+      "height": 20,
+      "seed": 1870973578,
+      "groupIds": [
+        "3PAnqhix8z46z6JRJjAO7"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 1,
+      "text": " Search >",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": " Search >",
+      "lineHeight": 1.25,
+      "baseline": 15
+    },
+    {
+      "type": "text",
+      "version": 1069,
+      "versionNonce": 1506273622,
+      "isDeleted": false,
+      "id": "SKqPLoIsGXgDL5mhOTqLv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 785.3284538162922,
+      "y": -458.46265045437633,
+      "strokeColor": "#1e1e1e",
+      "backgroundColor": "#6741d9",
+      "width": 251.21665954589844,
+      "height": 41.82578128069407,
+      "seed": 1533237462,
+      "groupIds": [
+        "3PAnqhix8z46z6JRJjAO7"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "CJaOip-NXs_WGJ3YnzrzX",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1691411901455,
+      "link": null,
+      "locked": false,
+      "fontSize": 16.730312512277628,
+      "fontFamily": 1,
+      "text": "   SELECT ... FROM txtai\n   WHERE SIMILAR('question')",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "   SELECT ... FROM txtai\n   WHERE SIMILAR('question')",
+      "lineHeight": 1.25,
+      "baseline": 35
+    },
+    {
+      "type": "arrow",
+      "version": 5999,
+      "versionNonce": 1002058634,
+      "isDeleted": false,
+      "id": "sf5i6bxPYE2ykeh2wWm8m",
+      "fillStyle": "hachure",
+      "strokeWidth": 2,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.8522503409429,
+      "y": -537.8502934180265,
+      "strokeColor": "#868e96",
+      "backgroundColor": "#e9ecef",
+      "width": 72.32385186561339,
+      "height": 64.0647548382936,
+      "seed": 1598499018,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1691412007529,
+      "link": null,
+      "locked": false,
+      "startBinding": null,
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": "arrow",
+      "endArrowhead": null,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -72.32385186561339,
+          64.0647548382936
+        ]
+      ]
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

docs/images/cloud.excalidraw

@@ -5,8 +5,8 @@
   "elements": [
     {
       "type": "rectangle",
-      "version": 3020,
-      "versionNonce": 574956379,
+      "version": 3062,
+      "versionNonce": 456539957,
       "isDeleted": false,
       "id": "Ufh5VUA3qmvJowuFyEWz4",
       "fillStyle": "hachure",
@@ -21,187 +21,119 @@
       "backgroundColor": "transparent",
       "width": 719.330217768941,
       "height": 100.49478848195537,
-      "seed": 1218477931,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
-      "boundElements": [
-        {
-          "type": "arrow",
-          "id": "QPHaiLoteZgu8lrk22yKh"
-        }
-      ],
-      "updated": 1644866249723,
-      "link": null
+      "seed": 2109972213,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688312603126,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1748,
-      "versionNonce": 1788187605,
+      "version": 1798,
+      "versionNonce": 1560857173,
       "isDeleted": false,
       "id": "NDWbJgM53uVck-LSQzD-0",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 740.6664767795138,
-      "y": 67.41041395399299,
+      "y": 67.92614920483035,
       "strokeColor": "#ffeb3b",
       "backgroundColor": "#ffeb3b",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 1934943141,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1782446165,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "type": "text",
           "id": "0VztU7lb7T9I5zoL2LjLt"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
-      "id": "0VztU7lb7T9I5zoL2LjLt",
       "type": "text",
-      "x": 745.6664767795138,
-      "y": 98.99999999999994,
-      "width": 75,
-      "height": 21,
-      "angle": 0,
-      "strokeColor": "#000000",
-      "backgroundColor": "#7950f2",
+      "version": 385,
+      "versionNonce": 1280754907,
+      "isDeleted": false,
+      "id": "0VztU7lb7T9I5zoL2LjLt",
       "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
-      "seed": 604922325,
-      "version": 333,
-      "versionNonce": 1646133243,
-      "isDeleted": false,
-      "boundElements": null,
-      "updated": 1644866249723,
+      "angle": 0,
+      "x": 745.6664767795138,
+      "y": 99.51573525083731,
+      "strokeColor": "#000000",
+      "backgroundColor": "#7950f2",
+      "width": 75,
+      "height": 21,
+      "seed": 2095457717,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688312605190,
       "link": null,
-      "text": "Container",
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
+      "text": "Container",
       "textAlign": "center",
       "verticalAlign": "middle",
-      "baseline": 15,
       "containerId": "NDWbJgM53uVck-LSQzD-0",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     },
     {
       "type": "rectangle",
-      "version": 1824,
-      "versionNonce": 1455313205,
+      "version": 1874,
+      "versionNonce": 847689653,
       "isDeleted": false,
       "id": "rr3YJsAyrrPqZbZ4qXh5x",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 1175.874140249897,
-      "y": 70.42614920483032,
+      "y": 67.92614920483035,
       "strokeColor": "#00e676",
       "backgroundColor": "#00e676",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 1158748789,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1491183381,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "id": "9NBdG6TyrBqjuDKvzAZXN",
           "type": "text"
-        },
-        {
-          "id": "9NBdG6TyrBqjuDKvzAZXN",
-          "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "9NBdG6TyrBqjuDKvzAZXN"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 404,
-      "versionNonce": 938625179,
+      "version": 456,
+      "versionNonce": 1820291451,
       "isDeleted": false,
       "id": "9NBdG6TyrBqjuDKvzAZXN",
       "fillStyle": "hachure",
@@ -211,100 +143,65 @@
       "opacity": 100,
       "angle": 0,
       "x": 1180.874140249897,
-      "y": 102.01573525083728,
+      "y": 99.51573525083731,
       "strokeColor": "#000000",
       "backgroundColor": "#7950f2",
       "width": 75,
       "height": 21,
-      "seed": 603698011,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 547748981,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1644866249723,
+      "updated": 1688312605190,
       "link": null,
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
       "text": "Container",
-      "baseline": 15,
       "textAlign": "center",
       "verticalAlign": "middle",
       "containerId": "rr3YJsAyrrPqZbZ4qXh5x",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     },
     {
       "type": "rectangle",
-      "version": 1814,
-      "versionNonce": 766010005,
+      "version": 1864,
+      "versionNonce": 1889532181,
       "isDeleted": false,
       "id": "ji9UBE590YJ5e8DKUKbPG",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 598.874140249897,
-      "y": 68.42614920483032,
+      "y": 67.92614920483035,
       "strokeColor": "#ff7043",
       "backgroundColor": "#ff7043",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 895199477,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 200844757,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "id": "2fdy7dRwF7ZVaYPrb1Q7p",
           "type": "text"
-        },
-        {
-          "id": "2fdy7dRwF7ZVaYPrb1Q7p",
-          "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "2fdy7dRwF7ZVaYPrb1Q7p"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 394,
-      "versionNonce": 695500091,
+      "version": 446,
+      "versionNonce": 673810971,
       "isDeleted": false,
       "id": "2fdy7dRwF7ZVaYPrb1Q7p",
       "fillStyle": "hachure",
@@ -314,96 +211,65 @@
       "opacity": 100,
       "angle": 0,
       "x": 603.874140249897,
-      "y": 100.01573525083728,
+      "y": 99.51573525083731,
       "strokeColor": "#000000",
       "backgroundColor": "#7950f2",
       "width": 75,
       "height": 21,
-      "seed": 507989211,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1039082293,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1644866249723,
+      "updated": 1688312605190,
       "link": null,
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
       "text": "Container",
-      "baseline": 15,
       "textAlign": "center",
       "verticalAlign": "middle",
       "containerId": "ji9UBE590YJ5e8DKUKbPG",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     },
     {
       "type": "rectangle",
-      "version": 1754,
-      "versionNonce": 873023477,
+      "version": 1805,
+      "versionNonce": 281677429,
       "isDeleted": false,
       "id": "Kod3erHdgkFtJcsgqHeQQ",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 889.874140249897,
-      "y": 68.42614920483032,
+      "y": 67.92614920483035,
       "strokeColor": "#03a9f4",
       "backgroundColor": "#03a9f4",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 1941388379,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1525609621,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "id": "vA5QfvPySVZE9WnP6d7dp",
           "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "vA5QfvPySVZE9WnP6d7dp"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 337,
-      "versionNonce": 1644633563,
+      "version": 390,
+      "versionNonce": 1682411195,
       "isDeleted": false,
       "id": "vA5QfvPySVZE9WnP6d7dp",
       "fillStyle": "hachure",
@@ -413,104 +279,65 @@
       "opacity": 100,
       "angle": 0,
       "x": 894.874140249897,
-      "y": 100.01573525083728,
+      "y": 99.51573525083731,
       "strokeColor": "#000000",
       "backgroundColor": "#7950f2",
       "width": 75,
       "height": 21,
-      "seed": 867011285,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 1588435445,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1644866249723,
+      "updated": 1688312605190,
       "link": null,
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
       "text": "Container",
-      "baseline": 15,
       "textAlign": "center",
       "verticalAlign": "middle",
       "containerId": "Kod3erHdgkFtJcsgqHeQQ",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     },
     {
       "type": "rectangle",
-      "version": 1863,
-      "versionNonce": 2078703957,
+      "version": 1916,
+      "versionNonce": 1103787989,
       "isDeleted": false,
       "id": "8p_AIsxC6UbaWESkrwl4g",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
       "x": 1035.874140249897,
-      "y": 69.42614920483032,
-      "strokeColor": "#5f3dc4",
+      "y": 67.92614920483035,
+      "strokeColor": "#7950f2",
       "backgroundColor": "#7950f2",
       "width": 85,
       "height": 84.17917209201391,
-      "seed": 1712249051,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 842044245,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [
-        {
-          "type": "arrow",
-          "id": "xz4WiNbnK-neZmBN7EIi8"
-        },
-        {
-          "type": "arrow",
-          "id": "OH6z46_TWktLzJb31zpRF"
-        },
-        {
-          "type": "arrow",
-          "id": "b07pmUMsHbkm7ES3OaM12"
-        },
-        {
-          "id": "HXM4G7z36lQFPmJTUAy8G",
-          "type": "arrow"
-        },
-        {
-          "id": "BmtKHcjATX2AOVt484fin",
-          "type": "arrow"
-        },
-        {
-          "id": "nR4v3IHv93pXRqc8UUXjz",
-          "type": "arrow"
-        },
-        {
-          "id": "095ylTvsupuqCzyoolf44",
-          "type": "arrow"
-        },
         {
           "id": "AceuLTPujneYjRp_njhIl",
           "type": "text"
-        },
-        {
-          "id": "AceuLTPujneYjRp_njhIl",
-          "type": "text"
-        },
-        {
-          "id": "AceuLTPujneYjRp_njhIl",
-          "type": "text"
-        },
-        {
-          "type": "text",
-          "id": "AceuLTPujneYjRp_njhIl"
         }
       ],
-      "updated": 1644866249723,
-      "link": null
+      "updated": 1688312605190,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 443,
-      "versionNonce": 2092739195,
+      "version": 499,
+      "versionNonce": 1132199771,
       "isDeleted": false,
       "id": "AceuLTPujneYjRp_njhIl",
       "fillStyle": "hachure",
@@ -519,28 +346,29 @@
       "roughness": 1,
       "opacity": 100,
       "angle": 0,
-      "x": 1040.874140249897,
-      "y": 101.01573525083728,
+      "x": 1041.907473074604,
+      "y": 99.51573525083731,
       "strokeColor": "#000000",
       "backgroundColor": "#7950f2",
-      "width": 75,
+      "width": 72.93333435058594,
       "height": 21,
-      "seed": 1770780245,
-      "groupIds": [
-        "xlXj5_q2htAY6GkFTIlfY"
-      ],
-      "strokeSharpness": "sharp",
+      "seed": 30236853,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1644866249723,
+      "updated": 1688312605190,
       "link": null,
+      "locked": false,
       "fontSize": 16,
       "fontFamily": 1,
       "text": "Container",
-      "baseline": 15,
       "textAlign": "center",
       "verticalAlign": "middle",
       "containerId": "8p_AIsxC6UbaWESkrwl4g",
-      "originalText": "Container"
+      "originalText": "Container",
+      "lineHeight": 1.3125,
+      "baseline": 15
     }
   ],
   "appState": {

docs/images/examples.excalidraw

@@ -5,8 +5,8 @@
   "elements": [
     {
       "type": "rectangle",
-      "version": 623,
-      "versionNonce": 1614613512,
+      "version": 658,
+      "versionNonce": 56883131,
       "isDeleted": false,
       "id": "28knmXv7UWxhQgfO1Ap_e",
       "fillStyle": "hachure",
@@ -23,14 +23,17 @@
       "height": 19.163298353951955,
       "seed": 333697800,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176184,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 724,
-      "versionNonce": 384800888,
+      "version": 759,
+      "versionNonce": 1321514357,
       "isDeleted": false,
       "id": "NWB9SsDdfRnKFM7WSMiDD",
       "fillStyle": "hachure",
@@ -47,14 +50,17 @@
       "height": 8.353548393013705,
       "seed": 1799143800,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176184,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 838,
-      "versionNonce": 1137593096,
+      "version": 873,
+      "versionNonce": 1965246555,
       "isDeleted": false,
       "id": "o8mRXU1Pv0Lht3Kvw3LjM",
       "fillStyle": "hachure",
@@ -71,14 +77,17 @@
       "height": 10.760364257767161,
       "seed": 84161032,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176184,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 817,
-      "versionNonce": 322230024,
+      "version": 852,
+      "versionNonce": 1473789653,
       "isDeleted": false,
       "id": "7XXBg9rngZPBd8BY3k-Dk",
       "fillStyle": "hachure",
@@ -95,14 +104,17 @@
       "height": 326.7582126998559,
       "seed": 1645597304,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641752945859
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 785,
-      "versionNonce": 1088399880,
+      "version": 820,
+      "versionNonce": 1609532667,
       "isDeleted": false,
       "id": "1djoflJbPq1iZYxeG6vvs",
       "fillStyle": "hachure",
@@ -119,14 +131,17 @@
       "height": 8.353548393013705,
       "seed": 1354805512,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1012,
-      "versionNonce": 2110642808,
+      "version": 1047,
+      "versionNonce": 2087951413,
       "isDeleted": false,
       "id": "61SvR173kGBlNiM2hYx-i",
       "fillStyle": "hachure",
@@ -143,14 +158,19 @@
       "height": 35.19095464139164,
       "seed": 497337208,
       "groupIds": [],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 1
+      },
       "boundElements": [],
-      "updated": 1641745822132
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 1144,
-      "versionNonce": 1288082440,
+      "version": 1179,
+      "versionNonce": 1800088987,
       "isDeleted": false,
       "id": "KweZDYL4pV266i29UQo2O",
       "fillStyle": "hachure",
@@ -167,22 +187,26 @@
       "height": 17,
       "seed": 244666376,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745960277,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.613804582094819,
       "fontFamily": 1,
       "text": "Show more",
-      "baseline": 12,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Show more"
+      "originalText": "Show more",
+      "lineHeight": 1.3477297741025214,
+      "baseline": 12
     },
     {
       "type": "line",
-      "version": 1020,
-      "versionNonce": 354143096,
+      "version": 1055,
+      "versionNonce": 612126101,
       "isDeleted": false,
       "id": "6UHr1F9sbQjfc3Pn3AUC9",
       "fillStyle": "hachure",
@@ -199,9 +223,14 @@
       "height": 7.91335492900009,
       "seed": 1278055544,
       "groupIds": [],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
       "boundElements": [],
-      "updated": 1641745822132,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "startBinding": null,
       "endBinding": null,
       "lastCommittedPoint": null,
@@ -224,11 +253,11 @@
     },
     {
       "type": "rectangle",
-      "version": 1546,
-      "versionNonce": 871548936,
+      "version": 1593,
+      "versionNonce": 711508539,
       "isDeleted": false,
       "id": "pQ7Helj-RkhLonwDn4uxq",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -244,17 +273,20 @@
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822133
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "line",
-      "version": 758,
-      "versionNonce": 1334025336,
+      "version": 806,
+      "versionNonce": 522703605,
       "isDeleted": false,
       "id": "bI-A-gEMD-yvyhVvuajAH",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -270,9 +302,14 @@
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
       "boundElements": [],
-      "updated": 1641745822133,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "startBinding": null,
       "endBinding": null,
       "lastCommittedPoint": null,
@@ -291,11 +328,11 @@
     },
     {
       "type": "line",
-      "version": 768,
-      "versionNonce": 292325128,
+      "version": 816,
+      "versionNonce": 640336603,
       "isDeleted": false,
       "id": "X3L53b-aDRBM9J_i7xfIb",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -311,9 +348,14 @@
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
       "boundElements": [],
-      "updated": 1641745822133,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "startBinding": null,
       "endBinding": null,
       "lastCommittedPoint": null,
@@ -332,11 +374,11 @@
     },
     {
       "type": "rectangle",
-      "version": 1720,
-      "versionNonce": 1858686328,
+      "version": 1768,
+      "versionNonce": 956017749,
       "isDeleted": false,
       "id": "Fk7rL2dsob9lBYBWtqIaE",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -345,24 +387,27 @@
       "x": 909.5034365835013,
       "y": 461.80381850437016,
       "strokeColor": "#000000",
-      "backgroundColor": "#03a9f4",
+      "backgroundColor": "#ffeb3b",
       "width": 12.48277017202394,
       "height": 62.41385086011974,
       "seed": 1096491528,
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822134
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1590,
-      "versionNonce": 928409096,
+      "version": 1638,
+      "versionNonce": 656650107,
       "isDeleted": false,
       "id": "V3UV1tCb1upEwCUx2NF_U",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -371,24 +416,27 @@
       "x": 940.7103620135604,
       "y": 480.5279737624063,
       "strokeColor": "#000000",
-      "backgroundColor": "#ffeb3b",
+      "backgroundColor": "#03a9f4",
       "width": 12.482770172023953,
       "height": 43.68969560208382,
       "seed": 1860559480,
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822134
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1597,
-      "versionNonce": 1546706552,
+      "version": 1650,
+      "versionNonce": 793292539,
       "isDeleted": false,
       "id": "H9TFct1zEuli3oFZ0J6Pi",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -397,24 +445,27 @@
       "x": 971.9172874436208,
       "y": 474.28658867639376,
       "strokeColor": "#000000",
-      "backgroundColor": "#00e676",
+      "backgroundColor": "#7950f2",
       "width": 12.482770172023953,
       "height": 49.9310806880958,
       "seed": 1153016072,
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822134
+      "updated": 1688314179109,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1754,
-      "versionNonce": 1655465224,
+      "version": 1806,
+      "versionNonce": 1066797083,
       "isDeleted": false,
       "id": "Yvl5ZFMXGqZoKBjooHuuL",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -423,21 +474,24 @@
       "x": 1003.1242128736812,
       "y": 468.04520359038156,
       "strokeColor": "#000000",
-      "backgroundColor": "#ff7043",
+      "backgroundColor": "#00e676",
       "width": 12.48277017202394,
       "height": 56.172465774107785,
       "seed": 905106296,
       "groupIds": [
         "0ClD8yewsWOTVzkWq2mkc"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641745822134
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false
     },
     {
       "type": "text",
-      "version": 785,
-      "versionNonce": 725387784,
+      "version": 820,
+      "versionNonce": 851598101,
       "isDeleted": false,
       "id": "k2RjGkR4842FWeQqwZNqP",
       "fillStyle": "hachure",
@@ -454,22 +508,26 @@
       "height": 46,
       "seed": 351902216,
       "groupIds": [],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746472537,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 36,
       "fontFamily": 1,
       "text": "Results",
-      "baseline": 32,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Results"
+      "originalText": "Results",
+      "lineHeight": 1.2777777777777777,
+      "baseline": 32
     },
     {
       "type": "line",
-      "version": 352,
-      "versionNonce": 1781891704,
+      "version": 387,
+      "versionNonce": 359943355,
       "isDeleted": false,
       "id": "9BG_CdPIU12PIVg57JFjb",
       "fillStyle": "hachure",
@@ -488,9 +546,14 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "round",
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
       "boundElements": [],
-      "updated": 1641746497832,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "startBinding": null,
       "endBinding": null,
       "lastCommittedPoint": null,
@@ -509,8 +572,8 @@
     },
     {
       "type": "text",
-      "version": 924,
-      "versionNonce": 382863880,
+      "version": 959,
+      "versionNonce": 1682976885,
       "isDeleted": false,
       "id": "ooEpA700FaTXoHICTApB5",
       "fillStyle": "hachure",
@@ -529,22 +592,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972927,
       "fontFamily": 1,
       "text": "0",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "0"
+      "originalText": "0",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 996,
-      "versionNonce": 2125560440,
+      "version": 1031,
+      "versionNonce": 1818295643,
       "isDeleted": false,
       "id": "n9FAYaIZeMXggjqKvdYxs",
       "fillStyle": "hachure",
@@ -563,22 +630,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972932,
       "fontFamily": 1,
       "text": "1",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "1"
+      "originalText": "1",
+      "lineHeight": 1.3556693953949972,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1072,
-      "versionNonce": 454265096,
+      "version": 1107,
+      "versionNonce": 1144805845,
       "isDeleted": false,
       "id": "fQfYp0ljtX1ESRr5wPM9b",
       "fillStyle": "hachure",
@@ -597,22 +668,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "2",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "2"
+      "originalText": "2",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 980,
-      "versionNonce": 2022402936,
+      "version": 1015,
+      "versionNonce": 1873713659,
       "isDeleted": false,
       "id": "6-kghWfO7GdeQUDWs_zBe",
       "fillStyle": "hachure",
@@ -631,22 +706,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972927,
       "fontFamily": 1,
       "text": "ID",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "ID"
+      "originalText": "ID",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 989,
-      "versionNonce": 952969224,
+      "version": 1024,
+      "versionNonce": 463230773,
       "isDeleted": false,
       "id": "UMowu2HWCm_mgorPKfHhX",
       "fillStyle": "hachure",
@@ -665,22 +744,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "_____",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "_____"
+      "originalText": "_____",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1058,
-      "versionNonce": 1677671544,
+      "version": 1093,
+      "versionNonce": 1012900507,
       "isDeleted": false,
       "id": "1_YJ1fjNPFhQdtKIp-DSI",
       "fillStyle": "hachure",
@@ -699,22 +782,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176185,
+      "link": null,
+      "locked": false,
       "fontSize": 12.90383362297293,
       "fontFamily": 1,
       "text": "_____",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "_____"
+      "originalText": "_____",
+      "lineHeight": 1.3556693953949974,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1133,
-      "versionNonce": 1274734344,
+      "version": 1168,
+      "versionNonce": 1960159381,
       "isDeleted": false,
       "id": "O5wbweMweVsQWwdNABtm7",
       "fillStyle": "hachure",
@@ -733,22 +820,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "____",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "____"
+      "originalText": "____",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1033,
-      "versionNonce": 852553080,
+      "version": 1068,
+      "versionNonce": 723949371,
       "isDeleted": false,
       "id": "v3I-eUd4WMJ8VzvC_7rB_",
       "fillStyle": "hachure",
@@ -767,22 +858,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972927,
       "fontFamily": 1,
       "text": "Text",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Text"
+      "originalText": "Text",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1084,
-      "versionNonce": 1360578056,
+      "version": 1119,
+      "versionNonce": 1906167285,
       "isDeleted": false,
       "id": "sdGrFMITxSJsejpV0zIFY",
       "fillStyle": "hachure",
@@ -801,22 +896,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972929,
       "fontFamily": 1,
       "text": "Score",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Score"
+      "originalText": "Score",
+      "lineHeight": 1.3556693953949976,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1007,
-      "versionNonce": 2118418040,
+      "version": 1042,
+      "versionNonce": 1516632027,
       "isDeleted": false,
       "id": "5tPTHdBAi9oW2Cme01uST",
       "fillStyle": "hachure",
@@ -835,22 +934,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972929,
       "fontFamily": 1,
       "text": "0.851",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "0.851"
+      "originalText": "0.851",
+      "lineHeight": 1.3556693953949976,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1082,
-      "versionNonce": 1428342024,
+      "version": 1117,
+      "versionNonce": 1627014997,
       "isDeleted": false,
       "id": "VkvcQQqVWAFSBPXeb-631",
       "fillStyle": "hachure",
@@ -869,22 +972,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972929,
       "fontFamily": 1,
       "text": "0.834",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "0.834"
+      "originalText": "0.834",
+      "lineHeight": 1.3556693953949976,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1154,
-      "versionNonce": 387837816,
+      "version": 1189,
+      "versionNonce": 2016429179,
       "isDeleted": false,
       "id": "r5V4P6WY1YTnZqynbGcPW",
       "fillStyle": "hachure",
@@ -903,22 +1010,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176186,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972924,
       "fontFamily": 1,
       "text": "0.653",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "0.653"
+      "originalText": "0.653",
+      "lineHeight": 1.355669395394998,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1129,
-      "versionNonce": 1704256520,
+      "version": 1164,
+      "versionNonce": 356553909,
       "isDeleted": false,
       "id": "tplkVB19dZGWPH04eoN3-",
       "fillStyle": "hachure",
@@ -937,22 +1048,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176187,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972922,
       "fontFamily": 1,
       "text": "Data",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "Data"
+      "originalText": "Data",
+      "lineHeight": 1.3556693953949983,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1056,
-      "versionNonce": 362889336,
+      "version": 1091,
+      "versionNonce": 1279786267,
       "isDeleted": false,
       "id": "-O_818hE0e37gLO8X85ED",
       "fillStyle": "hachure",
@@ -971,22 +1086,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176187,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "{____}",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "{____}"
+      "originalText": "{____}",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1083,
-      "versionNonce": 7021320,
+      "version": 1118,
+      "versionNonce": 2048275989,
       "isDeleted": false,
       "id": "q_5ogcScUemL3g6XJyETb",
       "fillStyle": "hachure",
@@ -1005,22 +1124,26 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176187,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "{____}",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "{____}"
+      "originalText": "{____}",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     },
     {
       "type": "text",
-      "version": 1076,
-      "versionNonce": 1771204984,
+      "version": 1111,
+      "versionNonce": 35627451,
       "isDeleted": false,
       "id": "hePUy_nbNvIpf6_6PBvbq",
       "fillStyle": "hachure",
@@ -1039,17 +1162,21 @@
       "groupIds": [
         "RaIXn6GkWfnJt6NeXQ4tC"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641746497530,
+      "updated": 1688314176187,
+      "link": null,
+      "locked": false,
       "fontSize": 12.903833622972925,
       "fontFamily": 1,
       "text": "{____}",
-      "baseline": 12.493332325933352,
       "textAlign": "left",
       "verticalAlign": "top",
       "containerId": null,
-      "originalText": "{____}"
+      "originalText": "{____}",
+      "lineHeight": 1.3556693953949979,
+      "baseline": 12
     }
   ],
   "appState": {

docs/images/faq.excalidraw

@@ -0,0 +1,367 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "ellipse",
+      "version": 236,
+      "versionNonce": 266746677,
+      "isDeleted": false,
+      "id": "pLSV--mc1HnQu2oyARr4o",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 593.4166948358098,
+      "y": -409.48426474916266,
+      "strokeColor": "#ff7043",
+      "backgroundColor": "#ff7043",
+      "width": 100,
+      "height": 92,
+      "seed": 513066133,
+      "groupIds": [
+        "agHSoiBea3xOkqCmOPa91",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 281,
+      "versionNonce": 1964375707,
+      "isDeleted": false,
+      "id": "n2qByon4fJyq2-FZxQqM0",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 629.5250284234563,
+      "y": -397.41226924952065,
+      "strokeColor": "#ff7043",
+      "backgroundColor": "#2e303e",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 852571451,
+      "groupIds": [
+        "agHSoiBea3xOkqCmOPa91",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    },
+    {
+      "type": "ellipse",
+      "version": 288,
+      "versionNonce": 239275157,
+      "isDeleted": false,
+      "id": "QIRNCrcWupjspegQn6VDV",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 744.4166948358098,
+      "y": -409.48426474916266,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 100,
+      "height": 92,
+      "seed": 1498649243,
+      "groupIds": [
+        "yonc0SR8jSwQLwuVvb9hG",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 372,
+      "versionNonce": 1982180597,
+      "isDeleted": false,
+      "id": "a-CPMQS9ApW4f7p9FjS8m",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.5250284234563,
+      "y": -397.41226924952065,
+      "strokeColor": "#ffd43b",
+      "backgroundColor": "#ffeb3b",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 1679591227,
+      "groupIds": [
+        "yonc0SR8jSwQLwuVvb9hG",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313209123,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    },
+    {
+      "type": "ellipse",
+      "version": 326,
+      "versionNonce": 1069462005,
+      "isDeleted": false,
+      "id": "nsYgl7js3PjMKKTKcLYf1",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 895.4166948358098,
+      "y": -409.48426474916266,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 100,
+      "height": 92,
+      "seed": 384682581,
+      "groupIds": [
+        "M7MHx-aSMrw_q1mhaAuG6",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 371,
+      "versionNonce": 641029083,
+      "isDeleted": false,
+      "id": "a_CBcLegs1ZMstjO9f4bx",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 931.5250284234563,
+      "y": -397.41226924952065,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 1741404085,
+      "groupIds": [
+        "M7MHx-aSMrw_q1mhaAuG6",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    },
+    {
+      "type": "ellipse",
+      "version": 328,
+      "versionNonce": 22981461,
+      "isDeleted": false,
+      "id": "l88Tj2DPIgTUA2Pij6hPA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1046.4166948358097,
+      "y": -409.48426474916266,
+      "strokeColor": "#7950f2",
+      "backgroundColor": "#7950f2",
+      "width": 100,
+      "height": 92,
+      "seed": 701954677,
+      "groupIds": [
+        "vFCiKG5iTbg2K5S2WQSsn",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 373,
+      "versionNonce": 1236541563,
+      "isDeleted": false,
+      "id": "oqmBYU6SpmJhOl9d1JLJy",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1082.5250284234562,
+      "y": -397.41226924952065,
+      "strokeColor": "#7950f2",
+      "backgroundColor": "#7950f2",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 39445461,
+      "groupIds": [
+        "vFCiKG5iTbg2K5S2WQSsn",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    },
+    {
+      "type": "ellipse",
+      "version": 329,
+      "versionNonce": 1937275061,
+      "isDeleted": false,
+      "id": "IFZTHrjZaEk7Avy2zkma-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1197.4166948358097,
+      "y": -409.48426474916266,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 100,
+      "height": 92,
+      "seed": 894921429,
+      "groupIds": [
+        "70qQx7edpvpMfsvEta9mo",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 374,
+      "versionNonce": 2029017371,
+      "isDeleted": false,
+      "id": "p0xoCeTYO1ZVy-xm2oET3",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1233.5250284234562,
+      "y": -397.41226924952065,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 29.78333282470704,
+      "height": 79.856009000716,
+      "seed": 959268917,
+      "groupIds": [
+        "70qQx7edpvpMfsvEta9mo",
+        "dKeNjYUtPsZHi90BFF7EY"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688313196175,
+      "link": null,
+      "locked": false,
+      "fontSize": 63.88480720057278,
+      "fontFamily": 1,
+      "text": "?",
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "?",
+      "lineHeight": 1.25,
+      "baseline": 56
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

docs/images/install.excalidraw

@@ -5,11 +5,11 @@
   "elements": [
     {
       "type": "rectangle",
-      "version": 1431,
-      "versionNonce": 726709768,
+      "version": 1435,
+      "versionNonce": 1860045045,
       "isDeleted": false,
       "id": "LgVOdCQnXRH2GYHIGLwOx",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -25,17 +25,20 @@
       "groupIds": [
         "s5mCNZzJxsETmWhZoD3qR"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641754344717
+      "updated": 1688311151676,
+      "link": null,
+      "locked": false
     },
     {
       "type": "rectangle",
-      "version": 1531,
-      "versionNonce": 2009304696,
+      "version": 1534,
+      "versionNonce": 2090527963,
       "isDeleted": false,
       "id": "GG_mFtlk6nEiYCQ8vhzjZ",
-      "fillStyle": "cross-hatch",
+      "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
@@ -51,41 +54,48 @@
       "groupIds": [
         "Rezl6DUXkEY8GT9kuDOQX"
       ],
-      "strokeSharpness": "sharp",
+      "frameId": null,
+      "roundness": null,
       "boundElements": [],
-      "updated": 1641754344717
+      "updated": 1688311151677,
+      "link": null,
+      "locked": false
     },
     {
-      "id": "NLfZWFLgvJj3ccuF5yxD8",
       "type": "text",
-      "x": 793,
-      "y": 216.86805555555557,
-      "width": 80,
-      "height": 16,
-      "angle": 0,
-      "strokeColor": "#343a40",
-      "backgroundColor": "transparent",
+      "version": 293,
+      "versionNonce": 1618682453,
+      "isDeleted": false,
+      "id": "NLfZWFLgvJj3ccuF5yxD8",
       "fillStyle": "hachure",
       "strokeWidth": 1,
       "strokeStyle": "solid",
       "roughness": 1,
       "opacity": 100,
-      "groupIds": [],
-      "strokeSharpness": "sharp",
+      "angle": 0,
+      "x": 793,
+      "y": 216.86805555555557,
+      "strokeColor": "#343a40",
+      "backgroundColor": "transparent",
+      "width": 80,
+      "height": 16,
       "seed": 297813880,
-      "version": 288,
-      "versionNonce": 145457528,
-      "isDeleted": false,
-      "boundElements": null,
-      "updated": 1641754464371,
-      "text": "Installing.......",
+      "groupIds": [],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688311151677,
+      "link": null,
+      "locked": false,
       "fontSize": 12.242424242424246,
       "fontFamily": 1,
+      "text": "Installing.......",
       "textAlign": "left",
       "verticalAlign": "top",
-      "baseline": 11,
       "containerId": null,
-      "originalText": "Installing......."
+      "originalText": "Installing.......",
+      "lineHeight": 1.3069306930693065,
+      "baseline": 11
     }
   ],
   "appState": {

docs/images/models.excalidraw

@@ -0,0 +1,531 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "line",
+      "version": 1608,
+      "versionNonce": 602661205,
+      "isDeleted": false,
+      "id": "yVN5CxzfpwAZ0J0GEsEim",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1702.2261496777674,
+      "y": -5.340583377694145,
+      "strokeColor": "#ffeb3b",
+      "backgroundColor": "#ffeb3b",
+      "width": 191.85486538200286,
+      "height": 186.6125755399512,
+      "seed": 1276926237,
+      "groupIds": [
+        "ZD4Gm6iTkUS0-nxoYPoXN"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688309214322,
+      "link": null,
+      "locked": false,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": null,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          0,
+          0
+        ],
+        [
+          -4.946284888409469,
+          1.310194580443298
+        ],
+        [
+          -71.92848036094448,
+          33.259712239166916
+        ],
+        [
+          -71.92848036094448,
+          33.259712239166916
+        ],
+        [
+          -76.36916810848369,
+          36.77184458512375
+        ],
+        [
+          -78.84005658267724,
+          41.86612631945928
+        ],
+        [
+          -95.23551981468633,
+          113.57223168992945
+        ],
+        [
+          -95.23551981468633,
+          113.57223168992945
+        ],
+        [
+          -95.35947746324585,
+          118.64172405287644
+        ],
+        [
+          -93.4324997019961,
+          123.35061560390292
+        ],
+        [
+          -92.71129163005317,
+          124.36030246259705
+        ],
+        [
+          -46.433776844719105,
+          181.9006920694186
+        ],
+        [
+          -46.433776844719105,
+          181.9006920694186
+        ],
+        [
+          -41.973552779866964,
+          185.39027898361584
+        ],
+        [
+          -36.44504150185128,
+          186.6125755399512
+        ],
+        [
+          37.77928857887258,
+          186.61257553995074
+        ],
+        [
+          37.77928857887258,
+          186.61257553995074
+        ],
+        [
+          43.320564233786016,
+          185.34295850958426
+        ],
+        [
+          47.78003745293803,
+          181.84659678914795
+        ],
+        [
+          94.03350816005033,
+          124.29419365112881
+        ],
+        [
+          94.03350816005033,
+          124.29419365112881
+        ],
+        [
+          96.49538791875702,
+          119.21118606558457
+        ],
+        [
+          96.43753267834124,
+          113.56021815873193
+        ],
+        [
+          79.9338983420456,
+          41.794005010746
+        ],
+        [
+          79.9338983420456,
+          41.794005010746
+        ],
+        [
+          77.45849189744823,
+          36.69972327641062
+        ],
+        [
+          73.02231208993142,
+          33.18759093045337
+        ],
+        [
+          6.148302879201523,
+          1.3162045700933498
+        ],
+        [
+          6.148302879201523,
+          1.3162045700933498
+        ],
+        [
+          -0.060096314222328626,
+          -3.469446951953614e-18
+        ],
+        [
+          0,
+          0
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 2051,
+      "versionNonce": 216616981,
+      "isDeleted": false,
+      "id": "ZTKEryJuyeKfeG4nAI4_e",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 2003.5691665051115,
+      "y": 81.12265056081542,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 30.25969046240687,
+      "height": 75.64922615601705,
+      "seed": 1188509340,
+      "groupIds": [
+        "SFux9iuPQVutxOMe0wlW3"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688309215087,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "rectangle",
+      "version": 2192,
+      "versionNonce": 1390810587,
+      "isDeleted": false,
+      "id": "CckqOoWddQBHuvidbpk4P",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 2066.323943610997,
+      "y": 9.547190432552021,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 30.25969046240682,
+      "height": 151.29845231203416,
+      "seed": 331107492,
+      "groupIds": [
+        "SFux9iuPQVutxOMe0wlW3"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "leDc6Y5qdU0OH3j90C072",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1688308769366,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "rectangle",
+      "version": 2072,
+      "versionNonce": 1184041301,
+      "isDeleted": false,
+      "id": "J8ox4BBdzEg-04DGRL-jh",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 2123.5254786767036,
+      "y": 54.936726126162256,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 30.259690462406844,
+      "height": 105.90891661842389,
+      "seed": 1316437916,
+      "groupIds": [
+        "SFux9iuPQVutxOMe0wlW3"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [
+        {
+          "id": "5pptt-wBgcgbxd5LUka3S",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1688308769366,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "rectangle",
+      "version": 2259,
+      "versionNonce": 1727832699,
+      "isDeleted": false,
+      "id": "V54vdkZmQeI13AEahvNPV",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 2177.649637157211,
+      "y": 22.527960822066646,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#00e676",
+      "width": 30.25969046240682,
+      "height": 136.16860708083075,
+      "seed": 1025829916,
+      "groupIds": [
+        "SFux9iuPQVutxOMe0wlW3"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688308769366,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "arrow",
+      "version": 442,
+      "versionNonce": 1922817787,
+      "isDeleted": false,
+      "id": "Inol-LWi8GThocPMvSZGX",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1374.656480958798,
+      "y": 99.23064378133208,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "transparent",
+      "width": 216.36132864000353,
+      "height": 0.12505261106101684,
+      "seed": 1653816315,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688309230624,
+      "link": null,
+      "locked": false,
+      "startBinding": null,
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          216.36132864000353,
+          -0.12505261106101684
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 917,
+      "versionNonce": 151223483,
+      "isDeleted": false,
+      "id": "5pptt-wBgcgbxd5LUka3S",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1802.88085185622,
+      "y": 95.10577155105176,
+      "strokeColor": "#00e676",
+      "backgroundColor": "#34bbde",
+      "width": 184.23585514045453,
+      "height": 3.170483019063738,
+      "seed": 1717004827,
+      "groupIds": [],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688309218767,
+      "link": null,
+      "locked": false,
+      "startBinding": null,
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          184.23585514045453,
+          3.170483019063738
+        ]
+      ]
+    },
+    {
+      "type": "line",
+      "version": 4226,
+      "versionNonce": 789213499,
+      "isDeleted": false,
+      "id": "pikmP5MLN0MZlpnSik5KN",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1235.1617590461694,
+      "y": 25.06522848370164,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "#03a9f4",
+      "width": 127.43688004487302,
+      "height": 164.47752527518452,
+      "seed": 844586005,
+      "groupIds": [
+        "xkmNAKuTv_iYXPLj0sxbd"
+      ],
+      "frameId": null,
+      "roundness": {
+        "type": 2
+      },
+      "boundElements": [],
+      "updated": 1688309232141,
+      "link": null,
+      "locked": false,
+      "startBinding": null,
+      "endBinding": null,
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": null,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          0.4202213858663673,
+          124.31122212244284
+        ],
+        [
+          0.019665374373081804,
+          138.46351494441197
+        ],
+        [
+          6.563281167657464,
+          144.5785456702537
+        ],
+        [
+          29.351096662508862,
+          149.75384171231534
+        ],
+        [
+          67.86895989535736,
+          151.36535158937204
+        ],
+        [
+          104.67023109801114,
+          148.79234001109992
+        ],
+        [
+          124.22301744981843,
+          142.63902935804376
+        ],
+        [
+          126.98067338289118,
+          137.45172392705157
+        ],
+        [
+          127.36797617588668,
+          126.05781617962573
+        ],
+        [
+          127.06398851859365,
+          10.429017682904831
+        ],
+        [
+          126.37870276277025,
+          -0.4957733094390701
+        ],
+        [
+          118.19596944321839,
+          -6.601710860670866
+        ],
+        [
+          100.96487933911183,
+          -10.137946798406993
+        ],
+        [
+          61.697553127560816,
+          -13.11217368581247
+        ],
+        [
+          30.215116414714352,
+          -11.338635495813458
+        ],
+        [
+          5.454393748609269,
+          -5.323010354025768
+        ],
+        [
+          -0.06890386898634299,
+          -0.074694110077688
+        ],
+        [
+          0,
+          0
+        ]
+      ]
+    },
+    {
+      "type": "ellipse",
+      "version": 5067,
+      "versionNonce": 1912378357,
+      "isDeleted": false,
+      "id": "sPZv-4m6hv0RYSW1Kzrrb",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 1,
+      "opacity": 100,
+      "angle": 0,
+      "x": 1235.9555759014816,
+      "y": 13.472824531333586,
+      "strokeColor": "#03a9f4",
+      "backgroundColor": "transparent",
+      "width": 126.61947902597214,
+      "height": 25.607837035548464,
+      "seed": 194842997,
+      "groupIds": [
+        "xkmNAKuTv_iYXPLj0sxbd"
+      ],
+      "frameId": null,
+      "roundness": null,
+      "boundElements": [],
+      "updated": 1688309232141,
+      "link": null,
+      "locked": false
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

docs/index.md

@@ -4,9 +4,9 @@
     <img src="https://raw.githubusercontent.com/neuml/txtai/master/logo.png"/>
 </p>
 
-<h3 align="center">
-    <p>Build AI-powered semantic search applications</p>
-</h3>
+<p align="center">
+    <b>All-in-one embeddings database</b>
+</p>
 
 <p align="center">
     <a href="https://github.com/neuml/txtai/releases">
@@ -29,42 +29,24 @@
     </a>
 </p>
 
--------------------------------------------------------------------------------------------------------------------------------------------------------
+txtai is an all-in-one embeddings database for semantic search, LLM orchestration and language model workflows.
 
-txtai executes machine-learning workflows to transform data and build AI-powered semantic search applications.
+![architecture](images/architecture.png#gh-light-mode-only)
+![architecture](images/architecture-dark.png#gh-dark-mode-only)
 
-![demo](https://raw.githubusercontent.com/neuml/txtai/master/demo.gif)
+Embeddings databases are a union of vector indexes (sparse and dense), graph networks and relational databases. This enables vector search with SQL, topic modeling, retrieval augmented generation and more.
 
-Traditional search systems use keywords to find data. Semantic search applications have an understanding of natural language and identify results that have the same meaning, not necessarily the same keywords.
-
-![search](images/search.png#only-light)
-![search](images/search-dark.png#only-dark)
-
-Backed by state-of-the-art machine learning models, data is transformed into vector representations for search (also known as embeddings). Innovation is happening at a rapid pace, models can understand concepts in documents, audio, images and video.
+Embeddings databases can stand on their own and/or serve as a powerful knowledge source for large language model (LLM) prompts.
 
 Summary of txtai features:
 
-- 🔎 Large-scale similarity search with multiple index backends ([Faiss](https://github.com/facebookresearch/faiss), [Annoy](https://github.com/spotify/annoy), [Hnswlib](https://github.com/nmslib/hnswlib)) and support for external vector databases
-- 📄 Create embeddings for text snippets, documents, audio, images and video
-- 💡 Machine-learning pipelines that run question-answering, labeling, transcription, translation, summarization, LLM prompts and more
-- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be microservices or full-fledged indexing workflows.
+- 🔎 Vector search with SQL, object storage, topic modeling, graph analysis and multimodal indexing
+- 📄 Create embeddings for text, documents, audio, images and video
+- 💡 Pipelines powered by language models that run LLM prompts, question-answering, labeling, transcription, translation, summarization and more
+- ↪️️ Workflows to join pipelines together and aggregate business logic. txtai processes can be simple microservices or multi-model workflows.
 - ⚙️ Build with Python or YAML. API bindings available for [JavaScript](https://github.com/neuml/txtai.js), [Java](https://github.com/neuml/txtai.java), [Rust](https://github.com/neuml/txtai.rs) and [Go](https://github.com/neuml/txtai.go).
-- ☁️ Cloud-native architecture that scales out with container orchestration systems (e.g. Kubernetes)
+- ☁️ Run local or scale out with container orchestration
 
-Applications range from similarity search to NLP-driven data extractions that generate structured data. Semantic workflows transform and find data driven by user intent.
+txtai is built with Python 3.8+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi). txtai is open-source under an Apache 2.0 license.
 
-![flows](images/flows.png#only-light)
-![flows](images/flows-dark.png#only-dark)
-
-The following applications are powered by txtai.
-
-![apps](https://raw.githubusercontent.com/neuml/txtai/master/apps.jpg)
-
-| Application  | Description  |
-|:----------|:-------------|
-| [paperai](https://github.com/neuml/paperai) | Semantic search and workflows for medical/scientific papers |
-| [codequestion](https://github.com/neuml/codequestion) | Semantic search for developers |
-| [tldrstory](https://github.com/neuml/tldrstory) | Semantic search for headlines and story text |
-| [neuspo](https://neuspo.com) | Fact-driven, real-time sports event and news site |
-
-txtai is built with Python 3.7+, [Hugging Face Transformers](https://github.com/huggingface/transformers), [Sentence Transformers](https://github.com/UKPLab/sentence-transformers) and [FastAPI](https://github.com/tiangolo/fastapi)
+*Interested in an easy and secure way to run hosted txtai applications? Then join the [txtai.cloud](https://txtai.cloud) preview to learn more.*

docs/install.md

@@ -9,7 +9,7 @@ The easiest way to install is via pip and PyPI
 pip install txtai
 ```
 
-Python 3.7+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
+Python 3.8+ is supported. Using a Python [virtual environment](https://docs.python.org/3/library/venv.html) is recommended.
 
 ## Optional dependencies
 
@@ -26,6 +26,14 @@ Install all dependencies.
 pip install txtai[all]
 ```
 
+### ANN
+
+Additional ANN backends.
+
+```
+pip install txtai[ann]
+```
+
 ### API
 
 Serve txtai via a web API.
@@ -82,14 +90,22 @@ All pipelines - default install comes with most common pipelines.
 pip install txtai[pipeline]
 ```
 
-More granular extras are available for pipeline categories: `pipeline-audio`, `pipeline-data`, `pipeline-image`, `pipeline-text`, and `pipeline-train`.
+More granular extras are available for pipeline categories: `pipeline-audio`, `pipeline-data`, `pipeline-image`, `pipeline-llm`, `pipeline-text`, and `pipeline-train`.
 
-### Similarity
+### Scoring
 
-Word vectors, support for sentence-transformers models not on the HF Hub and additional ANN libraries.
+Additional scoring methods.
 
 ```
-pip install txtai[similarity]
+pip install txtai[scoring]
+```
+
+### Vectors
+
+Additional vector methods.
+
+```
+pip install txtai[vectors]
 ```
 
 ### Workflow
@@ -118,7 +134,7 @@ Optional audio transcription requires a [system library to be installed](https:/
 
 ### macOS
 
-Run `brew install libomp` see [this link](https://github.com/kyamagu/faiss-wheels#prerequisite)
+Older versions of Faiss have a runtime dependency on `libomp` for macOS. Run `brew install libomp` in this case.
 
 ### Windows
 
@@ -132,6 +148,8 @@ txtai can also be installed directly from GitHub to access the latest, unrelease
 
     pip install git+https://github.com/neuml/txtai
 
+Extras can be installed from GitHub by adding `#egg=txtai[<name-of-extra>]` to the end of the above URL.
+
 ## Conda
 
 A [community-supported txtai package](https://anaconda.org/conda-forge/txtai) is available via conda-forge.

docs/models.md

@@ -0,0 +1,27 @@
+# Model guide
+
+![models](images/models.png)
+
+See the table below for the current recommended models. These models all allow commercial use and offer a blend of speed and performance.
+
+| Component                                            | Model(s)                                                                 |
+| ---------------------------------------------------- | ------------------------------------------------------------------------ |
+| [Embeddings](../embeddings)                          | [all-MiniLM-L6-v2](https://hf.co/sentence-transformers/all-MiniLM-L6-v2) | 
+| [Image Captions](./pipeline/image/caption.md)        | [BLIP](https://hf.co/Salesforce/blip-image-captioning-base)              |
+| [Labels - Zero Shot](./pipeline/text/labels.md)      | [BART-Large-MNLI](https://hf.co/facebook/bart-large)                     |
+| [Labels - Fixed](./pipeline/text/labels.md)          | Fine-tune with [training pipeline](./pipeline/train/trainer.md)          |
+| [Large Language Model (LLM)](./pipeline/text/llm.md) | [Mistral 7B OpenOrca](https://hf.co/Open-Orca/Mistral-7B-OpenOrca)       |
+| [Summarization](./pipeline/text/summary.md)          | [DistilBART](https://hf.co/sshleifer/distilbart-cnn-12-6)                |
+| [Text-to-Speech](./pipeline/audio/texttospeech.md)   | [ESPnet JETS](https://hf.co/NeuML/ljspeech-jets-onnx)                    |
+| [Transcription](./pipeline/audio/transcription.md)   | [Whisper](https://hf.co/openai/whisper-base)                             | 
+| [Translation](./pipeline/text/translation.md)        | [OPUS Model Series](https://hf.co/Helsinki-NLP)                          |
+
+Models can be loaded as either a path from the Hugging Face Hub or a local directory. Model paths are optional, defaults are loaded when not specified. For tasks with no recommended model, txtai uses the default models as shown in the Hugging Face Tasks guide.
+
+See the following links to learn more.
+
+- [Hugging Face Tasks](https://hf.co/tasks)
+- [Hugging Face Model Hub](https://hf.co/models)
+- [MTEB Leaderboard](https://hf.co/spaces/mteb/leaderboard)
+- [LMSYS LLM Leaderboard](https://chat.lmsys.org/?leaderboard)
+- [Open LLM Leaderboard](https://hf.co/spaces/HuggingFaceH4/open_llm_leaderboard)

docs/pipeline/data/textractor.md

@@ -5,6 +5,8 @@
 
 The Textractor pipeline extracts and splits text from documents. This pipeline uses either an [Apache Tika](https://github.com/chrismattmann/tika-python) backend (if Java is available) or [BeautifulSoup4](https://pypi.org/project/beautifulsoup4/).
 
+_Note: BeautifulSoup4 only supports HTML documents, anything else requires Tika and Java to be installed._
+
 ## Example
 
 The following shows a simple example using this pipeline.

docs/pipeline/index.md

@@ -9,7 +9,7 @@ Pipelines are run with Python or configuration. Pipelines can be instantiated in
 
 ## List of pipelines
 
-The following is a list of the default pipelines available in txtai.
+The following is a list of the default pipelines available in txtai. All pipelines use default models when otherwise not specified. See the [model guide](../models) for the current model recommentations.
 
 - Audio
     - [TextToSpeech](audio/texttospeech)
@@ -24,10 +24,9 @@ The following is a list of the default pipelines available in txtai.
     - [Objects](image/objects)
 - Text
     - [Entity](text/entity)
-    - [Extractive QA](text/extractor)
-    - [Generator](text/generator)
+    - [Extractor](text/extractor)
     - [Labeling](text/labels)
-    - [Sequences](text/sequences)
+    - [LLM](text/llm)
     - [Similarity](text/similarity)
     - [Summary](text/summary)
     - [Translation](text/translation)

docs/pipeline/text/extractor.md

@@ -3,7 +3,9 @@
 ![pipeline](../../images/pipeline.png#only-light)
 ![pipeline](../../images/pipeline-dark.png#only-dark)
 
-The Extractor pipeline is a combination of a similarity instance (embeddings or similarity pipeline) to build a question context and a model that answers questions. The model can be a prompt-driven large language model (LLM), an extractive question-answering model or a custom pipeline.
+The Extractor pipeline joins a prompt, context data store and generative model together to extract knowledge.
+
+The data store can be an embeddings database or a similarity instance with associated input text. The generative model can be a prompt-driven large language model (LLM), an extractive question-answering model or a custom pipeline. This is known as prompt-driven search or retrieval augmented generation (RAG).
 
 ## Example
 
@@ -13,23 +15,49 @@ The following shows a simple example using this pipeline.
 from txtai.embeddings import Embeddings
 from txtai.pipeline import Extractor
 
-# Embeddings model ranks candidates before passing to QA pipeline
-embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2"})
+# LLM prompt
+def prompt(question):
+  return f"""
+  Answer the following question using the provided context.
+
+  Question:
+  {question}
+
+  Context:
+  """
+
+# Input data
+data = [
+  "US tops 5 million confirmed virus cases",
+  "Canada's last fully intact ice shelf has suddenly collapsed, " +
+  "forming a Manhattan-sized iceberg",
+  "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
+  "The National Park Service warns against sacrificing slower friends " +
+  "in a bear attack",
+  "Maine man wins $1M from $25 lottery ticket",
+  "Make huge profits without work, earn up to $100,000 a day"
+]
+
+# Build embeddings index
+embeddings = Embeddings({"content": True})
+embeddings.index([(uid, text, None) for uid, text in enumerate(data)])
 
 # Create and run pipeline
-extractor = Extractor(embeddings, "distilbert-base-cased-distilled-squad")
-extractor([["What was won"] * 3 + [False]],
-          ["Maine man wins $1M from $25 lottery ticket"])
+extractor = Extractor(embeddings, "google/flan-t5-base")
+extractor([{"query": "What was won?", "question": prompt("What was won?")}])
 ```
 
 See the links below for more detailed examples.
 
 | Notebook  | Description  |       |
 |:----------|:-------------|------:|
+| [Prompt-driven search with LLMs](https://github.com/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) | Embeddings-guided and Prompt-driven search with Large Language Models (LLMs) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) |
+| [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
 | [Extractive QA with txtai](https://github.com/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) | Introduction to extractive question-answering with txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/05_Extractive_QA_with_txtai.ipynb) |
 | [Extractive QA with Elasticsearch](https://github.com/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) | Run extractive question-answering queries with Elasticsearch | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/06_Extractive_QA_with_Elasticsearch.ipynb) |
 | [Extractive QA to build structured data](https://github.com/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) | Build structured datasets using extractive question-answering | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/20_Extractive_QA_to_build_structured_data.ipynb) |
-| [Prompt-driven search with LLMs](https://github.com/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) | Embeddings-guided and Prompt-driven search with Large Language Models (LLMs) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) |
 
 ## Configuration-driven example
 
@@ -37,20 +65,52 @@ Pipelines are run with Python or configuration. Pipelines can be instantiated in
 
 ### config.yml
 ```yaml
-# Create pipeline using lower case class name
+# Allow documents to be indexed
+writable: True
+
+# Content is required for extractor pipeline
+embeddings:
+  content: True
+
 extractor:
+  path: google/flan-t5-base
+
+workflow:
+  search:
+    tasks:
+      - task: extractor
+        template: |
+          Answer the following question using the provided context.
+
+          Question:
+          {text}
+
+          Context:
+        action: extractor
 ```
 
 ### Run with Workflows
 
+Built in tasks make using the extractor pipeline easier.
+
 ```python
 from txtai.app import Application
 
 # Create and run pipeline with workflow
 app = Application("config.yml")
-list(app.extract([{"name": "What was won", "query": "What was won",
-                   "question", "What was won", "snippet": False}], 
-                 ["Maine man wins $1M from $25 lottery ticket"]))
+app.add([
+  "US tops 5 million confirmed virus cases",
+  "Canada's last fully intact ice shelf has suddenly collapsed, " +
+  "forming a Manhattan-sized iceberg",
+  "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
+  "The National Park Service warns against sacrificing slower friends " +
+  "in a bear attack",
+  "Maine man wins $1M from $25 lottery ticket",
+  "Make huge profits without work, earn up to $100,000 a day"
+])
+app.index()
+
+list(app.workflow("search", ["What was won?"]))
 ```
 
 ### Run with API
@@ -59,9 +119,9 @@ list(app.extract([{"name": "What was won", "query": "What was won",
 CONFIG=config.yml uvicorn "txtai.api:app" &
 
 curl \
-  -X POST "http://localhost:8000/extract" \
+  -X POST "http://localhost:8000/workflow" \
   -H "Content-Type: application/json" \
-  -d '{"queue": [{"name":"What was won", "query": "What was won", "question": "What was won", "snippet": false}], "texts": ["Maine man wins $1M from $25 lottery ticket"]}'
+  -d '{"name": "search", "elements": ["What was won"]}'
 ```
 
 ## Methods

docs/pipeline/text/generator.md

@@ -1,62 +0,0 @@
-# Generator
-
-![pipeline](../../images/pipeline.png#only-light)
-![pipeline](../../images/pipeline-dark.png#only-dark)
-
-The Generator pipeline takes an input prompt and generates follow-on text.
-
-## Example
-
-The following shows a simple example using this pipeline.
-
-```python
-from txtai.pipeline import Generator
-
-# Create and run pipeline
-generator = Generator()
-generator("Hello, how are you?")
-```
-
-## Configuration-driven example
-
-Pipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).
-
-### config.yml
-```yaml
-# Create pipeline using lower case class name
-generator:
-
-# Run pipeline with workflow
-workflow:
-  generator:
-    tasks:
-      - action: generator
-```
-
-### Run with Workflows
-
-```python
-from txtai.app import Application
-
-# Create and run pipeline with workflow
-app = Application("config.yml")
-list(app.workflow("generator", ["Hello, how are you?"]))
-```
-
-### Run with API
-
-```bash
-CONFIG=config.yml uvicorn "txtai.api:app" &
-
-curl \
-  -X POST "http://localhost:8000/workflow" \
-  -H "Content-Type: application/json" \
-  -d '{"name":"generator", "elements": ["Hello, how are you?"]}'
-```
-
-## Methods
-
-Python documentation for the pipeline.
-
-### ::: txtai.pipeline.Generator.__init__
-### ::: txtai.pipeline.Generator.__call__

docs/pipeline/text/llm.md

@@ -0,0 +1,141 @@
+# LLM
+
+![pipeline](../../images/pipeline.png#only-light)
+![pipeline](../../images/pipeline-dark.png#only-dark)
+
+The LLM pipeline runs prompts through a large language model (LLM). This pipeline autodetects the LLM framework based on the model path.
+
+## Example
+
+The following shows a simple example using this pipeline.
+
+```python
+from txtai.pipeline import LLM
+
+# Create and run LLM pipeline
+llm = LLM()
+llm(
+  """
+  Answer the following question using the provided context.
+
+  Question:
+  What are the applications of txtai?
+
+  Context:
+  txtai is an open-source platform for semantic search and
+  workflows powered by language models.
+  """
+)
+```
+
+The LLM pipeline automatically detects the underlying LLM framework. This can also be manually set.
+
+See the [LiteLLM documentation](https://litellm.vercel.app/docs/providers) for the options available with LiteLLM models. Models with llama.cpp support both local and remote GGUF paths on the HF Hub.
+
+```python
+from txtai.pipeline import LLM
+
+# Set method as litellm
+llm = LLM("vllm/Open-Orca/Mistral-7B-OpenOrca", method="litellm")
+
+# Set method as llama.cpp
+llm = LLM("TheBloke/Mistral-7B-OpenOrca-GGUF/mistral-7b-openorca.Q4_K_M.gguf",
+           method="llama.cpp")
+```
+
+Models can be externally loaded and passed to pipelines. This is useful for models that are not yet supported by Transformers and/or need special initialization.
+
+```python
+import torch
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from txtai.pipeline import LLM
+
+# Load Mistral-7B-OpenOrca
+path = "Open-Orca/Mistral-7B-OpenOrca"
+model = AutoModelForCausalLM.from_pretrained(
+  path,
+  torch_dtype=torch.bfloat16,
+)
+tokenizer = AutoTokenizer.from_pretrained(path)
+
+llm = LLM((model, tokenizer))
+```
+
+See the links below for more detailed examples.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Prompt-driven search with LLMs](https://github.com/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) | Embeddings-guided and Prompt-driven search with Large Language Models (LLMs) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/42_Prompt_driven_search_with_LLMs.ipynb) |
+| [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
+| [Generate knowledge with Semantic Graphs and RAG](https://github.com/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) | Knowledge exploration and discovery with Semantic Graphs and RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/55_Generate_knowledge_with_Semantic_Graphs_and_RAG.ipynb) |
+| [Build knowledge graphs with LLMs](https://github.com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) | Build knowledge graphs with LLM-driven entity extraction | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |
+| [Advanced RAG with graph path traversal](https://github.com/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) | Graph path traversal to collect complex sets of data for advanced RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) |
+| [Advanced RAG with guided generation](https://github.com/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) | Retrieval Augmented and Guided Generation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) |
+| [RAG with llama.cpp and external API services](https://github.com/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) | RAG with additional vector and LLM frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) |
+
+## Configuration-driven example
+
+Pipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).
+
+### config.yml
+```yaml
+# Create pipeline using lower case class name
+# Use `generator` or `sequences` to force model type
+llm:
+
+# Run pipeline with workflow
+workflow:
+  llm:
+    tasks:
+      - action: llm
+```
+
+Similar to the Python example above, the underlying [Hugging Face pipeline parameters](https://huggingface.co/docs/transformers/main/main_classes/pipelines#transformers.pipeline.model) and [model parameters](https://huggingface.co/docs/transformers/model_doc/auto#transformers.AutoModel.from_pretrained) can be set in pipeline configuration.
+
+```yaml
+llm:
+  path: Open-Orca/Mistral-7B-OpenOrca
+  torch_dtype: torch.bfloat16
+```
+
+### Run with Workflows
+
+```python
+from txtai.app import Application
+
+# Create and run pipeline with workflow
+app = Application("config.yml")
+list(app.workflow("llm", [
+  """
+  Answer the following question using the provided context.
+ 
+  Question:
+  What are the applications of txtai? 
+
+  Context:
+  txtai is an open-source platform for semantic search and
+  workflows powered by language models.
+  """
+]))
+```
+
+### Run with API
+
+```bash
+CONFIG=config.yml uvicorn "txtai.api:app" &
+
+curl \
+  -X POST "http://localhost:8000/workflow" \
+  -H "Content-Type: application/json" \
+  -d '{"name":"sequences", "elements": ["Answer the following question..."]}'
+```
+
+## Methods
+
+Python documentation for the pipeline.
+
+### ::: txtai.pipeline.LLM.__init__
+### ::: txtai.pipeline.LLM.__call__

docs/pipeline/text/sequences.md

@@ -1,69 +0,0 @@
-# Sequences
-
-![pipeline](../../images/pipeline.png#only-light)
-![pipeline](../../images/pipeline-dark.png#only-dark)
-
-The Sequences pipeline runs text through a sequence-sequence model and generates output text.
-
-## Example
-
-The following shows a simple example using this pipeline.
-
-```python
-from txtai.pipeline import Sequences
-
-# Create and run pipeline
-sequences = Sequences()
-sequences("Hello, how are you?", "translate English to French: ")
-```
-
-See the link below for a more detailed example.
-
-| Notebook  | Description  |       |
-|:----------|:-------------|------:|
-| [Query translation](https://github.com/neuml/txtai/blob/master/examples/33_Query_translation.ipynb) | Domain-specific natural language queries with query translation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/33_Query_translation.ipynb) |
-
-## Configuration-driven example
-
-Pipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).
-
-### config.yml
-```yaml
-# Create pipeline using lower case class name
-sequences:
-
-# Run pipeline with workflow
-workflow:
-  sequences:
-    tasks:
-      - action: sequences
-        args: ["translate English to French: "]
-```
-
-### Run with Workflows
-
-```python
-from txtai.app import Application
-
-# Create and run pipeline with workflow
-app = Application("config.yml")
-list(app.workflow("sequences", ["Hello, how are you?"]))
-```
-
-### Run with API
-
-```bash
-CONFIG=config.yml uvicorn "txtai.api:app" &
-
-curl \
-  -X POST "http://localhost:8000/workflow" \
-  -H "Content-Type: application/json" \
-  -d '{"name":"sequences", "elements": ["Hello, how are you?"]}'
-```
-
-## Methods
-
-Python documentation for the pipeline.
-
-### ::: txtai.pipeline.Sequences.__init__
-### ::: txtai.pipeline.Sequences.__call__

docs/pipeline/train/trainer.md

@@ -59,6 +59,45 @@ The HFTrainer pipeline builds and/or fine-tunes models for following training ta
 | text-classification | Classify text with a set of labels |
 | token-detection | ELECTRA-style pre-training with replaced token detection |
 
+## PEFT
+
+Parameter-Efficient Fine-Tuning (PEFT) is supported through [Hugging Face's PEFT library](https://github.com/huggingface/peft). Quantization is provided through [bitsandbytes](https://github.com/TimDettmers/bitsandbytes). See the examples below.
+
+```python
+from txtai.pipeline import HFTrainer
+
+trainer = HFTrainer()
+trainer(..., quantize=True, lora=True)
+```
+
+When these parameters are set to True, they use default configuration. This can also be customized.
+
+```python
+quantize = {
+    "load_in_4bit": True,
+    "bnb_4bit_use_double_quant": True,
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_compute_dtype": "bfloat16"
+}
+
+lora = {
+    "r": 16,
+    "lora_alpha": 8,
+    "target_modules": "all-linear",
+    "lora_dropout": 0.05,
+    "bias": "none"
+}
+
+trainer(..., quantize=quantize, lora=lora)
+```
+
+The parameters also accept `transformers.BitsAndBytesConfig` and `peft.LoraConfig` instances.
+
+See the following PEFT documentation links for more information.
+
+- [Quantization](https://huggingface.co/docs/peft/developer_guides/quantization)
+- [LoRA](https://huggingface.co/docs/peft/developer_guides/lora)
+
 ## Methods 
 
 Python documentation for the pipeline.

docs/poweredby.md

@@ -0,0 +1,14 @@
+# Powered by txtai
+
+The following applications are powered by txtai. 
+
+![apps](https://raw.githubusercontent.com/neuml/txtai/master/apps.jpg)
+
+| Application  | Description  |
+|:------------ |:-------------|
+| [txtchat](https://github.com/neuml/txtchat) | Retrieval Augmented Generation (RAG) powered search |
+| [paperai](https://github.com/neuml/paperai) | Semantic search and workflows for medical/scientific papers |
+| [codequestion](https://github.com/neuml/codequestion) | Semantic search for developers |
+| [tldrstory](https://github.com/neuml/tldrstory) | Semantic search for headlines and story text |
+
+In addition to this list, there are also many other [open-source projects](https://github.com/neuml/txtai/network/dependents), [published research](https://scholar.google.com/scholar?q=txtai&hl=en&as_ylo=2022) and closed proprietary/commercial projects that have built on txtai in production.

docs/usecases.md

@@ -0,0 +1,73 @@
+# Use Cases
+
+The following sections introduce common txtai use cases. A comprehensive set of over 50 [example notebooks and applications](../examples) are also available.
+
+## Semantic Search
+
+Build semantic/similarity/vector/neural search applications.
+
+![demo](https://raw.githubusercontent.com/neuml/txtai/master/demo.gif)
+
+Traditional search systems use keywords to find data. Semantic search has an understanding of natural language and identifies results that have the same meaning, not necessarily the same keywords.
+
+![search](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/search.png#gh-light-mode-only)
+![search](https://raw.githubusercontent.com/neuml/txtai/master/docs/images/search-dark.png#gh-dark-mode-only)
+
+Get started with the following examples.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Introducing txtai](https://github.com/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) [▶️](https://www.youtube.com/watch?v=SIezMnVdmMs) | Overview of the functionality provided by txtai | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/01_Introducing_txtai.ipynb) |
+| [Similarity search with images](https://github.com/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) | Embed images and text into the same space for search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/13_Similarity_search_with_images.ipynb) |
+| [Build a QA database](https://github.com/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) | Question matching with semantic search | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/34_Build_a_QA_database.ipynb) |
+| [Semantic Graphs](https://github.com/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) | Explore topics, data connectivity and run network analysis| [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/38_Introducing_the_Semantic_Graph.ipynb) |
+
+## LLM Orchestration
+
+LLM chains, retrieval augmented generation (RAG), chat with your data, pipelines and workflows that interface with large language models (LLMs).
+
+### Chains
+
+Integrate LLM chains (known as workflows in txtai), multiple LLM agents and self-critique.
+
+![llm](images/llm.png)
+
+See below to learn more.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Prompt templates and task chains](https://github.com/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) | Build model prompts and connect tasks together with workflows | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/44_Prompt_templates_and_task_chains.ipynb) |
+| [Integrate LLM frameworks](https://github.com/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) | Integrate llama.cpp, LiteLLM and custom generation frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/53_Integrate_LLM_Frameworks.ipynb) |
+| [Build knowledge graphs with LLMs](https://github.com/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) | Build knowledge graphs with LLM-driven entity extraction | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/57_Build_knowledge_graphs_with_LLM_driven_entity_extraction.ipynb) |
+
+### Retrieval augmented generation
+
+Retrieval augmented generation (RAG) reduces the risk of LLM hallucinations by constraining the output with a knowledge base as context. RAG is commonly used to "chat with your data".
+
+![rag](images/rag.png#gh-light-mode-only)
+![rag](images/rag-dark.png#gh-dark-mode-only)
+
+A novel feature of txtai is that it can provide both an answer and source citation.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Build RAG pipelines with txtai](https://github.com/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) | Guide on retrieval augmented generation including how to create citations | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/52_Build_RAG_pipelines_with_txtai.ipynb) |
+| [Advanced RAG with graph path traversal](https://github.com/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) | Graph path traversal to collect complex sets of data for advanced RAG | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/58_Advanced_RAG_with_graph_path_traversal.ipynb) |
+| [Advanced RAG with guided generation](https://github.com/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) | Retrieval Augmented and Guided Generation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/60_Advanced_RAG_with_guided_generation.ipynb) |
+| [RAG with llama.cpp and external API services](https://github.com/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) | RAG with additional vector and LLM frameworks | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/62_RAG_with_llama_cpp_and_external_API_services.ipynb) |
+
+## Language Model Workflows
+
+Language model workflows, also known as semantic workflows, connect language models together to build intelligent applications.
+
+![flows](images/flows.png#gh-light-mode-only)
+![flows](images/flows-dark.png#gh-dark-mode-only)
+
+While LLMs are powerful, there are plenty of smaller, more specialized models that work better and faster for specific tasks. This includes models for extractive question-answering, automatic summarization, text-to-speech, transcription and translation.
+
+| Notebook  | Description  |       |
+|:----------|:-------------|------:|
+| [Run pipeline workflows](https://github.com/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) [▶️](https://www.youtube.com/watch?v=UBMPDCn1gEU) | Simple yet powerful constructs to efficiently process data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/14_Run_pipeline_workflows.ipynb) |
+| [Building abstractive text summaries](https://github.com/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) | Run abstractive text summarization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/09_Building_abstractive_text_summaries.ipynb) |
+| [Transcribe audio to text](https://github.com/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) | Convert audio files to text | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/11_Transcribe_audio_to_text.ipynb) |
+| [Translate text between languages](https://github.com/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) | Streamline machine translation and language detection | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/12_Translate_text_between_languages.ipynb) |

docs/why.md

@@ -3,19 +3,19 @@
 ![why](images/why.png#only-light)
 ![why](images/why-dark.png#only-dark)
 
-In addition to traditional search systems, a growing number of semantic search solutions are available, so why txtai?
+New vector databases, LLM frameworks and everything in between are sprouting up daily. Why build with txtai?
 
 - Up and running in minutes with [pip](../install/) or [Docker](../cloud/)
 ```python
 # Get started in a couple lines
-from txtai.embeddings import Embeddings
+import txtai
 
-embeddings = Embeddings({"path": "sentence-transformers/all-MiniLM-L6-v2"})
-embeddings.index([(0, "Correct", None), (1, "Not what we hoped", None)])
+embeddings = txtai.Embeddings()
+embeddings.index(["Correct", "Not what we hoped"])
 embeddings.search("positive", 1)
-#[(0, 0.2986203730106354)]
+#[(0, 0.29862046241760254)]
 ```
-- Build applications in your programming language of choice via the API
+- Built-in API makes it easy to develop applications using your programming language of choice
 ```yaml
 # app.yml
 embeddings:
@@ -25,8 +25,7 @@ embeddings:
 CONFIG=app.yml uvicorn "txtai.api:app"
 curl -X GET "http://localhost:8000/search?query=positive"
 ```
-- Connect machine learning models together to build intelligent data processing workflows
-- Works with both small and big data - scale when needed
-- Supports micromodels all the way up to large language models (LLMs)
-- Low footprint - install additional dependencies when you need them
+- Run local - no need to ship data off to disparate remote services
+- Work with micromodels all the way up to large language models (LLMs)
+- Low footprint - install additional dependencies and scale up when needed
 - [Learn by example](../examples) - notebooks cover all available functionality

docs/workflow/index.md

@@ -5,7 +5,7 @@
 
 Workflows are a simple yet powerful construct that takes a callable and returns elements. Workflows operate well with pipelines but can work with any callable object. Workflows are streaming and work on data in batches, allowing large volumes of data to be processed efficiently.
 
-Given that pipelines are callable objects, workflows enable efficient processing of pipeline data. Transformers models typically work with smaller batches of data, workflows are well suited to feed a series of transformers pipelines. 
+Given that pipelines are callable objects, workflows enable efficient processing of pipeline data. Large language models typically work with smaller batches of data, workflows are well suited to feed a series of transformers pipelines. 
 
 An example of the most basic workflow:
 
@@ -81,7 +81,7 @@ embeddings.search("wildlife", 1)
 
 ## Configuration-driven example
 
-Workflows can be defined using Python as shown above but they can also run with YAML configuration.
+Workflows can also be defined with YAML configuration.
 
 ```yaml
 writable: true
@@ -125,7 +125,66 @@ list(app.workflow("index", [
 app.search("wildlife")
 ```
 
-The code above executes a workflow defined in the file `workflow.yml`. The API is used to run the workflow locally, there is minimal overhead running workflows in this manner. It's a matter of preference.
+The code above executes a workflow defined in the file `workflow.yml.
+
+## LLM workflow example
+
+Workflows can connect multiple LLM prompting tasks together.
+
+```yaml
+llm:
+  path: google/flan-t5-xl
+
+workflow:
+  llm:
+    tasks:
+      - task: template
+        template: |
+          Extract keywords for the following text.
+
+          {text}
+        action: llm
+      - task: template
+        template: |
+          Translate the following text into French.
+
+          {text}
+        action: llm
+```
+
+```python
+from txtai.app import Application
+
+app = Application("workflow.yml")
+list(app.workflow("llm", [
+  """
+  txtai is an open-source platform for semantic search
+  and workflows powered by language models.
+  """
+]))
+```
+
+Any txtai pipeline/workflow task can be connected in workflows with LLMs.
+
+```yaml
+llm:
+  path: google/flan-t5-xl
+
+translation:
+
+workflow:
+  llm:
+    tasks:
+      - task: template
+        template: |
+          Extract keywords for the following text.
+
+          {text}
+        action: llm
+      - action: translation
+        args:
+          - fr
+```
 
 See the following links for more information.
 

docs/workflow/task/template.md

@@ -0,0 +1,35 @@
+# Template Task
+
+![task](../../images/task.png#only-light)
+![task](../../images/task-dark.png#only-dark)
+
+The Template Task generates text from a template and task inputs. Templates can be used to prepare data for a number of tasks including generating large
+language model (LLM) prompts.
+
+## Example
+
+The following shows a simple example using this task as part of a workflow.
+
+```python
+from txtai.workflow import TemplateTask, Workflow
+
+workflow = Workflow([TemplateTask(template="This is a {text} task")])
+workflow([{"text": "template"}])
+```
+
+## Configuration-driven example
+
+This task can also be created with workflow configuration.
+
+```yaml
+workflow:
+  tasks:
+    - task: template
+      template: This is a {text} task
+```
+
+## Methods
+
+Python documentation for the task.
+
+### ::: txtai.workflow.TemplateTask.__init__

examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "02 - Build an Embeddings index with Hugging Face Datasets",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -22,8 +20,6 @@
       "source": [
         "# Build an Embeddings index with Hugging Face Datasets\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "This notebook shows how txtai can index and search with Hugging Face's [Datasets](https://github.com/huggingface/datasets) library. Datasets opens access to a large and growing list of publicly available datasets. Datasets has functionality to select, transform and filter data stored in each dataset.\n",
         "\n",
         "In this example, txtai will be used to index and query a dataset.\n",

examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "03 - Build an Embeddings index from a data source",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -21,9 +19,7 @@
       "source": [
         "# Build an Embeddings index from a data source\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
-        "In Part 1, we gave a general overview of txtai, the backing technology and examples of how to use it for similarity searches. Part 2 covered an embedding index with a larger dataset. \n",
+        "In Part 1, we gave a general overview of txtai, the backing technology and examples of how to use it for similarity searches. Part 2 covered an embedding index with a larger dataset.\n",
         "\n",
         "For real world large-scale use cases, data is often stored in a database (Elasticsearch, SQL, MongoDB, files, etc). Here we'll show how to read from SQLite, build a Embedding index backed by word embeddings and run queries against the generated Embeddings index.\n",
         "\n",
@@ -88,9 +84,7 @@
       "source": [
         "# Build Word Vectors\n",
         "\n",
-        "This example will build a search system backed by word embeddings. Word embeddings provide a good tradeoff of performance to functionality for a similarity search system. With that being said, transformer models are making great progress in scaling performance down to smaller models and are the preferred vector backend for txtai in most cases. \n",
-        "\n",
-        "While not quite as powerful as transformer embeddings, they often provide a g\n",
+        "This example will build a search system backed by word embeddings. Word embeddings provide a good tradeoff of performance to functionality for a similarity search system. With that being said, transformer models are making great progress in scaling performance down to smaller models and are the preferred vector backend for txtai in most cases.\n",
         "\n",
         "For this notebook, we'll build our own custom embeddings for demo purposes. A number of pre-trained word embedding models are available:\n",
         "\n",
@@ -167,7 +161,7 @@
       "source": [
         "# Build an embeddings index\n",
         "\n",
-        "The following steps build an embeddings index using the word vector model just created. This model builds a BM25 + fastText index. BM25 is used to build a weighted average of the word embeddings for a section. More information on this method can be found in this [Medium article](https://towardsdatascience.com/building-a-sentence-embedding-index-with-fasttext-and-bm25-f07e7148d240?gi=79da927aa10). "
+        "The following steps build an embeddings index using the word vector model just created. This model builds a BM25 + fastText index. BM25 is used to build a weighted average of the word embeddings for a section. More information on this method can be found in this [Medium article](https://towardsdatascience.com/building-a-sentence-embedding-index-with-fasttext-and-bm25-f07e7148d240?gi=79da927aa10)."
       ]
     },
     {

examples/04_Add_semantic_search_to_Elasticsearch.ipynb

@@ -4,9 +4,7 @@
   "metadata": {
     "accelerator": "GPU",
     "colab": {
-      "name": "04 - Add semantic search to Elasticsearch",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "display_name": "Python 3",
@@ -22,8 +20,6 @@
       "source": [
         "# Add semantic search to Elasticsearch\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "Part 2 and Part 3 of this series showed how to index and search data in txtai. Part 2 indexed and searched a Hugging Face Dataset, Part 3 indexed and searched an external data source. \n",
         "\n",
         "txtai is modular in design, it's components can be individually used. txtai has a similarity function that works on lists of text. This method can be integrated with any external search service, such as a REST API, a SQL query or anything else that returns text search results. \n",

examples/05_Extractive_QA_with_txtai.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "05 - Extractive QA with txtai",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -21,8 +19,6 @@
       "source": [
         "# Extractive QA with txtai\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "In Parts 1 through 4, we gave a general overview of txtai, the backing technology and examples of how to use it for similarity searches. This notebook builds on that and extends to building extractive question-answering systems."
       ]
     },

examples/06_Extractive_QA_with_Elasticsearch.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "06 - Extractive QA with Elasticsearch",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -21,8 +19,6 @@
       "source": [
         "# Extractive QA with Elasticsearch\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "txtai is datastore agnostic, the library analyzes sets of text. The following example shows how extractive question-answering can be added on top of an Elasticsearch system."
       ]
     },

examples/07_Apply_labels_with_zero_shot_classification.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "07 - Apply labels with zero-shot classification",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -21,8 +19,6 @@
       "source": [
         "# Apply labels with zero-shot classification\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "This notebook shows how zero-shot classification can be used to perform text classification, labeling and topic modeling. txtai provides a light-weight wrapper around the zero-shot-classification pipeline in Hugging Face Transformers. This method works impressively well out of the box. Kudos to the Hugging Face team for the phenomenal work on zero-shot classification!\n",
         "\n",
         "The examples in this notebook pick the best matching label using a list of labels for a snippet of text.\n",

examples/08_API_Gallery.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "08 - API Gallery",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "display_name": "Python 3",
@@ -21,8 +19,6 @@
       "source": [
         "# API Gallery\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "The txtai API is a web-based service backed by [FastAPI](https://fastapi.tiangolo.com/). All txtai functionality including similarity search, extractive QA and zero-shot labeling is available via the API.\n",
         "\n",
         "This notebook installs the txtai API and shows an example using each of the supported language bindings for txtai."

examples/09_Building_abstractive_text_summaries.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "09 - Building abstractive text summaries",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -21,8 +19,6 @@
       "source": [
         "# Building abstractive text summaries\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "In the field of text summarization, there are two primary categories of summarization, extractive and abstractive summarization.\n",
         "\n",
         "Extractive summarization takes subsections of the text and joins them together to form a summary. This is commonly backed by graph algorithms like TextRank to find the sections/sentences with the most commonality. These summaries can be highly effective but they are unable to transform text and don't have a contextual understanding.\n",

examples/10_Extract_text_from_documents.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "10 - Extract text from documents",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -21,11 +19,9 @@
       "source": [
         "# Extract text from documents\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
+        "Up to this point, all the examples have been working with sections of text, which have already been split through some other means. What happens if we're working with documents? First we need to get the text out of these documents, then figure out how to index to best support vector search.\n",
         "\n",
-        "Up to this point, all the examples have been working with sections of text, which have already been split through some other means. What happens if we're working with documents? First we need to get the text out of these documents, then figure out how to index to best support similarity search.\n",
-        "\n",
-        "This notebook shows how documents can have text extracted and segmented to support similarity search."
+        "This notebook shows how documents can have text extracted and split to support vector search and retrieval augmented generation (RAG)."
       ]
     },
     {
@@ -49,14 +45,14 @@
         "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline]\n",
         "\n",
         "# Get test data\n",
-        "!wget -N https://github.com/neuml/txtai/releases/download/v3.5.0/tests.tar.gz\n",
+        "!wget -N https://github.com/neuml/txtai/releases/download/v6.2.0/tests.tar.gz\n",
         "!tar -xvzf tests.tar.gz\n",
         "\n",
         "# Install NLTK\n",
         "import nltk\n",
         "nltk.download('punkt')"
       ],
-      "execution_count": null,
+      "execution_count": 19,
       "outputs": []
     },
     {
@@ -85,7 +81,7 @@
         "# Create textractor model\n",
         "textractor = Textractor()"
       ],
-      "execution_count": null,
+      "execution_count": 20,
       "outputs": []
     },
     {
@@ -104,28 +100,28 @@
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 106
+          "height": 118
         },
         "id": "-K2YJJzsVtfq",
-        "outputId": "a18f5b12-f90c-4d8d-e0b7-64350d76a086"
+        "outputId": "7754c508-264a-41fa-9843-83460719820f"
       },
       "source": [
         "textractor(\"txtai/article.pdf\")"
       ],
-      "execution_count": null,
+      "execution_count": 21,
       "outputs": [
         {
           "output_type": "execute_result",
           "data": {
+            "text/plain": [
+              "'Introducing txtai, an AI-powered search engine \\nbuilt on Transformers\\n\\nAdd Natural Language Understanding to any application\\n\\nSearch is the base of many applications. Once data starts to pile up, users want to be able to find it. It’s \\nthe foundation of the internet and an ever-growing challenge that is never solved or done.\\n\\nThe field of Natural Language Processing (NLP) is rapidly evolving with a number of new \\ndevelopments. Large-scale general language models are an exciting new capability allowing us to add \\namazing functionality quickly with limited compute and people. Innovation continues with new models\\nand advancements coming in at what seems a weekly basis.\\n\\nThis article introduces txtai, an AI-powered search engine that enables Natural Language \\nUnderstanding (NLU) based search in any application.\\n\\nIntroducing txtai\\ntxtai builds an AI-powered index over sections of text. txtai supports building text indices to perform \\nsimilarity searches and create extractive question-answering based systems. txtai also has functionality \\nfor zero-shot classification. txtai is open source and available on GitHub.\\n\\ntxtai and/or the concepts behind it has already been used to power the Natural Language Processing \\n(NLP) applications listed below:\\n\\n• paperai — AI-powered literature discovery and review engine for medical/scientific papers\\n• tldrstory — AI-powered understanding of headlines and story text\\n• neuspo — Fact-driven, real-time sports event and news site\\n• codequestion — Ask coding questions directly from the terminal\\n\\nBuild an Embeddings index\\nFor small lists of texts, the method above works. But for larger repositories of documents, it doesn’t \\nmake sense to tokenize and convert all embeddings for each query. txtai supports building pre-\\ncomputed indices which significantly improves performance.\\n\\nBuilding on the previous example, the following example runs an index method to build and store the \\ntext embeddings. In this case, only the query is converted to an embeddings vector each search.\\n\\nhttps://github.com/neuml/codequestion\\nhttps://neuspo.com/\\nhttps://github.com/neuml/tldrstory\\nhttps://github.com/neuml/paperai\\n - Introducing txtai, an AI-powered search engine built on Transformers\\n - Add Natural Language Understanding to any application\\n - Introducing txtai\\n - Build an Embeddings index'"
+            ],
             "application/vnd.google.colaboratory.intrinsic+json": {
               "type": "string"
-            },
-            "text/plain": [
-              "'Introducing txtai, an AI-powered search engine built on Transformers Add Natural Language Understanding to any application Search is the base of many applications. Once data starts to pile up, users want to be able to find it. It’s the foundation of the internet and an ever-growing challenge that is never solved or done. The field of Natural Language Processing (NLP) is rapidly evolving with a number of new developments. Large-scale general language models are an exciting new capability allowing us to add amazing functionality quickly with limited compute and people. Innovation continues with new models and advancements coming in at what seems a weekly basis. This article introduces txtai, an AI-powered search engine that enables Natural Language Understanding (NLU) based search in any application. Introducing txtai txtai builds an AI-powered index over sections of text. txtai supports building text indices to perform similarity searches and create extractive question-answering based systems. txtai also has functionality for zero-shot classification. txtai is open source and available on GitHub. txtai and/or the concepts behind it has already been used to power the Natural Language Processing (NLP) applications listed below: • paperai — AI-powered literature discovery and review engine for medical/scientific papers • tldrstory — AI-powered understanding of headlines and story text • neuspo — Fact-driven, real-time sports event and news site • codequestion — Ask coding questions directly from the terminal Build an Embeddings index For small lists of texts, the method above works. But for larger repositories of documents, it doesn’t make sense to tokenize and convert all embeddings for each query. txtai supports building pre- computed indices which significantly improves performance. Building on the previous example, the following example runs an index method to build and store the text embeddings. In this case, only the query is converted to an embeddings vector each search. https://github.com/neuml/codequestion https://neuspo.com/ https://github.com/neuml/tldrstory https://github.com/neuml/paperai Introducing txtai, an AI-powered search engine built on Transformers Add Natural Language Understanding to any application Introducing txtai Build an Embeddings index'"
-            ]
+            }
           },
           "metadata": {},
-          "execution_count": 3
+          "execution_count": 21
         }
       ]
     },
@@ -144,7 +140,9 @@
         "id": "1w2bhBCPOUdu"
       },
       "source": [
-        "# Extract sentences"
+        "# Extract sentences\n",
+        "\n",
+        "Sentence extraction uses a model that specializes in sentence detection. This call returns a list of sentences."
       ]
     },
     {
@@ -154,39 +152,39 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "PKZVK5vuOTqB",
-        "outputId": "cb8d688e-9d55-41a2-de26-a86d4e54b23d"
+        "outputId": "a31e182e-037b-4e29-c1b0-0f6815e3b2c9"
       },
       "source": [
         "textractor = Textractor(sentences=True)\n",
         "textractor(\"txtai/article.pdf\")"
       ],
-      "execution_count": null,
+      "execution_count": 22,
       "outputs": [
         {
           "output_type": "execute_result",
           "data": {
             "text/plain": [
-              "['Introducing txtai, an AI-powered search engine built on Transformers Add Natural Language Understanding to any application Search is the base of many applications.',\n",
+              "['Introducing txtai, an AI-powered search engine \\nbuilt on Transformers\\n\\nAdd Natural Language Understanding to any application\\n\\nSearch is the base of many applications.',\n",
               " 'Once data starts to pile up, users want to be able to find it.',\n",
-              " 'It’s the foundation of the internet and an ever-growing challenge that is never solved or done.',\n",
-              " 'The field of Natural Language Processing (NLP) is rapidly evolving with a number of new developments.',\n",
-              " 'Large-scale general language models are an exciting new capability allowing us to add amazing functionality quickly with limited compute and people.',\n",
-              " 'Innovation continues with new models and advancements coming in at what seems a weekly basis.',\n",
-              " 'This article introduces txtai, an AI-powered search engine that enables Natural Language Understanding (NLU) based search in any application.',\n",
-              " 'Introducing txtai txtai builds an AI-powered index over sections of text.',\n",
-              " 'txtai supports building text indices to perform similarity searches and create extractive question-answering based systems.',\n",
-              " 'txtai also has functionality for zero-shot classification.',\n",
+              " 'It’s \\nthe foundation of the internet and an ever-growing challenge that is never solved or done.',\n",
+              " 'The field of Natural Language Processing (NLP) is rapidly evolving with a number of new \\ndevelopments.',\n",
+              " 'Large-scale general language models are an exciting new capability allowing us to add \\namazing functionality quickly with limited compute and people.',\n",
+              " 'Innovation continues with new models\\nand advancements coming in at what seems a weekly basis.',\n",
+              " 'This article introduces txtai, an AI-powered search engine that enables Natural Language \\nUnderstanding (NLU) based search in any application.',\n",
+              " 'Introducing txtai\\ntxtai builds an AI-powered index over sections of text.',\n",
+              " 'txtai supports building text indices to perform \\nsimilarity searches and create extractive question-answering based systems.',\n",
+              " 'txtai also has functionality \\nfor zero-shot classification.',\n",
               " 'txtai is open source and available on GitHub.',\n",
-              " 'txtai and/or the concepts behind it has already been used to power the Natural Language Processing (NLP) applications listed below: • paperai — AI-powered literature discovery and review engine for medical/scientific papers • tldrstory — AI-powered understanding of headlines and story text • neuspo — Fact-driven, real-time sports event and news site • codequestion — Ask coding questions directly from the terminal Build an Embeddings index For small lists of texts, the method above works.',\n",
-              " 'But for larger repositories of documents, it doesn’t make sense to tokenize and convert all embeddings for each query.',\n",
-              " 'txtai supports building pre- computed indices which significantly improves performance.',\n",
-              " 'Building on the previous example, the following example runs an index method to build and store the text embeddings.',\n",
+              " 'txtai and/or the concepts behind it has already been used to power the Natural Language Processing \\n(NLP) applications listed below:\\n\\n• paperai — AI-powered literature discovery and review engine for medical/scientific papers\\n• tldrstory — AI-powered understanding of headlines and story text\\n• neuspo — Fact-driven, real-time sports event and news site\\n• codequestion — Ask coding questions directly from the terminal\\n\\nBuild an Embeddings index\\nFor small lists of texts, the method above works.',\n",
+              " 'But for larger repositories of documents, it doesn’t \\nmake sense to tokenize and convert all embeddings for each query.',\n",
+              " 'txtai supports building pre-\\ncomputed indices which significantly improves performance.',\n",
+              " 'Building on the previous example, the following example runs an index method to build and store the \\ntext embeddings.',\n",
               " 'In this case, only the query is converted to an embeddings vector each search.',\n",
-              " 'https://github.com/neuml/codequestion https://neuspo.com/ https://github.com/neuml/tldrstory https://github.com/neuml/paperai Introducing txtai, an AI-powered search engine built on Transformers Add Natural Language Understanding to any application Introducing txtai Build an Embeddings index']"
+              " 'https://github.com/neuml/codequestion\\nhttps://neuspo.com/\\nhttps://github.com/neuml/tldrstory\\nhttps://github.com/neuml/paperai\\n - Introducing txtai, an AI-powered search engine built on Transformers\\n - Add Natural Language Understanding to any application\\n - Introducing txtai\\n - Build an Embeddings index']"
             ]
           },
           "metadata": {},
-          "execution_count": 4
+          "execution_count": 22
         }
       ]
     },
@@ -205,7 +203,9 @@
         "id": "z1H8XYkaSoP4"
       },
       "source": [
-        "# Extract paragraphs"
+        "# Extract paragraphs\n",
+        "\n",
+        "Paragraph detection looks for consecutive newlines. This call returns a list of paragraphs."
       ]
     },
     {
@@ -215,35 +215,146 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "9VUito4ISoAe",
-        "outputId": "21770771-faa4-4340-a1ab-7fb0a8177f4c"
+        "outputId": "08079380-a7c8-4886-ecc3-de9f02be4584"
       },
       "source": [
         "textractor = Textractor(paragraphs=True)\n",
-        "textractor(\"txtai/article.pdf\")"
+        "for paragraph in textractor(\"txtai/article.pdf\"):\n",
+        "  print(paragraph, \"\\n----\")"
       ],
-      "execution_count": null,
+      "execution_count": 23,
       "outputs": [
         {
-          "output_type": "execute_result",
-          "data": {
-            "text/plain": [
-              "['Introducing txtai, an AI-powered search engine built on Transformers',\n",
-              " 'Add Natural Language Understanding to any application',\n",
-              " 'Search is the base of many applications. Once data starts to pile up, users want to be able to find it. It’s the foundation of the internet and an ever-growing challenge that is never solved or done.',\n",
-              " 'The field of Natural Language Processing (NLP) is rapidly evolving with a number of new developments. Large-scale general language models are an exciting new capability allowing us to add amazing functionality quickly with limited compute and people. Innovation continues with new models and advancements coming in at what seems a weekly basis.',\n",
-              " 'This article introduces txtai, an AI-powered search engine that enables Natural Language Understanding (NLU) based search in any application.',\n",
-              " 'Introducing txtai txtai builds an AI-powered index over sections of text. txtai supports building text indices to perform similarity searches and create extractive question-answering based systems. txtai also has functionality for zero-shot classification. txtai is open source and available on GitHub.',\n",
-              " 'txtai and/or the concepts behind it has already been used to power the Natural Language Processing (NLP) applications listed below:',\n",
-              " '• paperai — AI-powered literature discovery and review engine for medical/scientific papers • tldrstory — AI-powered understanding of headlines and story text • neuspo — Fact-driven, real-time sports event and news site • codequestion — Ask coding questions directly from the terminal',\n",
-              " 'Build an Embeddings index For small lists of texts, the method above works. But for larger repositories of documents, it doesn’t make sense to tokenize and convert all embeddings for each query. txtai supports building pre- computed indices which significantly improves performance.',\n",
-              " 'Building on the previous example, the following example runs an index method to build and store the text embeddings. In this case, only the query is converted to an embeddings vector each search.',\n",
-              " 'https://github.com/neuml/codequestion https://neuspo.com/ https://github.com/neuml/tldrstory https://github.com/neuml/paperai',\n",
-              " 'Introducing txtai, an AI-powered search engine built on Transformers Add Natural Language Understanding to any application',\n",
-              " 'Introducing txtai Build an Embeddings index']"
-            ]
-          },
-          "metadata": {},
-          "execution_count": 5
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Introducing txtai, an AI-powered search engine \n",
+            "built on Transformers \n",
+            "----\n",
+            "Add Natural Language Understanding to any application \n",
+            "----\n",
+            "Search is the base of many applications. Once data starts to pile up, users want to be able to find it. It’s \n",
+            "the foundation of the internet and an ever-growing challenge that is never solved or done. \n",
+            "----\n",
+            "The field of Natural Language Processing (NLP) is rapidly evolving with a number of new \n",
+            "developments. Large-scale general language models are an exciting new capability allowing us to add \n",
+            "amazing functionality quickly with limited compute and people. Innovation continues with new models\n",
+            "and advancements coming in at what seems a weekly basis. \n",
+            "----\n",
+            "This article introduces txtai, an AI-powered search engine that enables Natural Language \n",
+            "Understanding (NLU) based search in any application. \n",
+            "----\n",
+            "Introducing txtai\n",
+            "txtai builds an AI-powered index over sections of text. txtai supports building text indices to perform \n",
+            "similarity searches and create extractive question-answering based systems. txtai also has functionality \n",
+            "for zero-shot classification. txtai is open source and available on GitHub. \n",
+            "----\n",
+            "txtai and/or the concepts behind it has already been used to power the Natural Language Processing \n",
+            "(NLP) applications listed below: \n",
+            "----\n",
+            "• paperai — AI-powered literature discovery and review engine for medical/scientific papers\n",
+            "• tldrstory — AI-powered understanding of headlines and story text\n",
+            "• neuspo — Fact-driven, real-time sports event and news site\n",
+            "• codequestion — Ask coding questions directly from the terminal \n",
+            "----\n",
+            "Build an Embeddings index\n",
+            "For small lists of texts, the method above works. But for larger repositories of documents, it doesn’t \n",
+            "make sense to tokenize and convert all embeddings for each query. txtai supports building pre-\n",
+            "computed indices which significantly improves performance. \n",
+            "----\n",
+            "Building on the previous example, the following example runs an index method to build and store the \n",
+            "text embeddings. In this case, only the query is converted to an embeddings vector each search. \n",
+            "----\n",
+            "https://github.com/neuml/codequestion\n",
+            "https://neuspo.com/\n",
+            "https://github.com/neuml/tldrstory\n",
+            "https://github.com/neuml/paperai\n",
+            " - Introducing txtai, an AI-powered search engine built on Transformers\n",
+            " - Add Natural Language Understanding to any application\n",
+            " - Introducing txtai\n",
+            " - Build an Embeddings index \n",
+            "----\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Extract sections\n",
+        "\n",
+        "Section extraction is format dependent. If page breaks are available, each section is a page. Otherwise, this call returns logical sections such by headings."
+      ],
+      "metadata": {
+        "id": "Ae6dRQ2LvN-w"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "textractor = Textractor(sections=True)\n",
+        "print(\"\\n[PAGE BREAK]\\n\".join(section for section in textractor(\"txtai/article.pdf\")))"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "nQ6ev2UMwqnh",
+        "outputId": "3d45491d-3547-4218-d30c-ba0c4d161256"
+      },
+      "execution_count": 24,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Introducing txtai, an AI-powered search engine \n",
+            "built on Transformers\n",
+            "\n",
+            "Add Natural Language Understanding to any application\n",
+            "\n",
+            "Search is the base of many applications. Once data starts to pile up, users want to be able to find it. It’s \n",
+            "the foundation of the internet and an ever-growing challenge that is never solved or done.\n",
+            "\n",
+            "The field of Natural Language Processing (NLP) is rapidly evolving with a number of new \n",
+            "developments. Large-scale general language models are an exciting new capability allowing us to add \n",
+            "amazing functionality quickly with limited compute and people. Innovation continues with new models\n",
+            "and advancements coming in at what seems a weekly basis.\n",
+            "\n",
+            "This article introduces txtai, an AI-powered search engine that enables Natural Language \n",
+            "Understanding (NLU) based search in any application.\n",
+            "\n",
+            "Introducing txtai\n",
+            "txtai builds an AI-powered index over sections of text. txtai supports building text indices to perform \n",
+            "similarity searches and create extractive question-answering based systems. txtai also has functionality \n",
+            "for zero-shot classification. txtai is open source and available on GitHub.\n",
+            "\n",
+            "txtai and/or the concepts behind it has already been used to power the Natural Language Processing \n",
+            "(NLP) applications listed below:\n",
+            "\n",
+            "• paperai — AI-powered literature discovery and review engine for medical/scientific papers\n",
+            "• tldrstory — AI-powered understanding of headlines and story text\n",
+            "• neuspo — Fact-driven, real-time sports event and news site\n",
+            "• codequestion — Ask coding questions directly from the terminal\n",
+            "\n",
+            "Build an Embeddings index\n",
+            "For small lists of texts, the method above works. But for larger repositories of documents, it doesn’t \n",
+            "make sense to tokenize and convert all embeddings for each query. txtai supports building pre-\n",
+            "computed indices which significantly improves performance.\n",
+            "\n",
+            "Building on the previous example, the following example runs an index method to build and store the \n",
+            "text embeddings. In this case, only the query is converted to an embeddings vector each search.\n",
+            "\n",
+            "https://github.com/neuml/codequestion\n",
+            "https://neuspo.com/\n",
+            "https://github.com/neuml/tldrstory\n",
+            "https://github.com/neuml/paperai\n",
+            "[PAGE BREAK]\n",
+            "- Introducing txtai, an AI-powered search engine built on Transformers\n",
+            " - Add Natural Language Understanding to any application\n",
+            " - Introducing txtai\n",
+            " - Build an Embeddings index\n"
+          ]
         }
       ]
     }

examples/11_Transcribe_audio_to_text.ipynb

@@ -3,8 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -20,8 +19,6 @@
       "source": [
         "# Transcribe audio to text\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "This notebook covers the transcription of audio files to text using models provided by Hugging Face."
       ]
     },
@@ -49,7 +46,7 @@
         "!wget -N https://github.com/neuml/txtai/releases/download/v3.5.0/tests.tar.gz\n",
         "!tar -xvzf tests.tar.gz"
       ],
-      "execution_count": 8,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -79,7 +76,7 @@
         "# Create transcription model\n",
         "transcribe = Transcription()"
       ],
-      "execution_count": 9,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -114,7 +111,7 @@
         "  print(text)\n",
         "  print()\n"
       ],
-      "execution_count": 10,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "display_data",
@@ -303,7 +300,7 @@
         "id": "-KgYwAQzFVll",
         "outputId": "7b1e9541-1a6f-4814-ae14-21f4bd9794e7"
       },
-      "execution_count": 11,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -356,13 +353,13 @@
         "id": "k3Rb2OU9Mq3h",
         "outputId": "5e40ecd5-3df1-4864-c7a3-3e26bd087415"
       },
-      "execution_count": 12,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
           "name": "stdout",
           "text": [
-            "Writing workflow.yml\n"
+            "Overwriting workflow.yml\n"
           ]
         }
       ]
@@ -384,7 +381,7 @@
         "id": "cj49U095IxTc",
         "outputId": "a55b8717-f5fc-4e8f-ef1b-5fecc3f8200a"
       },
-      "execution_count": 26,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -431,7 +428,7 @@
         "id": "XDpqwf8PNHNo",
         "outputId": "7ec28122-69a3-47cc-d2ed-acaa186e4aa1"
       },
-      "execution_count": 29,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -463,4 +460,4 @@
       }
     }
   ]
-}
+}

examples/12_Translate_text_between_languages.ipynb

@@ -3,8 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "display_name": "Python 3",
@@ -20,8 +19,6 @@
       "source": [
         "# Translate text between languages\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "This notebook covers machine translation backed by Hugging Face models. The quality of machine translation via cloud services has come a very long way and produces high quality results. This notebook shows how the models from Hugging Face give developers a reasonable alternative for local machine translation."
       ]
     },
@@ -45,7 +42,7 @@
         "%%capture\n",
         "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline]"
       ],
-      "execution_count": 1,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -75,7 +72,7 @@
         "# Create translation model\n",
         "translate = Translation()"
       ],
-      "execution_count": 14,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -103,7 +100,7 @@
         "translation = translate(\"This is a test translation into Spanish\", \"es\")\n",
         "translation"
       ],
-      "execution_count": 16,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -133,7 +130,7 @@
       "source": [
         "translate(translation, \"en\")"
       ],
-      "execution_count": 17,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -185,7 +182,7 @@
         "# Run multiple translations\n",
         "run()"
       ],
-      "execution_count": 18,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -251,7 +248,7 @@
         "id": "REb0X2Kz60Ew",
         "outputId": "ae13d4d0-318f-4740-f725-0029ceeabeac"
       },
-      "execution_count": 11,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -291,7 +288,7 @@
         "id": "d5usam_jhKaz",
         "outputId": "c12368a0-ea26-4191-be5a-f2de70711003"
       },
-      "execution_count": 12,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -330,7 +327,7 @@
         "id": "JkOLnvKZWI95",
         "outputId": "fef6402c-e20e-43e1-a3eb-e4580913fa7e"
       },
-      "execution_count": 13,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",

examples/14_Run_pipeline_workflows.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "14 - Run pipeline workflows",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -22,8 +20,6 @@
       "source": [
         "# Run pipeline workflows\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "txtai has a growing list of models available through it's pipeline framework. Pipelines wrap a machine learning model and transform data. Currently, pipelines can wrap Hugging Face models, Hugging Face pipelines or PyTorch models (support for TensorFlow is in the backlog).\n",
         "\n",
         "The following is a list of the currently implemented pipelines.\n",
@@ -64,7 +60,7 @@
         "!wget -N https://github.com/neuml/txtai/releases/download/v2.0.0/tests.tar.gz\n",
         "!tar -xvzf tests.tar.gz"
       ],
-      "execution_count": 11,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -97,7 +93,7 @@
         "# Create a translation instance\n",
         "translate = Translation()"
       ],
-      "execution_count": 12,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -132,7 +128,7 @@
         "# Workflows are generators for efficiency, read results to list for display\n",
         "list(workflow(data))"
       ],
-      "execution_count": 13,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -201,7 +197,7 @@
         "# Run workflow\n",
         "list(workflow(data))"
       ],
-      "execution_count": 14,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -288,7 +284,7 @@
         "# Cleanup temporary storage\n",
         "documents.close()"
       ],
-      "execution_count": 15,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -327,7 +323,7 @@
         "# Run a search query and show the result.\n",
         "embeddings.search(\"changement climatique\", 1)[0]"
       ],
-      "execution_count": 16,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -356,7 +352,7 @@
         "# Run a search query and show the result.\n",
         "embeddings.search(\"traitement du langage naturel\", 1)[0]"
       ],
-      "execution_count": 17,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -437,7 +433,7 @@
         "id": "HoVlk_vNJKHY",
         "outputId": "34b68bcb-a6d5-4029-9bf2-f33e4381d1bc"
       },
-      "execution_count": 18,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -470,7 +466,7 @@
         "id": "a_klVZAXHJcw",
         "outputId": "33229268-0f98-4ca1-af7d-212bcbde6482"
       },
-      "execution_count": 19,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -499,7 +495,7 @@
         "id": "I5xin0VNHJOu",
         "outputId": "2fbb9a93-b860-437d-c361-ee21eed75b6b"
       },
-      "execution_count": 20,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",

examples/15_Distributed_embeddings_cluster.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "15 - Distributed embeddings cluster",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -21,8 +19,6 @@
       "source": [
         "# Distributed embeddings cluster\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "The txtai API is a web-based service backed by [FastAPI](https://fastapi.tiangolo.com/). All txtai functionality is available via the API. The API can also cluster multiple embeddings indices into a single logical index to horizontally scale over multiple nodes. \n",
         "\n",
         "This notebook installs the txtai API and shows an example of building an embeddings cluster."

examples/16_Train_a_text_labeler.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "16 - Train a text labeler",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -22,8 +20,6 @@
       "source": [
         "# Train a text labeler\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "The [Hugging Face Model Hub](https://huggingface.co/models) has a wide range of models that can handle many tasks. While these models perform well, the best performance often is found when fine-tuning a model with task-specific data. \n",
         "\n",
         "Hugging Face provides a [number of full-featured examples](https://github.com/huggingface/transformers/tree/master/examples) available to assist with training task-specific models. When building models from the command line, these scripts are a great way to get started.\n",
@@ -56,7 +52,7 @@
       },
       "source": [
         "%%capture\n",
-        "!pip install git+https://github.com/neuml/txtai datasets pandas"
+        "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline-train] datasets pandas"
       ],
       "execution_count": null,
       "outputs": []

examples/17_Train_without_labels.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "17 - Train without labels",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -22,8 +20,6 @@
       "source": [
         "# Train without labels\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "Almost all data available is unlabeled. Labeled data takes effort to manually review and/or takes time to collect. Zero-shot classification takes existing large language models and runs a similarity comparison between candidate text and a list of labels. This has been shown to perform surprisingly well.\n",
         "\n",
         "The problem with zero-shot classifiers is that they need to have a large number of parameters (400M+) to perform well against general tasks, which comes with sizable hardware requirements.\n",
@@ -49,7 +45,7 @@
       },
       "source": [
         "%%capture\n",
-        "!pip install git+https://github.com/neuml/txtai datasets pandas"
+        "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline-train] datasets pandas"
       ],
       "execution_count": null,
       "outputs": []

examples/18_Export_and_run_models_with_ONNX.ipynb

@@ -21,8 +21,6 @@
       "source": [
         "# Export and run models with ONNX\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "The [ONNX runtime](https://onnx.ai/) provides a common serialization format for machine learning models. ONNX supports a number of [different platforms/languages](https://onnxruntime.ai/docs/how-to/install.html#requirements) and has features built in to help reduce inference time. \n",
         "\n",
         "PyTorch has robust support for exporting Torch models to ONNX. This enables exporting Hugging Face Transformer and/or other downstream models directly to ONNX. \n",
@@ -1351,4 +1349,4 @@
       ]
     }
   ]
-}
+}

examples/19_Train_a_QA_model.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "19 - Train a QA model",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -22,8 +20,6 @@
       "source": [
         "# Train a QA model\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "The [Hugging Face Model Hub](https://huggingface.co/models) has a wide range of models that can handle many tasks. While these models perform well, the best performance is often found when fine-tuning a model with task-specific data. \n",
         "\n",
         "Hugging Face provides a [number of full-featured examples](https://github.com/huggingface/transformers/tree/master/examples) to assist with training task-specific models. When building models from the command line, these scripts are a great way to get started.\n",
@@ -51,7 +47,7 @@
       },
       "source": [
         "%%capture\n",
-        "!pip install git+https://github.com/neuml/txtai datasets pandas"
+        "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline-train] datasets pandas"
       ],
       "execution_count": null,
       "outputs": []
@@ -296,4 +292,4 @@
       ]
     }
   ]
-}
+}

examples/20_Extractive_QA_to_build_structured_data.ipynb

@@ -3,9 +3,7 @@
   "nbformat_minor": 0,
   "metadata": {
     "colab": {
-      "name": "20 - Extractive QA to build structured data",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -21,8 +19,6 @@
       "source": [
         "# Extractive QA to build structured data\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "Traditional ETL/data parsing systems establish rules to extract information of interest. Regular expressions, string parsing and similar methods define fixed rules. This works in many cases but what if you are working with unstructured data containing numerous variations? The rules can be cumbersome and hard to maintain over time.\n",
         "\n",
         "This notebook uses machine learning and extractive question-answering (QA) to utilize the vast knowledge built into large language models. These models have been trained on extremely large datasets, learning the many variations of natural language. "
@@ -46,7 +42,7 @@
       },
       "source": [
         "%%capture\n",
-        "!pip install git+https://github.com/neuml/txtai"
+        "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline-train]"
       ],
       "execution_count": null,
       "outputs": []
@@ -530,4 +526,4 @@
       ]
     }
   ]
-}
+}

examples/21_Export_and_run_other_machine_learning_models.ipynb

@@ -22,8 +22,6 @@
       "source": [
         "# Export and run other machine learning models\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "txtai primarily has support for [Hugging Face Transformers](https://github.com/huggingface/transformers) and [ONNX](https://github.com/microsoft/onnxruntime) models. This enables txtai to hook into the rich model framework available in Python, export this functionality via the API to other languages (JavaScript, Java, Go, Rust) and even export and natively load models with ONNX.\n",
         "\n",
         "What about other machine learning frameworks? Say we have an existing TF-IDF + Logistic Regression model that has been well tuned. Can this model be exported to ONNX and used in txtai for labeling and similarity queries? Or what about a simple PyTorch text classifier? Yes, both of these can be done!\n",

examples/22_Transform_tabular_data_with_composable_workflows.ipynb

@@ -22,9 +22,7 @@
       "source": [
         "# Transform tabular data with composable workflows\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
-        "txtai executes machine-learning workflows to transform data and build AI-powered semantic search applications. txtai has support for processing both unstructured and structured data. Structured or tabular data is grouped into rows and columns. This can be a spreadsheet, an API call that returns JSON or XML or even list of key-value pairs.\n",
+        "txtai has support for processing both unstructured and structured data. Structured or tabular data is grouped into rows and columns. This can be a spreadsheet, an API call that returns JSON or XML or even list of key-value pairs.\n",
         "\n",
         "This notebook will walk through examples on how to use workflows with the tabular pipeline to transform and index structured data.\n"
       ]

examples/23_Tensor_workflows.ipynb

@@ -22,8 +22,6 @@
       "source": [
         "# Tensor workflows\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "Many of the examples and use cases for txtai focus on transforming text. Makes sense as txt is even in the name! But that doesn't mean txtai only works with text.\n",
         "\n",
         "This notebook will cover examples of how to efficiently process tensors using txtai workflows."

examples/24_Whats_new_in_txtai_4_0.ipynb

@@ -34,10 +34,6 @@
       "source": [
         "# 💡 What's new in txtai 4.0\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
-        "[txtai](https://github.com/neuml/txtai) executes machine-learning workflows to transform data and build AI-powered semantic search applications.\n",
-        "\n",
         "txtai 4.0 brings a number of major feature enhancements, most importantly the capability to store full document content and text right in txtai. This notebook will cover all the changes with examples."
       ]
     },
@@ -573,4 +569,4 @@
       ]
     }
   ]
-}
+}

examples/25_Generate_image_captions_and_detect_objects.ipynb

@@ -8,8 +8,6 @@
       "source": [
         "# Generate image captions and detect objects\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "txtai as the name implies works with text and ai, pretty straightforward. But that doesn't mean it can't work with different types of content. For example, an image can be described with words. We can use that description to compare an image to a query or other documents. This notebook shows how images and text can be embedded into the same space to generate image captions and detect objects."
       ]
     },

examples/26_Entity_extraction_workflows.ipynb

@@ -34,8 +34,6 @@
       "source": [
         "# Entity extraction workflows\n",
         "\n",
-        "_This notebook is part of a tutorial series on [txtai](https://github.com/neuml/txtai), an AI-powered semantic search platform._\n",
-        "\n",
         "Entity extraction is the process of identifying names, locations, organizations and other entity-like tokens in unstructured text. Entity extraction can organize data into topics and/or feed downstream machine learning pipelines.\n",
         "\n",
         "This notebook will show how to use the entity extraction pipeline in txtai with workflows."